+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-103.ec2.internal ++ [[ -n ip-10-0-2-103.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-103.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0327 23:01:49.900425756 - disable conntrack on geneve port' I0327 23:01:49.900425756 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0327 23:01:49.917072427 - starting ovnkube-node' I0327 23:01:49.917072427 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-103.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-103.ec2.internal --init-node ip-10-0-2-103.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-103.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0327 23:01:49.956395 3881 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0327 23:01:49.956439 3881 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-134f8e697a.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0327 23:01:49.958285 3881 kube.go:419] Waiting for certificate I0327 23:01:49.958330 3881 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0327 23:01:49.958371 3881 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0327 23:01:49.959211 3881 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 23:01:49.983113 3881 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 23:01:49.983132 3881 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 23:01:49.985803 3881 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 23:01:49.988313 3881 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-mmv4z" I0327 23:01:49.995590 3881 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-mmv4z" I0327 23:01:49.995672 3881 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 23:01:50.958400 3881 kube.go:426] Certificate found I0327 23:01:50.959126 3881 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 23:01:50.960394 3881 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0327 23:01:51.003441 3881 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-28 22:56:49 +0000 UTC" deadline="2026-03-28 18:47:59.516559141 +0000 UTC" I0327 23:01:51.003489 3881 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h46m8.513085496s" I0327 23:01:51.024695 3881 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 23:01:51.041277 3881 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0327 23:01:51.042368 3881 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0327 23:01:51.042570 3881 factory.go:531] Starting watch factory I0327 23:01:51.042676 3881 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.042741 3881 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.043091 3881 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.043150 3881 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.043653 3881 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.043716 3881 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.044010 3881 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.044027 3881 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.044246 3881 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.044296 3881 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.044460 3881 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.044484 3881 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.051592 3881 controller_manager.go:368] Starting the ovnkube controller I0327 23:01:51.051602 3881 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-103.ec2.internal I0327 23:01:51.051643 3881 controller_manager.go:393] NBDB zone sync took: 34.636µs I0327 23:01:51.051652 3881 factory.go:531] Starting watch factory I0327 23:01:51.082035 3881 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.082075 3881 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.082109 3881 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.082227 3881 ovn_northd.go:121] Registering OVN northd metrics I0327 23:01:51.082308 3881 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.082500 3881 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0327 23:01:51.082602 3881 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.086690 3881 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0327 23:01:51.086713 3881 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0327 23:01:51.090267 3881 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 23:01:51.101592 3881 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0327 23:01:51.148108 3881 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 23:01:51.148138 3881 factory.go:1890] *v1.Node informer cache synced successfully I0327 23:01:51.148142 3881 factory.go:1890] *v1.Service informer cache synced successfully I0327 23:01:51.148145 3881 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 23:01:51.148149 3881 factory.go:1890] *v1.Pod informer cache synced successfully I0327 23:01:51.148151 3881 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 23:01:51.148223 3881 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 23:01:51.148237 3881 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 23:01:51.148244 3881 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 23:01:51.148276 3881 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 23:01:51.152190 3881 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 23:01:51.152193 3881 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 23:01:51.154121 3881 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 23:01:51.154131 3881 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 23:01:51.154237 3881 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.154252 3881 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.157064 3881 factory.go:1890] *v1.Pod informer cache synced successfully I0327 23:01:51.157075 3881 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 23:01:51.157078 3881 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 23:01:51.157081 3881 factory.go:1890] *v1.Node informer cache synced successfully I0327 23:01:51.157084 3881 factory.go:1890] *v1.Service informer cache synced successfully I0327 23:01:51.157087 3881 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 23:01:51.157294 3881 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.159313 3881 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 23:01:51.159418 3881 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.159430 3881 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.162265 3881 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 23:01:51.162278 3881 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 23:01:51.162549 3881 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.164548 3881 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 23:01:51.164605 3881 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.164621 3881 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.167328 3881 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 23:01:51.167641 3881 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.169600 3881 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 23:01:51.169712 3881 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.169727 3881 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.172602 3881 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 23:01:51.172858 3881 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.174866 3881 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 23:01:51.175004 3881 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.175019 3881 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.177814 3881 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 23:01:51.178060 3881 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.180039 3881 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 23:01:51.180193 3881 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 23:01:51.180212 3881 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 23:01:51.183388 3881 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 23:01:51.183473 3881 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 23:01:51.185537 3881 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 23:01:51.185604 3881 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.185616 3881 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.185665 3881 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.185687 3881 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.188506 3881 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 23:01:51.188530 3881 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.188663 3881 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 23:01:51.190596 3881 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 23:01:51.190609 3881 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 23:01:51.190618 3881 factory.go:636] Watch Factory start up complete, took: 148.054089ms I0327 23:01:51.190676 3881 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0327 23:01:51.193877 3881 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 23:01:51.199054 3881 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 23:01:51.199063 3881 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 23:01:51.199071 3881 factory.go:636] Watch Factory start up complete, took: 147.416782ms I0327 23:01:51.199081 3881 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-2-103.ec2.internal" zone I0327 23:01:51.233722 3881 default_node_network_controller.go:709] Initializing the default node network controller I0327 23:01:51.279752 3881 udn_isolation.go:101] Starting UDN host isolation manager I0327 23:01:51.285979 3881 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0327 23:01:51.329313 3881 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0327 23:01:51.329384 3881 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0327 23:01:51.329398 3881 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0327 23:01:51.360671 3881 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0327 23:01:51.360728 3881 default_node_network_controller.go:808] Node ip-10-0-2-103.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0327 23:01:51.365450 3881 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.103"] k8s.ovn.org/zone-name:ip-10-0-2-103.ec2.internal] on node ip-10-0-2-103.ec2.internal I0327 23:01:51.387201 3881 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0327 23:01:51.392347 3881 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0327 23:01:51.392478 3881 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0327 23:01:51.392607 3881 gateway_init.go:263] Preparing Gateway I0327 23:01:51.392618 3881 gateway_shared_intf.go:1658] Creating new gateway I0327 23:01:51.449567 3881 controller_manager.go:427] Waiting for node in zone sync took: 250.475196ms I0327 23:01:51.455519 3881 controller_manager.go:608] Switching to transit router for layer2 networks I0327 23:01:51.455531 3881 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-103.ec2.internal I0327 23:01:51.461533 3881 iptables.go:108] Creating table: filter chain: FORWARD I0327 23:01:51.467389 3881 iptables.go:108] Creating table: filter chain: OUTPUT I0327 23:01:51.469989 3881 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 23:01:51.473341 3881 gateway_shared_intf.go:1787] Gateway Creation Complete I0327 23:01:51.473353 3881 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-103.ec2.internal","mac-address":"0e:17:2b:83:01:3f","ip-addresses":["10.0.2.103/24"],"ip-address":"10.0.2.103/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:06deb6dd-411c-4894-93ce-06db4d6915c3 k8s.ovn.org/node-encap-ips:["10.0.2.103"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.103/24"} k8s.ovn.org/zone-name:ip-10-0-2-103.ec2.internal] on node ip-10-0-2-103.ec2.internal I0327 23:01:51.474990 3881 controller_manager.go:288] SCTP support detected in OVN I0327 23:01:51.475492 3881 services_controller.go:69] Creating services controller for network=default I0327 23:01:51.475589 3881 udn_enabled_svc.go:80] Waiting for service informer to sync I0327 23:01:51.475648 3881 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0327 23:01:51.475699 3881 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0327 23:01:51.475720 3881 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0327 23:01:51.475731 3881 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 23:01:51.475922 3881 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0327 23:01:51.475977 3881 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0327 23:01:51.475996 3881 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 41.792µs I0327 23:01:51.476010 3881 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 23.539µs I0327 23:01:51.476019 3881 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 41.859µs I0327 23:01:51.476024 3881 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0327 23:01:51.476033 3881 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0327 23:01:51.476041 3881 default_network_controller.go:321] Starting the default network controller I0327 23:01:51.476059 3881 udn_enabled_svc.go:95] Performing full resync I0327 23:01:51.476193 3881 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 13.556µs I0327 23:01:51.476290 3881 udn_enabled_svc.go:99] Waiting for handler to sync I0327 23:01:51.476299 3881 udn_enabled_svc.go:104] Starting worker I0327 23:01:51.483333 3881 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0327 23:01:51.483396 3881 repair.go:33] Syncing exgw routes took 51.292µs I0327 23:01:51.483414 3881 default_network_controller.go:407] Starting all the Watchers... I0327 23:01:51.483557 3881 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0327 23:01:51.483564 3881 namespace.go:98] [openshift-backplane-tam] adding namespace I0327 23:01:51.483564 3881 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0327 23:01:51.483564 3881 namespace.go:98] [openshift-backplane] adding namespace I0327 23:01:51.483581 3881 namespace.go:98] [openshift-node] adding namespace I0327 23:01:51.483586 3881 namespace.go:98] [openshift-console-user-settings] adding namespace I0327 23:01:51.483559 3881 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0327 23:01:51.483662 3881 namespace.go:98] [openshift-config] adding namespace I0327 23:01:51.483669 3881 namespace.go:98] [openshift-etcd] adding namespace I0327 23:01:51.483672 3881 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0327 23:01:51.483674 3881 namespace.go:98] [openshift-backplane-cee] adding namespace I0327 23:01:51.483679 3881 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0327 23:01:51.483682 3881 namespace.go:98] [openshift-authentication] adding namespace I0327 23:01:51.483681 3881 namespace.go:98] [openshift-authentication-operator] adding namespace I0327 23:01:51.483687 3881 namespace.go:98] [openshift-apiserver-operator] adding namespace I0327 23:01:51.484212 3881 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 642.333µs I0327 23:01:51.484225 3881 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0327 23:01:51.484636 3881 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 405.939µs I0327 23:01:51.484647 3881 namespace.go:98] [openshift-host-network] adding namespace I0327 23:01:51.485252 3881 namespace.go:102] [openshift-host-network] adding namespace took 595.614µs I0327 23:01:51.485268 3881 namespace.go:98] [openshift-image-registry] adding namespace I0327 23:01:51.485675 3881 namespace.go:102] [openshift-backplane-tam] adding namespace took 2.100208ms I0327 23:01:51.485691 3881 namespace.go:98] [openshift-operators-redhat] adding namespace I0327 23:01:51.486171 3881 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 2.592008ms I0327 23:01:51.486197 3881 namespace.go:98] [openshift-machine-api] adding namespace I0327 23:01:51.486618 3881 namespace.go:102] [openshift-backplane] adding namespace took 3.037319ms I0327 23:01:51.486629 3881 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0327 23:01:51.487088 3881 namespace.go:102] [openshift-node] adding namespace took 3.495963ms I0327 23:01:51.487108 3881 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0327 23:01:51.487562 3881 namespace.go:102] [openshift-console-user-settings] adding namespace took 3.965992ms I0327 23:01:51.487582 3881 namespace.go:98] [default] adding namespace I0327 23:01:51.488036 3881 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 4.382218ms I0327 23:01:51.488050 3881 namespace.go:98] [openshift-backplane-mobb] adding namespace I0327 23:01:51.488470 3881 namespace.go:102] [openshift-config] adding namespace took 4.794584ms I0327 23:01:51.488490 3881 namespace.go:98] [openshift-network-diagnostics] adding namespace I0327 23:01:51.488957 3881 namespace.go:102] [openshift-etcd] adding namespace took 5.278347ms I0327 23:01:51.488975 3881 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0327 23:01:51.489438 3881 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 5.759061ms I0327 23:01:51.489452 3881 namespace.go:98] [openshift-machine-config-operator] adding namespace I0327 23:01:51.489908 3881 namespace.go:102] [openshift-backplane-cee] adding namespace took 6.222247ms I0327 23:01:51.489926 3881 namespace.go:98] [openshift-controller-manager] adding namespace I0327 23:01:51.490446 3881 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.759441ms I0327 23:01:51.490464 3881 namespace.go:98] [openshift-customer-monitoring] adding namespace I0327 23:01:51.490916 3881 namespace.go:102] [openshift-authentication] adding namespace took 7.224876ms I0327 23:01:51.490932 3881 namespace.go:98] [openshift-must-gather-operator] adding namespace I0327 23:01:51.491338 3881 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.646174ms I0327 23:01:51.491353 3881 namespace.go:98] [openshift-console-operator] adding namespace I0327 23:01:51.491832 3881 namespace.go:102] [openshift-apiserver-operator] adding namespace took 8.136152ms I0327 23:01:51.491849 3881 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0327 23:01:51.492277 3881 namespace.go:102] [openshift-image-registry] adding namespace took 7.002458ms I0327 23:01:51.492292 3881 namespace.go:98] [openshift-dns-operator] adding namespace I0327 23:01:51.492663 3881 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.96516ms I0327 23:01:51.492678 3881 namespace.go:98] [openshift-backplane-cse] adding namespace I0327 23:01:51.493108 3881 namespace.go:102] [openshift-machine-api] adding namespace took 6.90422ms I0327 23:01:51.493136 3881 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0327 23:01:51.493530 3881 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.893921ms I0327 23:01:51.493545 3881 namespace.go:98] [openshift-multus] adding namespace I0327 23:01:51.494052 3881 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.93761ms I0327 23:01:51.494067 3881 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0327 23:01:51.494576 3881 namespace.go:102] [default] adding namespace took 6.984509ms I0327 23:01:51.494590 3881 namespace.go:98] [openshift-backplane-srep-ro] adding namespace W0327 23:01:51.494967 3881 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-67rsf: pod openshift-network-diagnostics/network-check-target-67rsf: no pod IPs found W0327 23:01:51.494981 3881 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-9ptpd: pod openshift-network-diagnostics/network-check-target-9ptpd: no pod IPs found W0327 23:01:51.494989 3881 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-gnmgz: pod openshift-network-diagnostics/network-check-target-gnmgz: no pod IPs found I0327 23:01:51.495008 3881 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.954698ms I0327 23:01:51.495017 3881 namespace.go:98] [openshift-console] adding namespace I0327 23:01:51.495460 3881 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.963027ms I0327 23:01:51.495475 3881 namespace.go:98] [openshift-infra] adding namespace I0327 23:01:51.495910 3881 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.927407ms I0327 23:01:51.495924 3881 namespace.go:98] [kube-public] adding namespace I0327 23:01:51.496394 3881 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.920361ms I0327 23:01:51.496407 3881 namespace.go:98] [openshift-logging] adding namespace I0327 23:01:51.496634 3881 default_node_network_controller.go:518] Node connection status = connected I0327 23:01:51.496953 3881 namespace.go:102] [openshift-controller-manager] adding namespace took 7.020762ms I0327 23:01:51.496968 3881 namespace.go:98] [openshift-network-operator] adding namespace I0327 23:01:51.497580 3881 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.107874ms I0327 23:01:51.497613 3881 namespace.go:98] [openshift-cluster-version] adding namespace I0327 23:01:51.498124 3881 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.185314ms I0327 23:01:51.498138 3881 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0327 23:01:51.498645 3881 namespace.go:102] [openshift-console-operator] adding namespace took 7.284347ms I0327 23:01:51.498665 3881 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0327 23:01:51.499165 3881 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.309523ms I0327 23:01:51.499179 3881 namespace.go:98] [kube-node-lease] adding namespace I0327 23:01:51.499651 3881 namespace.go:102] [openshift-dns-operator] adding namespace took 7.352524ms I0327 23:01:51.500179 3881 namespace.go:102] [openshift-backplane-cse] adding namespace took 7.495228ms I0327 23:01:51.500190 3881 namespace.go:98] [openshift-insights] adding namespace W0327 23:01:51.500618 3881 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-pl42s: pod openshift-multus/network-metrics-daemon-pl42s: no pod IPs found W0327 23:01:51.500628 3881 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-wq5tj: pod openshift-multus/network-metrics-daemon-wq5tj: no pod IPs found W0327 23:01:51.500634 3881 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-49qg4: pod openshift-multus/network-metrics-daemon-49qg4: no pod IPs found I0327 23:01:51.500666 3881 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.519327ms I0327 23:01:51.500683 3881 namespace.go:98] [openshift-ingress-canary] adding namespace I0327 23:01:51.501186 3881 namespace.go:102] [openshift-multus] adding namespace took 7.634085ms I0327 23:01:51.501200 3881 namespace.go:98] [openshift-backplane-srep] adding namespace I0327 23:01:51.501682 3881 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.610279ms I0327 23:01:51.501694 3881 namespace.go:98] [openshift-kube-apiserver] adding namespace I0327 23:01:51.502121 3881 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.525618ms I0327 23:01:51.502136 3881 namespace.go:98] [openshift-config-managed] adding namespace I0327 23:01:51.502715 3881 namespace.go:102] [openshift-console] adding namespace took 7.693633ms I0327 23:01:51.502726 3881 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0327 23:01:51.503149 3881 namespace.go:102] [openshift-infra] adding namespace took 7.666204ms I0327 23:01:51.503163 3881 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0327 23:01:51.503655 3881 namespace.go:102] [kube-public] adding namespace took 7.72641ms I0327 23:01:51.503666 3881 namespace.go:98] [openshift-package-operator] adding namespace I0327 23:01:51.504620 3881 namespace.go:102] [openshift-logging] adding namespace took 8.207967ms I0327 23:01:51.504630 3881 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0327 23:01:51.505050 3881 namespace.go:102] [openshift-network-operator] adding namespace took 8.073055ms I0327 23:01:51.505070 3881 namespace.go:98] [openshift-ingress] adding namespace I0327 23:01:51.505554 3881 namespace.go:102] [openshift-cluster-version] adding namespace took 7.930709ms I0327 23:01:51.505564 3881 namespace.go:98] [openshift-network-node-identity] adding namespace I0327 23:01:51.506084 3881 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 7.94037ms I0327 23:01:51.506098 3881 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0327 23:01:51.506511 3881 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.840257ms I0327 23:01:51.506523 3881 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0327 23:01:51.506928 3881 namespace.go:102] [kube-node-lease] adding namespace took 7.743788ms I0327 23:01:51.506940 3881 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0327 23:01:51.507323 3881 namespace.go:102] [openshift-insights] adding namespace took 7.127787ms I0327 23:01:51.507335 3881 namespace.go:98] [openshift-dns] adding namespace I0327 23:01:51.507703 3881 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.015192ms I0327 23:01:51.507715 3881 namespace.go:98] [openshift-service-ca-operator] adding namespace I0327 23:01:51.508181 3881 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.976571ms I0327 23:01:51.508202 3881 namespace.go:98] [openshift-marketplace] adding namespace I0327 23:01:51.508526 3881 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.828011ms I0327 23:01:51.508538 3881 namespace.go:98] [kube-system] adding namespace I0327 23:01:51.509063 3881 namespace.go:102] [openshift-config-managed] adding namespace took 6.923532ms I0327 23:01:51.509071 3881 namespace.go:98] [openshift-config-operator] adding namespace I0327 23:01:51.509485 3881 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.754328ms I0327 23:01:51.509498 3881 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0327 23:01:51.509987 3881 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.820591ms I0327 23:01:51.509996 3881 namespace.go:98] [openshift-kube-scheduler] adding namespace I0327 23:01:51.510664 3881 namespace.go:102] [openshift-package-operator] adding namespace took 6.991731ms I0327 23:01:51.510679 3881 namespace.go:98] [openshift-network-console] adding namespace I0327 23:01:51.511249 3881 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.61208ms I0327 23:01:51.511263 3881 namespace.go:98] [dedicated-admin] adding namespace I0327 23:01:51.511614 3881 namespace.go:102] [openshift-ingress] adding namespace took 6.538783ms I0327 23:01:51.511625 3881 namespace.go:98] [openshift-backplane-csm] adding namespace I0327 23:01:51.512133 3881 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.563521ms I0327 23:01:51.512145 3881 namespace.go:98] [openshift-apiserver] adding namespace I0327 23:01:51.512515 3881 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.413554ms I0327 23:01:51.512524 3881 namespace.go:98] [openshift-operators] adding namespace I0327 23:01:51.513053 3881 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.52418ms I0327 23:01:51.513065 3881 namespace.go:98] [open-cluster-management-2pabkugc0r4kj7d3r2omus9hg3vasti0] adding namespace I0327 23:01:51.513535 3881 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.590778ms I0327 23:01:51.513545 3881 namespace.go:98] [openshift-route-controller-manager] adding namespace I0327 23:01:51.513995 3881 namespace.go:102] [openshift-dns] adding namespace took 6.654807ms I0327 23:01:51.514008 3881 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0327 23:01:51.514420 3881 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.700254ms W0327 23:01:51.514857 3881 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-5mcbr: pod kube-system/global-pull-secret-syncer-5mcbr: no pod IPs found W0327 23:01:51.514868 3881 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-l9lt8: pod kube-system/global-pull-secret-syncer-l9lt8: no pod IPs found I0327 23:01:51.514900 3881 namespace.go:102] [openshift-marketplace] adding namespace took 6.693454ms I0327 23:01:51.514911 3881 namespace.go:98] [openshift] adding namespace I0327 23:01:51.515316 3881 namespace.go:102] [kube-system] adding namespace took 6.771141ms I0327 23:01:51.515783 3881 namespace.go:102] [openshift-config-operator] adding namespace took 6.707431ms I0327 23:01:51.515795 3881 namespace.go:98] [openshift-ingress-operator] adding namespace I0327 23:01:51.516260 3881 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.757007ms I0327 23:01:51.516271 3881 namespace.go:98] [openshift-monitoring] adding namespace I0327 23:01:51.516733 3881 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.731799ms I0327 23:01:51.517155 3881 namespace.go:102] [openshift-network-console] adding namespace took 6.471439ms I0327 23:01:51.517514 3881 namespace.go:102] [dedicated-admin] adding namespace took 6.246298ms I0327 23:01:51.518033 3881 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.403795ms I0327 23:01:51.518391 3881 namespace.go:102] [openshift-apiserver] adding namespace took 6.242454ms I0327 23:01:51.518891 3881 namespace.go:102] [openshift-operators] adding namespace took 6.361242ms I0327 23:01:51.519321 3881 namespace.go:102] [open-cluster-management-2pabkugc0r4kj7d3r2omus9hg3vasti0] adding namespace took 6.241893ms I0327 23:01:51.520025 3881 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.475343ms I0327 23:01:51.520449 3881 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.437328ms I0327 23:01:51.520936 3881 namespace.go:102] [openshift] adding namespace took 6.01955ms I0327 23:01:51.521313 3881 namespace.go:102] [openshift-ingress-operator] adding namespace took 5.5124ms I0327 23:01:51.521729 3881 namespace.go:102] [openshift-monitoring] adding namespace took 5.453231ms I0327 23:01:51.522106 3881 zone_ic_handler.go:186] Time taken to create transit switch: 278.959µs I0327 23:01:51.522200 3881 master.go:532] Adding or Updating local node "ip-10-0-2-103.ec2.internal" for network "default" I0327 23:01:51.522214 3881 obj_retry.go:544] Failed to create *v1.Node ip-10-0-1-182.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-182.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-182.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-182.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-182.ec2.internal I0327 23:01:51.522242 3881 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-182.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-182.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-182.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-182.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-182.ec2.internal W0327 23:01:51.529315 3881 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-5mcbr: pod kube-system/global-pull-secret-syncer-5mcbr: no pod IPs found W0327 23:01:51.529339 3881 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-wq5tj: pod openshift-multus/network-metrics-daemon-wq5tj: no pod IPs found W0327 23:01:51.529353 3881 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-67rsf: pod openshift-network-diagnostics/network-check-target-67rsf: no pod IPs found I0327 23:01:51.530226 3881 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-2-103.ec2.internal for the network default I0327 23:01:51.530988 3881 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0327 23:01:51.531016 3881 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0327 23:01:51.531029 3881 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0327 23:01:51.531056 3881 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-103.ec2.internal in network "default" I0327 23:01:51.531098 3881 node_tracker.go:169] Node ip-10-0-2-103.ec2.internal switch + router changed, syncing services in network "default" I0327 23:01:51.531111 3881 services_controller.go:574] Full service sync requested for network=default I0327 23:01:51.531121 3881 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-182.ec2.internal in network "default" I0327 23:01:51.531132 3881 node_tracker.go:235] Node ip-10-0-0-182.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-182.ec2.internal" I0327 23:01:51.531138 3881 node_tracker.go:169] Node ip-10-0-0-182.ec2.internal switch + router changed, syncing services in network "default" I0327 23:01:51.531142 3881 services_controller.go:574] Full service sync requested for network=default I0327 23:01:51.531148 3881 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-182.ec2.internal in network "default" I0327 23:01:51.531154 3881 node_tracker.go:235] Node ip-10-0-1-182.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-182.ec2.internal" I0327 23:01:51.531160 3881 node_tracker.go:169] Node ip-10-0-1-182.ec2.internal switch + router changed, syncing services in network "default" I0327 23:01:51.531163 3881 services_controller.go:574] Full service sync requested for network=default I0327 23:01:51.631507 3881 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0327 23:01:51.631522 3881 services_controller.go:208] Setting up event handlers for services for network=default I0327 23:01:51.631567 3881 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0327 23:01:51.631616 3881 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0327 23:01:51.631631 3881 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0327 23:01:51.732044 3881 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0327 23:01:51.732058 3881 repair.go:60] Starting repairing loop for services I0327 23:01:51.732150 3881 repair.go:131] Deleted 0 stale service LBs I0327 23:01:51.732167 3881 repair.go:137] Deleted 0 stale Chassis Template Vars I0327 23:01:51.732181 3881 repair.go:62] Finished repairing loop for services: 125.498µs I0327 23:01:51.732206 3881 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0327 23:01:51.732215 3881 services_controller.go:254] Starting workers for network=default I0327 23:01:51.732564 3881 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-gnmgz, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-gnmgz: suppressed error logged: pod openshift-network-diagnostics/network-check-target-gnmgz: no pod IPs found I0327 23:01:51.732593 3881 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-pl42s, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-pl42s: suppressed error logged: pod openshift-multus/network-metrics-daemon-pl42s: no pod IPs found I0327 23:01:51.732597 3881 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-49qg4, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-49qg4: suppressed error logged: pod openshift-multus/network-metrics-daemon-49qg4: no pod IPs found I0327 23:01:51.732597 3881 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-9ptpd, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-9ptpd: suppressed error logged: pod openshift-network-diagnostics/network-check-target-9ptpd: no pod IPs found I0327 23:01:51.732617 3881 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-5mcbr] creating logical port kube-system_global-pull-secret-syncer-5mcbr for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:51.732706 3881 obj_retry.go:544] Failed to create *v1.Pod kube-system/global-pull-secret-syncer-l9lt8, error: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-l9lt8: suppressed error logged: pod kube-system/global-pull-secret-syncer-l9lt8: no pod IPs found I0327 23:01:51.732710 3881 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-wq5tj] creating logical port openshift-multus_network-metrics-daemon-wq5tj for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:51.732617 3881 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-67rsf] creating logical port openshift-network-diagnostics_network-check-target-67rsf for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:51.732792 3881 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-5mcbr I0327 23:01:51.732872 3881 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-wq5tj I0327 23:01:51.732880 3881 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-67rsf I0327 23:01:51.749725 3881 pod.go:62] [openshift-multus/network-metrics-daemon-wq5tj] pod update took 16.927987ms I0327 23:01:51.749742 3881 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-wq5tj] addLogicalPort annotation time took 16.947376ms I0327 23:01:51.750581 3881 pods.go:271] [openshift-multus/network-metrics-daemon-wq5tj] addLogicalPort took 17.878226ms, libovsdb time 594.261µs I0327 23:01:51.752337 3881 pod.go:62] [kube-system/global-pull-secret-syncer-5mcbr] pod update took 19.644195ms I0327 23:01:51.752352 3881 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-5mcbr] addLogicalPort annotation time took 19.661138ms I0327 23:01:51.753079 3881 pods.go:271] [kube-system/global-pull-secret-syncer-5mcbr] addLogicalPort took 20.448306ms, libovsdb time 547.781µs I0327 23:01:51.753295 3881 pod.go:62] [openshift-network-diagnostics/network-check-target-67rsf] pod update took 20.450627ms I0327 23:01:51.753315 3881 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-67rsf] addLogicalPort annotation time took 20.488398ms I0327 23:01:51.754036 3881 pods.go:271] [openshift-network-diagnostics/network-check-target-67rsf] addLogicalPort took 21.429127ms, libovsdb time 562.235µs I0327 23:01:51.754430 3881 admin_network_policy_controller.go:227] Starting controller default-network-controller I0327 23:01:51.754448 3881 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0327 23:01:51.754461 3881 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0327 23:01:51.754465 3881 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0327 23:01:51.754646 3881 repair.go:29] Repairing admin network policies took 176.087µs I0327 23:01:51.754749 3881 repair.go:92] Repairing baseline admin network policies took 92.001µs I0327 23:01:51.755204 3881 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0327 23:01:51.755220 3881 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0327 23:01:51.756556 3881 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0327 23:01:51.756575 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0327 23:01:51.756590 3881 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0327 23:01:51.758019 3881 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0327 23:01:51.758036 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0327 23:01:51.758043 3881 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0327 23:01:51.758316 3881 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0327 23:01:51.758336 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0327 23:01:51.758701 3881 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0327 23:01:51.758716 3881 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 23:01:51.760344 3881 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0327 23:01:51.760365 3881 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0327 23:01:51.760376 3881 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0327 23:01:51.761615 3881 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0327 23:01:51.761631 3881 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0327 23:01:51.761639 3881 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0327 23:01:51.763282 3881 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0327 23:01:51.763300 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0327 23:01:51.763307 3881 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0327 23:01:51.764645 3881 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0327 23:01:51.764661 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0327 23:01:51.764669 3881 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0327 23:01:51.765144 3881 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0327 23:01:51.765163 3881 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0327 23:01:51.765175 3881 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0327 23:01:51.766246 3881 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0327 23:01:51.766263 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0327 23:01:51.766270 3881 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0327 23:01:51.766688 3881 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0327 23:01:51.766705 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0327 23:01:51.767133 3881 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0327 23:01:51.767600 3881 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0327 23:01:51.767616 3881 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0327 23:01:51.767623 3881 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0327 23:01:51.768988 3881 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0327 23:01:51.769005 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0327 23:01:51.769012 3881 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0327 23:01:51.769459 3881 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0327 23:01:51.769477 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0327 23:01:51.769483 3881 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0327 23:01:51.769749 3881 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0327 23:01:51.769785 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0327 23:01:51.769796 3881 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0327 23:01:51.771296 3881 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0327 23:01:51.771318 3881 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0327 23:01:51.771328 3881 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0327 23:01:51.772601 3881 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0327 23:01:51.772625 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0327 23:01:51.772635 3881 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0327 23:01:51.773254 3881 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0327 23:01:51.773270 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0327 23:01:51.773290 3881 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 23:01:51.774044 3881 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0327 23:01:51.774069 3881 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0327 23:01:51.774080 3881 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0327 23:01:51.775316 3881 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0327 23:01:51.775340 3881 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0327 23:01:51.775355 3881 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0327 23:01:51.778153 3881 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0327 23:01:51.778188 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0327 23:01:51.778201 3881 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0327 23:01:51.779072 3881 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0327 23:01:51.779097 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0327 23:01:51.779109 3881 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0327 23:01:51.779723 3881 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0327 23:01:51.779742 3881 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0327 23:01:51.779750 3881 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0327 23:01:51.780438 3881 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0327 23:01:51.780454 3881 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0327 23:01:51.780464 3881 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0327 23:01:51.781549 3881 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0327 23:01:51.781570 3881 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0327 23:01:51.781580 3881 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0327 23:01:51.781943 3881 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0327 23:01:51.781964 3881 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0327 23:01:51.781977 3881 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0327 23:01:51.783200 3881 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0327 23:01:51.783224 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0327 23:01:51.783235 3881 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0327 23:01:51.783918 3881 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0327 23:01:51.783943 3881 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0327 23:01:51.783953 3881 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0327 23:01:51.785209 3881 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0327 23:01:51.785229 3881 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0327 23:01:51.785239 3881 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0327 23:01:51.785874 3881 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0327 23:01:51.791069 3881 egressfirewall.go:319] Starting EgressFirewall controller I0327 23:01:51.791094 3881 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0327 23:01:51.791117 3881 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0327 23:01:51.791126 3881 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0327 23:01:51.791157 3881 controller.go:133] Adding controller egress-firewall-controller event handlers I0327 23:01:51.791186 3881 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0327 23:01:51.791197 3881 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0327 23:01:51.791439 3881 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0327 23:01:51.791452 3881 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0327 23:01:51.791461 3881 egressqos.go:193] Setting up event handlers for EgressQoS I0327 23:01:51.791551 3881 egressfirewall.go:628] Syncing node "ip-10-0-0-182.ec2.internal" for egress firewall I0327 23:01:51.791566 3881 egressfirewall.go:628] Syncing node "ip-10-0-1-182.ec2.internal" for egress firewall I0327 23:01:51.791573 3881 egressfirewall.go:628] Syncing node "ip-10-0-2-103.ec2.internal" for egress firewall I0327 23:01:51.791601 3881 egressqos.go:245] Starting EgressQoS Controller I0327 23:01:51.791621 3881 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0327 23:01:51.791637 3881 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0327 23:01:51.791648 3881 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0327 23:01:51.791656 3881 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0327 23:01:51.791703 3881 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0327 23:01:51.791711 3881 shared_informer.go:356] "Caches are synced" controller="egressqos" I0327 23:01:51.791715 3881 egressqos.go:259] Repairing EgressQoSes I0327 23:01:51.791720 3881 egressqos.go:399] Starting repairing loop for egressqos I0327 23:01:51.791853 3881 egressqos.go:401] Finished repairing loop for egressqos: 133.204µs I0327 23:01:51.791873 3881 egressservice_zone.go:132] Setting up event handlers for Egress Services I0327 23:01:51.791918 3881 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-103.ec2.internal I0327 23:01:51.791978 3881 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-103.ec2.internal : 60.985µs I0327 23:01:51.792161 3881 egressservice_zone.go:210] Starting Egress Services Controller I0327 23:01:51.792178 3881 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 23:01:51.792211 3881 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 23:01:51.792247 3881 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 23:01:51.792264 3881 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 23:01:51.792282 3881 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 23:01:51.792289 3881 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 23:01:51.792298 3881 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0327 23:01:51.792305 3881 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0327 23:01:51.792310 3881 egressservice_zone.go:228] Repairing Egress Services I0327 23:01:51.792678 3881 master_controller.go:88] Starting Admin Policy Based Route Controller I0327 23:01:51.792687 3881 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 23:01:51.792715 3881 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-182.ec2.internal I0327 23:01:51.792728 3881 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-182.ec2.internal: 15.748µs I0327 23:01:51.792738 3881 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-103.ec2.internal I0327 23:01:51.792743 3881 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-103.ec2.internal: 6.464µs I0327 23:01:51.792750 3881 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-182.ec2.internal I0327 23:01:51.792755 3881 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-182.ec2.internal: 5.415µs I0327 23:01:51.792829 3881 default_network_controller.go:568] Completing all the Watchers took 309.400584ms I0327 23:01:51.792849 3881 default_network_controller.go:572] Starting unidling controllers I0327 23:01:51.792866 3881 unidle.go:46] Registering OVN SB ControllerEvent handler I0327 23:01:51.792876 3881 unidle.go:63] Populating Initial ContollerEvent events I0327 23:01:51.792888 3881 unidle.go:79] Setting up event handlers for services I0327 23:01:51.792957 3881 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0327 23:01:51.998319 3881 default_node_network_controller.go:518] Node connection status = connected I0327 23:01:52.497961 3881 default_node_network_controller.go:518] Node connection status = connected I0327 23:01:52.699086 3881 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0327 23:01:52.997294 3881 default_node_network_controller.go:518] Node connection status = connected I0327 23:01:53.012421 3881 bridgeconfig.go:537] Gateway is ready I0327 23:01:53.012442 3881 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0327 23:01:53.022076 3881 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.103. Updating OVN encap IP. I0327 23:01:53.026830 3881 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.103 - it is already configured I0327 23:01:53.026843 3881 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.103/24:{} 172.20.0.1/32:{}]. Updating annotations. I0327 23:01:53.027144 3881 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.103/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-103.ec2.internal","mac-address":"0e:17:2b:83:01:3f","ip-addresses":["10.0.2.103/24"],"ip-address":"10.0.2.103/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:06deb6dd-411c-4894-93ce-06db4d6915c3 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.103/24"}] on node ip-10-0-2-103.ec2.internal I0327 23:01:53.044290 3881 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-103.ec2.internal I0327 23:01:53.044308 3881 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-103.ec2.internal: 26.257µs I0327 23:01:53.044327 3881 master.go:532] Adding or Updating local node "ip-10-0-2-103.ec2.internal" for network "default" I0327 23:01:53.044370 3881 egressfirewall.go:628] Syncing node "ip-10-0-2-103.ec2.internal" for egress firewall I0327 23:01:53.044381 3881 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-103.ec2.internal in network "default" I0327 23:01:53.044424 3881 node_tracker.go:169] Node ip-10-0-2-103.ec2.internal switch + router changed, syncing services in network "default" I0327 23:01:53.044437 3881 services_controller.go:574] Full service sync requested for network=default I0327 23:01:53.045144 3881 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.2.103 I0327 23:01:53.045263 3881 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-103.ec2.internal I0327 23:01:53.049176 3881 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-103.ec2.internal. Updating no re-route policies I0327 23:01:53.049256 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-103.ec2.internal I0327 23:01:53.049266 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-v9r4s I0327 23:01:53.049272 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-103.ec2.internal I0327 23:01:53.049278 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-tvkp8 I0327 23:01:53.049291 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-tvkp8 I0327 23:01:53.049296 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-67rsf I0327 23:01:53.049305 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-9ptpd I0327 23:01:53.049309 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-67rsf I0327 23:01:53.049312 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-9ptpd I0327 23:01:53.049319 3881 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-103.ec2.internal I0327 23:01:53.049319 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-gnmgz I0327 23:01:53.049324 3881 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-l9lt8 I0327 23:01:53.049330 3881 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-103.ec2.internal I0327 23:01:53.049280 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-v9r4s I0327 23:01:53.049339 3881 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-l9lt8 I0327 23:01:53.049353 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-wq5tj I0327 23:01:53.049291 3881 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-103.ec2.internal after 0 failed attempt(s) I0327 23:01:53.049358 3881 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-9ptpd, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-9ptpd: suppressed error logged: pod openshift-network-diagnostics/network-check-target-9ptpd: no pod IPs found I0327 23:01:53.049265 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-7cgz8 I0327 23:01:53.049365 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-scfc4 I0327 23:01:53.049364 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-gtj76 I0327 23:01:53.049373 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-7cgz8 I0327 23:01:53.049371 3881 obj_retry.go:375] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-l9lt8, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-l9lt8: suppressed error logged: pod kube-system/global-pull-secret-syncer-l9lt8: no pod IPs found I0327 23:01:53.049377 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-2r7h5 I0327 23:01:53.049322 3881 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-tvkp8 after 0 failed attempt(s) I0327 23:01:53.049380 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-scfc4 I0327 23:01:53.049384 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-2r7h5 I0327 23:01:53.049401 3881 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-scfc4 after 0 failed attempt(s) I0327 23:01:53.049307 3881 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-7gw9m I0327 23:01:53.049433 3881 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-67rsf] creating logical port openshift-network-diagnostics_network-check-target-67rsf for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:53.049354 3881 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-b9fwq I0327 23:01:53.049463 3881 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-b9fwq I0327 23:01:53.049508 3881 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-b9fwq after 0 failed attempt(s) I0327 23:01:53.049336 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-gnmgz I0327 23:01:53.049560 3881 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-gnmgz, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-gnmgz: suppressed error logged: pod openshift-network-diagnostics/network-check-target-gnmgz: no pod IPs found I0327 23:01:53.049355 3881 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-103.ec2.internal after 0 failed attempt(s) I0327 23:01:53.049365 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-wq5tj I0327 23:01:53.049618 3881 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-wq5tj] creating logical port openshift-multus_network-metrics-daemon-wq5tj for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:53.049292 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-w62pw I0327 23:01:53.049724 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-w62pw I0327 23:01:53.049278 3881 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-5mcbr I0327 23:01:53.049744 3881 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-5mcbr I0327 23:01:53.049753 3881 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-w62pw after 0 failed attempt(s) I0327 23:01:53.049797 3881 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-5mcbr] creating logical port kube-system_global-pull-secret-syncer-5mcbr for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:53.049362 3881 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-v9r4s after 0 failed attempt(s) I0327 23:01:53.049386 3881 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-7cgz8 after 0 failed attempt(s) I0327 23:01:53.049385 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-gtj76 I0327 23:01:53.049886 3881 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-2r7h5 after 0 failed attempt(s) I0327 23:01:53.049887 3881 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-gtj76 after 0 failed attempt(s) I0327 23:01:53.049307 3881 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-nssjq I0327 23:01:53.049915 3881 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-nssjq I0327 23:01:53.049934 3881 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-nssjq after 0 failed attempt(s) I0327 23:01:53.049441 3881 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-7gw9m I0327 23:01:53.049949 3881 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-7gw9m after 0 failed attempt(s) I0327 23:01:53.050505 3881 pods.go:271] [openshift-network-diagnostics/network-check-target-67rsf] addLogicalPort took 1.141324ms, libovsdb time 513.541µs I0327 23:01:53.050518 3881 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-67rsf after 0 failed attempt(s) I0327 23:01:53.050647 3881 pods.go:271] [kube-system/global-pull-secret-syncer-5mcbr] addLogicalPort took 859.102µs, libovsdb time 406.026µs I0327 23:01:53.050659 3881 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-5mcbr after 0 failed attempt(s) I0327 23:01:53.050794 3881 pods.go:271] [openshift-multus/network-metrics-daemon-wq5tj] addLogicalPort took 1.186597ms, libovsdb time 490.257µs I0327 23:01:53.050807 3881 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-wq5tj after 0 failed attempt(s) I0327 23:01:53.060269 3881 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0327 23:01:53.070735 3881 iptables.go:108] Creating table: mangle chain: OUTPUT I0327 23:01:53.073971 3881 iptables.go:108] Creating table: nat chain: OUTPUT I0327 23:01:53.098935 3881 iptables.go:108] Creating table: nat chain: PREROUTING I0327 23:01:53.143738 3881 iptables.go:108] Creating table: filter chain: FORWARD I0327 23:01:53.160009 3881 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0327 23:01:53.160125 3881 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0327 23:01:53.160167 3881 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0327 23:01:53.160241 3881 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0327 23:01:53.160287 3881 gateway_init.go:325] Gateway and management port readiness took 1.666418617s I0327 23:01:53.160311 3881 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0327 23:01:53.160342 3881 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0327 23:01:53.160354 3881 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0327 23:01:53.160362 3881 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 23:01:53.160414 3881 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0327 23:01:53.160495 3881 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 49.916µs I0327 23:01:53.160683 3881 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0327 23:01:53.160706 3881 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 14.075µs I0327 23:01:53.160713 3881 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 30.91µs I0327 23:01:53.160718 3881 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0327 23:01:53.160726 3881 nad_controller.go:173] [node-nad-controller NAD controller]: started I0327 23:01:53.160730 3881 default_node_network_controller.go:898] Starting the default node network controller I0327 23:01:53.160735 3881 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0327 23:01:53.160777 3881 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 32.463µs I0327 23:01:53.160931 3881 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0327 23:01:53.160941 3881 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0327 23:01:53.160967 3881 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a8b000 0xc0002afa40 0xc0002afab0 0xc0002afb20 0xc0002afb90 0xc0002afc00 0xc0002afc70 0xc000aa7360 0xc0002afce0 0xc0002afd50 0xc00084c320 0xc00647f180 0xc0002afdc0 map[0x2f67040:0xc002114300 0x2f67580:0xc002114480 0x2f68000:0xc002114240 0x2f682a0:0xc0021142a0 0x2f687e0:0xc002114360 0x2f68a80:0xc0021143c0 0x2f68d20:0xc002114420 0x2f69500:0xc0021144e0 0x2f697a0:0xc0021146c0 0x2f93d40:0xc000a28720 0x2f940e0:0xc000a28780 0x2f94bc0:0xc000ae9680 0x2f96520:0xc001af8000 0x2f97000:0xc003a080c0 0x2f97e80:0xc000a286c0] 0xc000aa82a0 0} I0327 23:01:53.161027 3881 gateway.go:151] Starting gateway service sync I0327 23:01:53.161592 3881 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0327 23:01:53.163724 3881 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0327 23:01:53.174439 3881 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0327 23:01:53.175553 3881 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0327 23:01:53.176756 3881 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0327 23:01:53.211455 3881 gateway.go:168] Gateway service sync done. Time taken: 50.415064ms I0327 23:01:53.211941 3881 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31317 I0327 23:01:53.212252 3881 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a8b000 0xc0002afa40 0xc0002afab0 0xc0002afb20 0xc0002afb90 0xc0002afc00 0xc0002afc70 0xc000aa7360 0xc0002afce0 0xc0002afd50 0xc00084c320 0xc00647f180 0xc0002afdc0 map[0x2f67040:0xc002114300 0x2f67580:0xc002114480 0x2f68000:0xc002114240 0x2f682a0:0xc0021142a0 0x2f687e0:0xc002114360 0x2f68a80:0xc0021143c0 0x2f68d20:0xc002114420 0x2f69500:0xc0021144e0 0x2f697a0:0xc0021146c0 0x2f93d40:0xc000a28720 0x2f940e0:0xc000a28780 0x2f94bc0:0xc000ae9680 0x2f96520:0xc001af8000 0x2f97000:0xc003a080c0 0x2f97e80:0xc000a286c0] 0xc000aa82a0 0} I0327 23:01:53.212684 3881 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a8b000 0xc0002afa40 0xc0002afab0 0xc0002afb20 0xc0002afb90 0xc0002afc00 0xc0002afc70 0xc000aa7360 0xc0002afce0 0xc0002afd50 0xc00084c320 0xc00647f180 0xc0002afdc0 map[0x2f67040:0xc002114300 0x2f67580:0xc002114480 0x2f68000:0xc002114240 0x2f682a0:0xc0021142a0 0x2f687e0:0xc002114360 0x2f68a80:0xc0021143c0 0x2f68d20:0xc002114420 0x2f69500:0xc0021144e0 0x2f697a0:0xc0021146c0 0x2f93d40:0xc000a28720 0x2f940e0:0xc000a28780 0x2f94bc0:0xc000ae9680 0x2f96520:0xc001af8000 0x2f97000:0xc003a080c0 0x2f97e80:0xc000a286c0] 0xc000aa82a0 0} I0327 23:01:53.212723 3881 gateway.go:519] Reconciling gateway with updates I0327 23:01:53.218517 3881 gateway.go:326] Spawning Conntrack Rule Check Thread I0327 23:01:53.218615 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 23:01:53.218635 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 23:01:53.218618 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 23:01:53.218646 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 23:01:53.218648 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 23:01:53.218652 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 23:01:53.218653 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 23:01:53.218666 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 23:01:53.218680 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 23:01:53.218685 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 23:01:53.218698 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 23:01:53.218704 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 23:01:53.218711 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 23:01:53.218638 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 23:01:53.218717 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 23:01:53.218721 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 23:01:53.218714 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 23:01:53.218729 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 23:01:53.218739 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 23:01:53.218743 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 23:01:53.218636 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 23:01:53.218751 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 23:01:53.218670 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 23:01:53.218747 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 23:01:53.218777 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 23:01:53.218783 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 23:01:53.218789 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 23:01:53.218787 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 23:01:53.218796 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 23:01:53.218796 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 23:01:53.218802 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 23:01:53.218802 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 23:01:53.218809 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 23:01:53.218811 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 23:01:53.218813 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 23:01:53.218755 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 23:01:53.218816 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 23:01:53.218721 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 23:01:53.218821 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 23:01:53.218745 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 23:01:53.218829 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 23:01:53.218831 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 23:01:53.218836 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 23:01:53.218840 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 23:01:53.218842 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 23:01:53.218850 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 23:01:53.218853 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 23:01:53.218857 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 23:01:53.218858 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 23:01:53.218862 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 23:01:53.218864 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 23:01:53.218866 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 23:01:53.218872 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 23:01:53.218872 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 23:01:53.218864 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 23:01:53.218882 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 23:01:53.218886 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 23:01:53.218702 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 23:01:53.218885 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 23:01:53.218899 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 23:01:53.218904 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 23:01:53.218905 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 23:01:53.218913 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 23:01:53.218914 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 23:01:53.218853 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 23:01:53.218923 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 23:01:53.218924 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 23:01:53.218930 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 23:01:53.218871 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 23:01:53.218938 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 23:01:53.218940 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 23:01:53.218945 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 23:01:53.218817 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 23:01:53.218952 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 23:01:53.218957 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 23:01:53.218957 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 23:01:53.218964 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 23:01:53.218973 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 23:01:53.218967 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 23:01:53.218841 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 23:01:53.218940 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 23:01:53.218979 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 23:01:53.218757 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 23:01:53.218625 3881 default_node_network_controller.go:943] Gateway and management port readiness took 57.674642ms I0327 23:01:53.218960 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 23:01:53.218906 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 23:01:53.219007 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 23:01:53.219014 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 23:01:53.218975 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 23:01:53.218708 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 23:01:53.219035 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 23:01:53.218843 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 23:01:53.218792 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 23:01:53.218798 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 23:01:53.218855 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 23:01:53.218817 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 23:01:53.219066 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 23:01:53.218916 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 23:01:53.218896 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 23:01:53.218894 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 23:01:53.219093 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 23:01:53.219096 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 23:01:53.218917 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 23:01:53.218933 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 23:01:53.218943 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 23:01:53.219128 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 23:01:53.218944 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 23:01:53.218989 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 23:01:53.218994 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 23:01:53.218995 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 23:01:53.219155 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 23:01:53.218959 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 23:01:53.218947 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 23:01:53.219205 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 23:01:53.219172 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 23:01:53.218827 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 23:01:53.219303 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 23:01:53.218823 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 23:01:53.219402 3881 port_claim.go:71] Svc openshift-ingress/router-default: port 32599 is already open, no action needed I0327 23:01:53.219418 3881 port_claim.go:71] Svc openshift-ingress/router-default: port 30987 is already open, no action needed I0327 23:01:53.219870 3881 node_ip_handler_linux.go:160] Node IP manager is running W0327 23:01:53.257368 3881 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 23:01:53.257620 3881 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 23:01:53.257793 3881 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0327 23:01:53.268369 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 23:01:53.290122 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 23:01:53.290135 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 23:01:53.290152 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 23:01:53.290160 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 23:01:53.290166 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 23:01:53.290172 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 23:01:53.290187 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 23:01:53.290201 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 23:01:53.290214 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 23:01:53.290221 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 23:01:53.290226 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 23:01:53.290234 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 23:01:53.290239 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 23:01:53.329797 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 23:01:53.583350 3881 default_network_controller.go:1008] Node "ip-10-0-1-182.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 23:01:55.334244 3881 default_node_network_controller.go:1364] Starting node controller node sync W0327 23:01:55.334329 3881 egressip_healthcheck.go:76] Health checking using insecure connection I0327 23:01:55.334600 3881 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0327 23:01:55.346821 3881 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 12.329301ms I0327 23:01:55.346991 3881 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0327 23:01:55.347310 3881 egressservice_node.go:93] Setting up event handlers for Egress Services I0327 23:01:55.347441 3881 egressservice_node.go:183] Starting Egress Services Controller I0327 23:01:55.347456 3881 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 23:01:55.347465 3881 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 23:01:55.347472 3881 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 23:01:55.347478 3881 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 23:01:55.347483 3881 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 23:01:55.347496 3881 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 23:01:55.347503 3881 egressservice_node.go:197] Repairing Egress Services I0327 23:01:55.358001 3881 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0327 23:01:55.358015 3881 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 23:01:55.358222 3881 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0327 23:01:55.358237 3881 link_network_manager.go:117] Link manager is running I0327 23:01:55.358246 3881 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0327 23:01:55.358304 3881 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0327 23:01:55.358659 3881 vrf_manager.go:128] VRF manager is running I0327 23:01:55.358966 3881 node_controller_manager.go:437] Removing flows to drop GARP I0327 23:01:55.358976 3881 gateway.go:519] Reconciling gateway with updates I0327 23:01:55.381748 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 23:01:55.381781 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 23:01:55.381787 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 23:01:55.381798 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 23:01:55.381800 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 23:01:55.381785 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 23:01:55.381810 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 23:01:55.381815 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 23:01:55.381818 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 23:01:55.381830 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 23:01:55.381830 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 23:01:55.381845 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 23:01:55.381847 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 23:01:55.381857 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 23:01:55.381845 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 23:01:55.381871 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 23:01:55.381875 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 23:01:55.381808 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 23:01:55.381881 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 23:01:55.381887 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 23:01:55.381855 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 23:01:55.381894 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 23:01:55.381899 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 23:01:55.381900 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 23:01:55.381815 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 23:01:55.381829 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 23:01:55.381911 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 23:01:55.381918 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 23:01:55.381924 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 23:01:55.381924 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 23:01:55.381905 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 23:01:55.381931 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 23:01:55.381936 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 23:01:55.381945 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 23:01:55.381945 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 23:01:55.381878 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 23:01:55.381963 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 23:01:55.381961 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 23:01:55.381976 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 23:01:55.381987 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 23:01:55.381996 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 23:01:55.381999 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 23:01:55.382000 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 23:01:55.382007 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 23:01:55.382012 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 23:01:55.382015 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 23:01:55.381802 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 23:01:55.382007 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 23:01:55.382042 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 23:01:55.382049 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 23:01:55.381943 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 23:01:55.381979 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 23:01:55.381907 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 23:01:55.381895 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 23:01:55.382070 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 23:01:55.381914 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 23:01:55.382083 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 23:01:55.382089 3881 port_claim.go:71] Svc openshift-ingress/router-default: port 32599 is already open, no action needed I0327 23:01:55.382094 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 23:01:55.382095 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 23:01:55.382104 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 23:01:55.382108 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 23:01:55.382116 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 23:01:55.382118 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 23:01:55.382123 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 23:01:55.382124 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 23:01:55.382128 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 23:01:55.382131 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 23:01:55.382135 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 23:01:55.382131 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 23:01:55.382140 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 23:01:55.382139 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 23:01:55.382144 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 23:01:55.382146 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 23:01:55.382150 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 23:01:55.382152 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 23:01:55.382152 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 23:01:55.382156 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 23:01:55.382158 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 23:01:55.382159 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 23:01:55.382164 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 23:01:55.382165 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 23:01:55.382180 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 23:01:55.382192 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 23:01:55.382110 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 23:01:55.382148 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 23:01:55.381911 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 23:01:55.382103 3881 port_claim.go:71] Svc openshift-ingress/router-default: port 30987 is already open, no action needed I0327 23:01:55.382194 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 23:01:55.382125 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 23:01:55.382140 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 23:01:55.382119 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 23:01:55.381982 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 23:01:55.382315 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 23:01:55.382323 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 23:01:55.381992 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 23:01:55.382066 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 23:01:55.381907 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 23:01:55.382142 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 23:01:55.382369 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 23:01:55.382146 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 23:01:55.382152 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 23:01:55.382157 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 23:01:55.381893 3881 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 23:01:55.382444 3881 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 23:01:55.381987 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 23:01:55.422535 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 23:01:55.422556 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 23:01:55.422562 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 23:01:55.422568 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 23:01:55.459369 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 23:01:55.459401 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 23:01:55.459411 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 23:01:55.459430 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 23:01:55.459445 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 23:01:55.459453 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 23:01:55.459461 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 23:01:55.459467 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 23:01:55.459472 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 23:01:55.459480 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 23:01:55.459497 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 23:01:55.459517 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 23:01:55.459542 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 23:01:55.459551 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 23:01:55.502188 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 23:01:55.502207 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 23:01:55.502216 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 23:01:55.502225 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 23:01:55.502233 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 23:01:55.502247 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 23:01:55.502256 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 23:01:55.502273 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 23:01:55.502282 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 23:01:55.502293 3881 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 23:01:56.404877 3881 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-b9fwq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:56.404911 3881 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-b9fwq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:56.898334 3881 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-103.ec2.internal I0327 23:01:56.898368 3881 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-103.ec2.internal: 40.942µs I0327 23:01:56.928782 3881 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-5b6869f466-hd4bm] creating logical port openshift-image-registry_image-registry-5b6869f466-hd4bm for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:56.928899 3881 kube.go:256] Updating pod openshift-image-registry/image-registry-5b6869f466-hd4bm I0327 23:01:56.954919 3881 pod.go:62] [openshift-image-registry/image-registry-5b6869f466-hd4bm] pod update took 26.049789ms I0327 23:01:56.954942 3881 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-5b6869f466-hd4bm] addLogicalPort annotation time took 26.075036ms I0327 23:01:56.956346 3881 pods.go:271] [openshift-image-registry/image-registry-5b6869f466-hd4bm] addLogicalPort took 27.59447ms, libovsdb time 1.025229ms I0327 23:01:56.985519 3881 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-6jw22] creating logical port openshift-service-ca-operator_service-ca-operator-7b89bc695f-6jw22 for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:56.985649 3881 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-7b89bc695f-6jw22 I0327 23:01:56.985688 3881 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-4x8xk] creating logical port openshift-dns_dns-default-4x8xk for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:56.985817 3881 kube.go:256] Updating pod openshift-dns/dns-default-4x8xk I0327 23:01:56.993257 3881 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-kdwkt] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-555cc59f77-kdwkt for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:56.993347 3881 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-kdwkt I0327 23:01:56.999036 3881 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-945sj] creating logical port openshift-monitoring_cluster-monitoring-operator-6dd897695f-945sj for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:56.999221 3881 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-6dd897695f-945sj I0327 23:01:56.999239 3881 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-mcwgp] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-6ff64bb8b6-mcwgp for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:56.999479 3881 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-mcwgp I0327 23:01:57.004829 3881 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-mpp7k] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-mpp7k for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:57.004998 3881 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-mpp7k I0327 23:01:57.020088 3881 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-kdwkt] pod update took 26.753816ms I0327 23:01:57.020109 3881 base_network_controller_pods.go:937] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-kdwkt] addLogicalPort annotation time took 26.776873ms I0327 23:01:57.020180 3881 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-wqpfj] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-wqpfj for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:57.020216 3881 pod.go:62] [openshift-dns/dns-default-4x8xk] pod update took 34.422927ms I0327 23:01:57.020276 3881 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-4x8xk] addLogicalPort annotation time took 34.483497ms I0327 23:01:57.020361 3881 pod.go:62] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-6jw22] pod update took 34.747501ms I0327 23:01:57.020413 3881 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-wqpfj I0327 23:01:57.020426 3881 base_network_controller_pods.go:937] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-6jw22] addLogicalPort annotation time took 34.815613ms I0327 23:01:57.021394 3881 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-kdwkt] addLogicalPort took 28.143876ms, libovsdb time 929.053µs I0327 23:01:57.021684 3881 pods.go:271] [openshift-dns/dns-default-4x8xk] addLogicalPort took 36.008883ms, libovsdb time 835.295µs I0327 23:01:57.021961 3881 pods.go:271] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-6jw22] addLogicalPort took 36.454907ms, libovsdb time 1.056732ms I0327 23:01:57.022102 3881 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-8777bd9bb-qzpgd] creating logical port openshift-console-operator_console-operator-8777bd9bb-qzpgd for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:57.022164 3881 kube.go:256] Updating pod openshift-console-operator/console-operator-8777bd9bb-qzpgd I0327 23:01:57.022697 3881 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-75d6fd4fb5-7jfmc] creating logical port openshift-ingress_router-default-75d6fd4fb5-7jfmc for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:57.022746 3881 kube.go:256] Updating pod openshift-ingress/router-default-75d6fd4fb5-7jfmc I0327 23:01:57.026654 3881 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-945sj] pod update took 27.492379ms I0327 23:01:57.026674 3881 base_network_controller_pods.go:937] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-945sj] addLogicalPort annotation time took 27.512897ms I0327 23:01:57.027557 3881 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-945sj] addLogicalPort took 28.6252ms, libovsdb time 628.429µs I0327 23:01:57.027597 3881 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-rrjv8] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67b99c68bb-rrjv8 for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:57.027638 3881 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-rrjv8 I0327 23:01:57.053593 3881 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-mpp7k] pod update took 48.631852ms I0327 23:01:57.053618 3881 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-mpp7k] addLogicalPort annotation time took 48.65618ms I0327 23:01:57.053644 3881 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-wqpfj] pod update took 33.288573ms I0327 23:01:57.053655 3881 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-wqpfj] addLogicalPort annotation time took 33.302393ms I0327 23:01:57.053721 3881 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-mcwgp] pod update took 54.278666ms I0327 23:01:57.053754 3881 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-mcwgp] addLogicalPort annotation time took 54.323616ms I0327 23:01:57.054007 3881 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-5577457d66-4gkzq] creating logical port openshift-insights_insights-operator-5577457d66-4gkzq for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:57.054142 3881 kube.go:256] Updating pod openshift-insights/insights-operator-5577457d66-4gkzq I0327 23:01:57.054627 3881 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-tl9fl] creating logical port openshift-ingress-canary_ingress-canary-tl9fl for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:57.054721 3881 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-tl9fl I0327 23:01:57.055064 3881 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-mpp7k] addLogicalPort took 50.246915ms, libovsdb time 866.063µs I0327 23:01:57.055586 3881 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-wqpfj] addLogicalPort took 35.416755ms, libovsdb time 1.297226ms I0327 23:01:57.055705 3881 pod.go:62] [openshift-console-operator/console-operator-8777bd9bb-qzpgd] pod update took 33.56547ms I0327 23:01:57.055720 3881 base_network_controller_pods.go:937] [default/openshift-console-operator/console-operator-8777bd9bb-qzpgd] addLogicalPort annotation time took 33.581376ms I0327 23:01:57.056052 3881 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-mcwgp] addLogicalPort took 56.819496ms, libovsdb time 1.34671ms I0327 23:01:57.056161 3881 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5fdfb574f5-bfc42] creating logical port openshift-ingress_router-default-5fdfb574f5-bfc42 for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:57.056240 3881 kube.go:256] Updating pod openshift-ingress/router-default-5fdfb574f5-bfc42 I0327 23:01:57.057167 3881 pods.go:271] [openshift-console-operator/console-operator-8777bd9bb-qzpgd] addLogicalPort took 35.070848ms, libovsdb time 685.906µs I0327 23:01:57.058647 3881 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-7582l] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-7582l for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:57.058720 3881 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-sx8wf] creating logical port openshift-network-diagnostics_network-check-source-5c76bd6f9b-sx8wf for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:57.058738 3881 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-7582l I0327 23:01:57.058896 3881 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-5c76bd6f9b-sx8wf I0327 23:01:57.058935 3881 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29577540-2l24x for pod on switch ip-10-0-2-103.ec2.internal I0327 23:01:57.059034 3881 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x I0327 23:01:57.067963 3881 pod.go:62] [openshift-ingress/router-default-75d6fd4fb5-7jfmc] pod update took 45.231073ms I0327 23:01:57.067987 3881 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-75d6fd4fb5-7jfmc] addLogicalPort annotation time took 45.255127ms I0327 23:01:57.069154 3881 pods.go:271] [openshift-ingress/router-default-75d6fd4fb5-7jfmc] addLogicalPort took 46.461171ms, libovsdb time 671.689µs I0327 23:01:57.075707 3881 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-rrjv8] pod update took 48.081926ms I0327 23:01:57.075726 3881 base_network_controller_pods.go:937] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-rrjv8] addLogicalPort annotation time took 48.098763ms I0327 23:01:57.077054 3881 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-rrjv8] addLogicalPort took 49.459904ms, libovsdb time 642.82µs I0327 23:01:57.083840 3881 pod.go:62] [openshift-ingress-canary/ingress-canary-tl9fl] pod update took 29.151795ms I0327 23:01:57.083859 3881 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-tl9fl] addLogicalPort annotation time took 29.169026ms I0327 23:01:57.084006 3881 pod.go:62] [openshift-insights/insights-operator-5577457d66-4gkzq] pod update took 29.883184ms I0327 23:01:57.084027 3881 base_network_controller_pods.go:937] [default/openshift-insights/insights-operator-5577457d66-4gkzq] addLogicalPort annotation time took 29.908103ms I0327 23:01:57.084121 3881 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x] pod update took 25.114795ms I0327 23:01:57.084143 3881 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x] addLogicalPort annotation time took 25.136525ms I0327 23:01:57.084585 3881 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-7582l] pod update took 25.866255ms I0327 23:01:57.084607 3881 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-7582l] addLogicalPort annotation time took 25.887784ms I0327 23:01:57.085143 3881 pods.go:271] [openshift-ingress-canary/ingress-canary-tl9fl] addLogicalPort took 30.52662ms, libovsdb time 925.314µs I0327 23:01:57.085409 3881 pods.go:271] [openshift-insights/insights-operator-5577457d66-4gkzq] addLogicalPort took 31.416302ms, libovsdb time 1.093297ms I0327 23:01:57.085737 3881 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x] addLogicalPort took 26.810677ms, libovsdb time 1.157028ms I0327 23:01:57.086071 3881 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-7582l] addLogicalPort took 27.429284ms, libovsdb time 1.193889ms I0327 23:01:57.087820 3881 pod.go:62] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-sx8wf] pod update took 28.958967ms I0327 23:01:57.087838 3881 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-sx8wf] addLogicalPort annotation time took 28.983261ms I0327 23:01:57.088651 3881 pods.go:271] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-sx8wf] addLogicalPort took 29.939672ms, libovsdb time 575.143µs I0327 23:01:57.091548 3881 pod.go:62] [openshift-ingress/router-default-5fdfb574f5-bfc42] pod update took 35.340187ms I0327 23:01:57.091564 3881 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-5fdfb574f5-bfc42] addLogicalPort annotation time took 35.356757ms I0327 23:01:57.092377 3881 pods.go:271] [openshift-ingress/router-default-5fdfb574f5-bfc42] addLogicalPort took 36.224495ms, libovsdb time 562.153µs I0327 23:01:57.347704 3881 cni.go:403] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-6jw22 643fb9d40eb22991fca1ec634f04b74559e6d2292e0a3f0d66e7e3b2cd7e42a4 network default NAD default] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-6jw22 643fb9d40eb22991fca1ec634f04b74559e6d2292e0a3f0d66e7e3b2cd7e42a4 network default NAD default] I0327 23:01:57.364295 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-7b89bc695f-6jw22, hostIfaceName: 643fb9d40eb2299, network: default, NAD default, SandboxID: "643fb9d40eb22991fca1ec634f04b74559e6d2292e0a3f0d66e7e3b2cd7e42a4", PCI device ID: , UID: "23f938eb-d702-47cd-9645-918c5e9526dd", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0327 23:01:57.405832 3881 cni.go:424] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-6jw22 643fb9d40eb22991fca1ec634f04b74559e6d2292e0a3f0d66e7e3b2cd7e42a4 network default NAD default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-6jw22 643fb9d40eb22991fca1ec634f04b74559e6d2292e0a3f0d66e7e3b2cd7e42a4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:d2:27:c9:9b:8b\",\"name\":\"643fb9d40eb2299\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/363f16ae-9e64-4e14-bd28-390c35165323\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:01:57.432612 3881 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-b9fwq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:57.432636 3881 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-b9fwq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:57.481737 3881 cni.go:403] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-sx8wf 48032ddb06c33e3c6fc5d37e8e9e2e1d872dc4dc0aef90d8802595c947da7913 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-sx8wf 48032ddb06c33e3c6fc5d37e8e9e2e1d872dc4dc0aef90d8802595c947da7913 network default NAD default] I0327 23:01:57.482035 3881 cni.go:403] [openshift-insights/insights-operator-5577457d66-4gkzq 4a64bb78fa6e89a7b3ff0a5bc8f8df24a44192f14502323080726b8e41d9efd6 network default NAD default] ADD starting CNI request [openshift-insights/insights-operator-5577457d66-4gkzq 4a64bb78fa6e89a7b3ff0a5bc8f8df24a44192f14502323080726b8e41d9efd6 network default NAD default] I0327 23:01:57.482958 3881 cni.go:403] [openshift-ingress-canary/ingress-canary-tl9fl 594b5c38e109a994fa3c9f7b8de76912a8e733ed50b111a9dc9c1971ea096092 network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-tl9fl 594b5c38e109a994fa3c9f7b8de76912a8e733ed50b111a9dc9c1971ea096092 network default NAD default] I0327 23:01:57.482963 3881 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x 37f53b9c053919c1ac1d38409d1ddf0710682d4bd8985db55c29202af609063c network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x 37f53b9c053919c1ac1d38409d1ddf0710682d4bd8985db55c29202af609063c network default NAD default] I0327 23:01:57.484020 3881 cni.go:403] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-kdwkt ce9cab387d24b75ccc4ad26f9798b13e565e990a430ea005bd6b1cf43933ccf3 network default NAD default] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-kdwkt ce9cab387d24b75ccc4ad26f9798b13e565e990a430ea005bd6b1cf43933ccf3 network default NAD default] I0327 23:01:57.484810 3881 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-7582l 4f784ba0f77cea540cecb67cdd65f455c0eac6c848f8f881f8e658879db6450b network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-7582l 4f784ba0f77cea540cecb67cdd65f455c0eac6c848f8f881f8e658879db6450b network default NAD default] I0327 23:01:57.500391 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-5c76bd6f9b-sx8wf, hostIfaceName: 48032ddb06c33e3, network: default, NAD default, SandboxID: "48032ddb06c33e3c6fc5d37e8e9e2e1d872dc4dc0aef90d8802595c947da7913", PCI device ID: , UID: "5cf18ec8-97e5-4e48-8cd8-0ab1cd2c9f87", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0327 23:01:57.503105 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-555cc59f77-kdwkt, hostIfaceName: ce9cab387d24b75, network: default, NAD default, SandboxID: "ce9cab387d24b75ccc4ad26f9798b13e565e990a430ea005bd6b1cf43933ccf3", PCI device ID: , UID: "13d3c5b5-556f-40f8-8810-ed715da1ac2e", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0327 23:01:57.503239 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-tl9fl, hostIfaceName: 594b5c38e109a99, network: default, NAD default, SandboxID: "594b5c38e109a994fa3c9f7b8de76912a8e733ed50b111a9dc9c1971ea096092", PCI device ID: , UID: "a9416d52-b511-4bfc-86ce-240fb851a837", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0327 23:01:57.507368 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29577540-2l24x, hostIfaceName: 37f53b9c053919c, network: default, NAD default, SandboxID: "37f53b9c053919c1ac1d38409d1ddf0710682d4bd8985db55c29202af609063c", PCI device ID: , UID: "97f82dfe-a29f-4477-b475-b089f8e3f26a", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0327 23:01:57.508426 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-5577457d66-4gkzq, hostIfaceName: 4a64bb78fa6e89a, network: default, NAD default, SandboxID: "4a64bb78fa6e89a7b3ff0a5bc8f8df24a44192f14502323080726b8e41d9efd6", PCI device ID: , UID: "eb451b45-b6c4-4013-b65a-d9772091af65", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0327 23:01:57.510583 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-7582l, hostIfaceName: 4f784ba0f77cea5, network: default, NAD default, SandboxID: "4f784ba0f77cea540cecb67cdd65f455c0eac6c848f8f881f8e658879db6450b", PCI device ID: , UID: "eadb2080-c097-48c3-a86f-49bda2b2af25", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0327 23:01:57.548719 3881 cni.go:424] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-sx8wf 48032ddb06c33e3c6fc5d37e8e9e2e1d872dc4dc0aef90d8802595c947da7913 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-sx8wf 48032ddb06c33e3c6fc5d37e8e9e2e1d872dc4dc0aef90d8802595c947da7913 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:fb:d7:c2:b5:7d\",\"name\":\"48032ddb06c33e3\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/07802235-5c4a-4cd6-b5fc-793662a13c33\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:01:57.557460 3881 cni.go:424] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-kdwkt ce9cab387d24b75ccc4ad26f9798b13e565e990a430ea005bd6b1cf43933ccf3 network default NAD default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-kdwkt ce9cab387d24b75ccc4ad26f9798b13e565e990a430ea005bd6b1cf43933ccf3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:0c:0f:6d:7b:87\",\"name\":\"ce9cab387d24b75\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4fcea590-57c2-4e98-8fdb-f7c2db030a61\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:01:57.562190 3881 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x 37f53b9c053919c1ac1d38409d1ddf0710682d4bd8985db55c29202af609063c network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x 37f53b9c053919c1ac1d38409d1ddf0710682d4bd8985db55c29202af609063c network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:bb:0e:a7:a5:84\",\"name\":\"37f53b9c053919c\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e025cead-d3f7-4647-9961-8a7558a12603\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:01:57.563493 3881 cni.go:424] [openshift-ingress-canary/ingress-canary-tl9fl 594b5c38e109a994fa3c9f7b8de76912a8e733ed50b111a9dc9c1971ea096092 network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-tl9fl 594b5c38e109a994fa3c9f7b8de76912a8e733ed50b111a9dc9c1971ea096092 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:f6:ae:7b:05:2a\",\"name\":\"594b5c38e109a99\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9ebcbd9d-8f8b-464e-95b2-fe489f2f39d5\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:01:57.571555 3881 cni.go:424] [openshift-insights/insights-operator-5577457d66-4gkzq 4a64bb78fa6e89a7b3ff0a5bc8f8df24a44192f14502323080726b8e41d9efd6 network default NAD default] ADD finished CNI request [openshift-insights/insights-operator-5577457d66-4gkzq 4a64bb78fa6e89a7b3ff0a5bc8f8df24a44192f14502323080726b8e41d9efd6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:8f:da:fb:b8:a4\",\"name\":\"4a64bb78fa6e89a\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8bab4246-2c8c-40ec-9768-b545203c66b5\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:01:57.571811 3881 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-7582l 4f784ba0f77cea540cecb67cdd65f455c0eac6c848f8f881f8e658879db6450b network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-7582l 4f784ba0f77cea540cecb67cdd65f455c0eac6c848f8f881f8e658879db6450b network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:06:0f:27:4f:5c\",\"name\":\"4f784ba0f77cea5\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0318054a-c010-4654-9b6d-b1cb3cff473a\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:01:57.666138 3881 cni.go:403] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-mcwgp 6691238f179af927be88d77be22ced3aa350068950746d8d34b620ca511ab36d network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-mcwgp 6691238f179af927be88d77be22ced3aa350068950746d8d34b620ca511ab36d network default NAD default] I0327 23:01:57.675131 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-6ff64bb8b6-mcwgp, hostIfaceName: 6691238f179af92, network: default, NAD default, SandboxID: "6691238f179af927be88d77be22ced3aa350068950746d8d34b620ca511ab36d", PCI device ID: , UID: "5e6f8673-08b8-4c86-ab4d-4609ee770ec8", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0327 23:01:57.685599 3881 cni.go:403] [openshift-console-operator/console-operator-8777bd9bb-qzpgd ae29783328914a2b2f8f23433be040589ed9daa9b2f1b2ab5386a29ceda2bfc4 network default NAD default] ADD starting CNI request [openshift-console-operator/console-operator-8777bd9bb-qzpgd ae29783328914a2b2f8f23433be040589ed9daa9b2f1b2ab5386a29ceda2bfc4 network default NAD default] I0327 23:01:57.693399 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-8777bd9bb-qzpgd, hostIfaceName: ae29783328914a2, network: default, NAD default, SandboxID: "ae29783328914a2b2f8f23433be040589ed9daa9b2f1b2ab5386a29ceda2bfc4", PCI device ID: , UID: "3abfa07e-0604-47b7-a498-a8c1546291d8", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0327 23:01:57.717849 3881 cni.go:424] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-mcwgp 6691238f179af927be88d77be22ced3aa350068950746d8d34b620ca511ab36d network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-mcwgp 6691238f179af927be88d77be22ced3aa350068950746d8d34b620ca511ab36d network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:6c:9a:c3:69:f0\",\"name\":\"6691238f179af92\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b968ec44-1f7c-4142-bd4e-e04b49ec0b46\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:01:57.732543 3881 cni.go:424] [openshift-console-operator/console-operator-8777bd9bb-qzpgd ae29783328914a2b2f8f23433be040589ed9daa9b2f1b2ab5386a29ceda2bfc4 network default NAD default] ADD finished CNI request [openshift-console-operator/console-operator-8777bd9bb-qzpgd ae29783328914a2b2f8f23433be040589ed9daa9b2f1b2ab5386a29ceda2bfc4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:58:49:94:d2:1f\",\"name\":\"ae29783328914a2\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e91a77f5-e7e4-4c71-80de-18f5121893ff\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:01:59.294273 3881 default_network_controller.go:1008] Node "ip-10-0-0-182.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 23:02:04.344618 3881 default_network_controller.go:1008] Node "ip-10-0-1-182.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 23:02:04.344628 3881 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-182.ec2.internal I0327 23:02:04.344705 3881 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-182.ec2.internal: 82.105µs I0327 23:02:04.344708 3881 egressfirewall.go:628] Syncing node "ip-10-0-1-182.ec2.internal" for egress firewall I0327 23:02:05.438388 3881 cni.go:403] [openshift-network-diagnostics/network-check-target-67rsf b2d175565b3d8e7f1ec6fcd690c70996a8c7ee4b45c0301acf8c99916ab6adb7 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-67rsf b2d175565b3d8e7f1ec6fcd690c70996a8c7ee4b45c0301acf8c99916ab6adb7 network default NAD default] I0327 23:02:05.448222 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-67rsf, hostIfaceName: b2d175565b3d8e7, network: default, NAD default, SandboxID: "b2d175565b3d8e7f1ec6fcd690c70996a8c7ee4b45c0301acf8c99916ab6adb7", PCI device ID: , UID: "097d09dd-a849-4189-9b9c-3d82f35d04a4", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0327 23:02:05.490284 3881 cni.go:424] [openshift-network-diagnostics/network-check-target-67rsf b2d175565b3d8e7f1ec6fcd690c70996a8c7ee4b45c0301acf8c99916ab6adb7 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-67rsf b2d175565b3d8e7f1ec6fcd690c70996a8c7ee4b45c0301acf8c99916ab6adb7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:2d:08:b5:75:24\",\"name\":\"b2d175565b3d8e7\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/22273c3a-3217-4599-8239-cbb75f3a2a9a\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:05.577389 3881 cni.go:403] [kube-system/global-pull-secret-syncer-5mcbr 9a24a4741dae27793ef19f0419447eb167b713a8b2c349a106f569bb498bc538 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-5mcbr 9a24a4741dae27793ef19f0419447eb167b713a8b2c349a106f569bb498bc538 network default NAD default] I0327 23:02:05.587312 3881 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-5mcbr, hostIfaceName: 9a24a4741dae277, network: default, NAD default, SandboxID: "9a24a4741dae27793ef19f0419447eb167b713a8b2c349a106f569bb498bc538", PCI device ID: , UID: "524a73b6-4b3e-4533-afdd-ca8f71a6cd86", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0327 23:02:05.627604 3881 cni.go:424] [kube-system/global-pull-secret-syncer-5mcbr 9a24a4741dae27793ef19f0419447eb167b713a8b2c349a106f569bb498bc538 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-5mcbr 9a24a4741dae27793ef19f0419447eb167b713a8b2c349a106f569bb498bc538 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:51:84:5e:84:7c\",\"name\":\"9a24a4741dae277\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ee5638b6-a700-4a91-8c6f-8cef9d75cf50\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:02:06.304887 3881 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0327 23:02:06.487124 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0327 23:02:06.513213 3881 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 23:02:07.494232 3881 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0327 23:02:07.495417 3881 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 1.156708ms I0327 23:02:07.524714 3881 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 23:02:07.616648 3881 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-llknw] creating logical port openshift-kube-storage-version-migrator_migrator-854b5fb5c9-llknw for pod on switch ip-10-0-2-103.ec2.internal I0327 23:02:07.616785 3881 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-854b5fb5c9-llknw I0327 23:02:07.637657 3881 pod.go:62] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-llknw] pod update took 20.923595ms I0327 23:02:07.637684 3881 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-llknw] addLogicalPort annotation time took 20.95416ms I0327 23:02:07.639383 3881 pods.go:271] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-llknw] addLogicalPort took 22.748766ms, libovsdb time 1.23448ms I0327 23:02:08.002212 3881 namespace.go:98] [openshift-service-ca] adding namespace I0327 23:02:08.003421 3881 namespace.go:102] [openshift-service-ca] adding namespace took 1.11541ms I0327 23:02:08.019673 3881 namespace.go:142] [openshift-service-ca] updating namespace I0327 23:02:08.043489 3881 cni.go:403] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-llknw 44391ffeba0f72b07c159730dd50a8c90ccf60b6e409a1eecb958104d1150de6 network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-llknw 44391ffeba0f72b07c159730dd50a8c90ccf60b6e409a1eecb958104d1150de6 network default NAD default] I0327 23:02:08.066826 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-854b5fb5c9-llknw, hostIfaceName: 44391ffeba0f72b, network: default, NAD default, SandboxID: "44391ffeba0f72b07c159730dd50a8c90ccf60b6e409a1eecb958104d1150de6", PCI device ID: , UID: "12bc3e7f-9cf9-4d19-9a4a-9a812841720b", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0327 23:02:08.110821 3881 cni.go:424] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-llknw 44391ffeba0f72b07c159730dd50a8c90ccf60b6e409a1eecb958104d1150de6 network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-llknw 44391ffeba0f72b07c159730dd50a8c90ccf60b6e409a1eecb958104d1150de6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:6f:95:4e:05:91\",\"name\":\"44391ffeba0f72b\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fdac2825-847a-430b-92d3-9f67e9fb19a4\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:08.430654 3881 default_network_controller.go:1008] Node "ip-10-0-1-182.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 23:02:08.456152 3881 default_network_controller.go:1008] Node "ip-10-0-0-182.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 23:02:09.140102 3881 egressfirewall.go:628] Syncing node "ip-10-0-0-182.ec2.internal" for egress firewall I0327 23:02:09.140109 3881 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-182.ec2.internal I0327 23:02:09.140129 3881 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-182.ec2.internal: 25.41µs I0327 23:02:09.140163 3881 default_network_controller.go:1008] Node "ip-10-0-0-182.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 23:02:09.380728 3881 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-182.ec2.internal in network "default" I0327 23:02:09.380806 3881 node_tracker.go:235] Node ip-10-0-1-182.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-182.ec2.internal" I0327 23:02:09.380828 3881 node_tracker.go:169] Node ip-10-0-1-182.ec2.internal switch + router changed, syncing services in network "default" I0327 23:02:09.380847 3881 services_controller.go:574] Full service sync requested for network=default I0327 23:02:09.380845 3881 default_network_controller.go:1008] Node "ip-10-0-1-182.ec2.internal" in remote zone "ip-10-0-1-182.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0327 23:02:09.381143 3881 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0327 23:02:09.381408 3881 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 23:02:09.381433 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 23:02:09.464107 3881 default_network_controller.go:1008] Node "ip-10-0-1-182.ec2.internal" in remote zone "ip-10-0-1-182.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 23:02:09.508870 3881 default_network_controller.go:1008] Node "ip-10-0-1-182.ec2.internal" in remote zone "ip-10-0-1-182.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 23:02:09.508916 3881 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-182.ec2.internal in network "default" I0327 23:02:09.508974 3881 node_tracker.go:169] Node ip-10-0-1-182.ec2.internal switch + router changed, syncing services in network "default" I0327 23:02:09.508990 3881 services_controller.go:574] Full service sync requested for network=default W0327 23:02:09.509256 3881 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0327 23:02:09.509399 3881 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 23:02:09.509411 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 23:02:09.509638 3881 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-182.ec2.internal for the network default I0327 23:02:09.511068 3881 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-182.ec2.internal" on network "default" took: 1.443053ms I0327 23:02:09.676563 3881 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-f59fc589b-k5xhr] creating logical port openshift-service-ca_service-ca-f59fc589b-k5xhr for pod on switch ip-10-0-2-103.ec2.internal I0327 23:02:09.676688 3881 kube.go:256] Updating pod openshift-service-ca/service-ca-f59fc589b-k5xhr I0327 23:02:09.689498 3881 pod.go:62] [openshift-service-ca/service-ca-f59fc589b-k5xhr] pod update took 12.833285ms I0327 23:02:09.689519 3881 base_network_controller_pods.go:937] [default/openshift-service-ca/service-ca-f59fc589b-k5xhr] addLogicalPort annotation time took 12.855906ms I0327 23:02:09.690636 3881 pods.go:271] [openshift-service-ca/service-ca-f59fc589b-k5xhr] addLogicalPort took 14.090344ms, libovsdb time 779.925µs I0327 23:02:10.070438 3881 cni.go:403] [openshift-service-ca/service-ca-f59fc589b-k5xhr b2bef12d743955e6c9ef4c82e3a23042f64cf74ec0030580e53b62ff377de36c network default NAD default] ADD starting CNI request [openshift-service-ca/service-ca-f59fc589b-k5xhr b2bef12d743955e6c9ef4c82e3a23042f64cf74ec0030580e53b62ff377de36c network default NAD default] I0327 23:02:10.079439 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-f59fc589b-k5xhr, hostIfaceName: b2bef12d743955e, network: default, NAD default, SandboxID: "b2bef12d743955e6c9ef4c82e3a23042f64cf74ec0030580e53b62ff377de36c", PCI device ID: , UID: "8a455090-44f7-415b-90f5-41e5a59e0191", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0327 23:02:10.126277 3881 cni.go:424] [openshift-service-ca/service-ca-f59fc589b-k5xhr b2bef12d743955e6c9ef4c82e3a23042f64cf74ec0030580e53b62ff377de36c network default NAD default] ADD finished CNI request [openshift-service-ca/service-ca-f59fc589b-k5xhr b2bef12d743955e6c9ef4c82e3a23042f64cf74ec0030580e53b62ff377de36c network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:c2:0d:49:74:a4\",\"name\":\"b2bef12d743955e\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9f7c1d7f-9ff6-47db-9c7b-a77dcf3f1b12\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:11.060211 3881 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-182.ec2.internal in network "default" I0327 23:02:11.060231 3881 egressfirewall.go:628] Syncing node "ip-10-0-1-182.ec2.internal" for egress firewall I0327 23:02:11.060243 3881 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-182.ec2.internal I0327 23:02:11.060262 3881 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-182.ec2.internal: 24.089µs I0327 23:02:11.060279 3881 node_tracker.go:169] Node ip-10-0-1-182.ec2.internal switch + router changed, syncing services in network "default" I0327 23:02:11.060286 3881 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-182.ec2.internal", IP: 10.0.1.182 to PMTUD blocking rules I0327 23:02:11.060299 3881 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-182.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 23:02:11.060291 3881 services_controller.go:574] Full service sync requested for network=default W0327 23:02:11.060574 3881 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0327 23:02:11.060683 3881 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 23:02:11.060717 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 23:02:11.066509 3881 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-182.ec2.internal. Updating no re-route policies I0327 23:02:11.077352 3881 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-182.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 23:02:11.077366 3881 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-182.ec2.internal", IP: 10.0.1.182 to PMTUD blocking rules I0327 23:02:12.644060 3881 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-ppfcq] creating logical port openshift-insights_insights-runtime-extractor-ppfcq for pod on switch ip-10-0-2-103.ec2.internal I0327 23:02:12.644158 3881 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-ppfcq I0327 23:02:12.658755 3881 pod.go:62] [openshift-insights/insights-runtime-extractor-ppfcq] pod update took 14.623627ms I0327 23:02:12.658795 3881 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-ppfcq] addLogicalPort annotation time took 14.669103ms I0327 23:02:12.659967 3881 pods.go:271] [openshift-insights/insights-runtime-extractor-ppfcq] addLogicalPort took 15.921129ms, libovsdb time 830.311µs I0327 23:02:13.280872 3881 cni.go:403] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-rrjv8 0bf636f78c5d2c464bb2ea8c33e4343f5a3697c1d7b9cbcbc09f04f25998471e network default NAD default] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-rrjv8 0bf636f78c5d2c464bb2ea8c33e4343f5a3697c1d7b9cbcbc09f04f25998471e network default NAD default] I0327 23:02:13.290428 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67b99c68bb-rrjv8, hostIfaceName: 0bf636f78c5d2c4, network: default, NAD default, SandboxID: "0bf636f78c5d2c464bb2ea8c33e4343f5a3697c1d7b9cbcbc09f04f25998471e", PCI device ID: , UID: "b1468328-b3e5-434f-a668-cec50bf4880e", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0327 23:02:13.332807 3881 cni.go:424] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-rrjv8 0bf636f78c5d2c464bb2ea8c33e4343f5a3697c1d7b9cbcbc09f04f25998471e network default NAD default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-rrjv8 0bf636f78c5d2c464bb2ea8c33e4343f5a3697c1d7b9cbcbc09f04f25998471e network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:f9:7b:8f:8e:da\",\"name\":\"0bf636f78c5d2c4\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8a2478da-1415-4c8a-9bc2-568f14de4a74\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:14.754521 3881 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-182.ec2.internal I0327 23:02:14.754550 3881 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-182.ec2.internal: 38.103µs W0327 23:02:14.902886 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 23:02:15.159838 3881 default_network_controller.go:1008] Node "ip-10-0-0-182.ec2.internal" in remote zone "ip-10-0-0-182.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 23:02:15.159888 3881 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-182.ec2.internal in network "default" I0327 23:02:15.159913 3881 node_tracker.go:235] Node ip-10-0-0-182.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-182.ec2.internal" I0327 23:02:15.159929 3881 node_tracker.go:169] Node ip-10-0-0-182.ec2.internal switch + router changed, syncing services in network "default" I0327 23:02:15.159942 3881 services_controller.go:574] Full service sync requested for network=default W0327 23:02:15.160243 3881 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0327 23:02:15.160361 3881 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 23:02:15.160370 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 23:02:15.203527 3881 default_network_controller.go:1008] Node "ip-10-0-0-182.ec2.internal" in remote zone "ip-10-0-0-182.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 23:02:15.274630 3881 default_network_controller.go:1008] Node "ip-10-0-0-182.ec2.internal" in remote zone "ip-10-0-0-182.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 23:02:15.274713 3881 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-182.ec2.internal in network "default" I0327 23:02:15.274781 3881 node_tracker.go:169] Node ip-10-0-0-182.ec2.internal switch + router changed, syncing services in network "default" I0327 23:02:15.274796 3881 services_controller.go:574] Full service sync requested for network=default W0327 23:02:15.275135 3881 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0327 23:02:15.275247 3881 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 23:02:15.275262 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 23:02:15.275410 3881 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-182.ec2.internal for the network default I0327 23:02:15.276773 3881 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-182.ec2.internal" on network "default" took: 1.366733ms W0327 23:02:15.295055 3881 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-69f999f58f-8wxsl IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-69f999f58f-8wxsl: no pod IPs found I0327 23:02:16.401185 3881 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5b6869f466-hd4bm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:16.401219 3881 pods.go:176] Deleting pod: openshift-image-registry/image-registry-5b6869f466-hd4bm I0327 23:02:16.402514 3881 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-5b6869f466-hd4bm, ips: 10.128.0.6 I0327 23:02:16.402537 3881 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5b6869f466-hd4bm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:16.413245 3881 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-d7f86bd67-mwdgd] creating logical port openshift-image-registry_image-registry-d7f86bd67-mwdgd for pod on switch ip-10-0-2-103.ec2.internal I0327 23:02:16.413324 3881 kube.go:256] Updating pod openshift-image-registry/image-registry-d7f86bd67-mwdgd I0327 23:02:16.414426 3881 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5b6869f466-hd4bm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:02:16.414492 3881 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5b6869f466-hd4bm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:02:16.424196 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5b6869f466-hd4bm I0327 23:02:16.424218 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5b6869f466-hd4bm I0327 23:02:16.432139 3881 pod.go:62] [openshift-image-registry/image-registry-d7f86bd67-mwdgd] pod update took 18.839254ms I0327 23:02:16.432160 3881 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-d7f86bd67-mwdgd] addLogicalPort annotation time took 18.859917ms I0327 23:02:16.433163 3881 pods.go:271] [openshift-image-registry/image-registry-d7f86bd67-mwdgd] addLogicalPort took 19.928031ms, libovsdb time 688.321µs W0327 23:02:16.498199 3881 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 23:02:16.815011 3881 cni.go:403] [openshift-image-registry/image-registry-d7f86bd67-mwdgd b54436a81d35d8addb883da21ddb907a2a520dd879e54d4502c387dd174d6e56 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-d7f86bd67-mwdgd b54436a81d35d8addb883da21ddb907a2a520dd879e54d4502c387dd174d6e56 network default NAD default] I0327 23:02:16.823814 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-d7f86bd67-mwdgd, hostIfaceName: b54436a81d35d8a, network: default, NAD default, SandboxID: "b54436a81d35d8addb883da21ddb907a2a520dd879e54d4502c387dd174d6e56", PCI device ID: , UID: "0e52205b-6214-44a9-9ee2-993d27973b6b", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0327 23:02:16.828250 3881 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-182.ec2.internal I0327 23:02:16.828269 3881 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-182.ec2.internal: 21.241µs I0327 23:02:16.828338 3881 egressfirewall.go:628] Syncing node "ip-10-0-0-182.ec2.internal" for egress firewall I0327 23:02:16.828349 3881 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-182.ec2.internal in network "default" I0327 23:02:16.828411 3881 node_tracker.go:169] Node ip-10-0-0-182.ec2.internal switch + router changed, syncing services in network "default" I0327 23:02:16.828418 3881 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-182.ec2.internal", IP: 10.0.0.182 to PMTUD blocking rules I0327 23:02:16.828429 3881 services_controller.go:574] Full service sync requested for network=default I0327 23:02:16.828432 3881 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-182.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0327 23:02:16.828517 3881 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0327 23:02:16.828736 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0327 23:02:16.828747 3881 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 23:02:16.828923 3881 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 23:02:16.835046 3881 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-182.ec2.internal. Updating no re-route policies I0327 23:02:16.848660 3881 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-182.ec2.internal", IP: 10.0.0.182 to PMTUD blocking rules I0327 23:02:16.848676 3881 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-182.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 23:02:16.864692 3881 cni.go:424] [openshift-image-registry/image-registry-d7f86bd67-mwdgd b54436a81d35d8addb883da21ddb907a2a520dd879e54d4502c387dd174d6e56 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-d7f86bd67-mwdgd b54436a81d35d8addb883da21ddb907a2a520dd879e54d4502c387dd174d6e56 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:84:be:29:fc:71\",\"name\":\"b54436a81d35d8a\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/38b5e8d1-b9e0-42dc-9c5a-991b574c388a\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:28.658918 3881 cni.go:403] [openshift-insights/insights-runtime-extractor-ppfcq f1c3604be8f15d6a1d9bdf49f85c8f42a33f497a2e8a47e3ed137e84dc80a13d network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-ppfcq f1c3604be8f15d6a1d9bdf49f85c8f42a33f497a2e8a47e3ed137e84dc80a13d network default NAD default] I0327 23:02:28.667370 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-ppfcq, hostIfaceName: f1c3604be8f15d6, network: default, NAD default, SandboxID: "f1c3604be8f15d6a1d9bdf49f85c8f42a33f497a2e8a47e3ed137e84dc80a13d", PCI device ID: , UID: "c6d7b7db-6e3e-4787-8a0e-f53476c4b75e", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0327 23:02:28.709135 3881 cni.go:424] [openshift-insights/insights-runtime-extractor-ppfcq f1c3604be8f15d6a1d9bdf49f85c8f42a33f497a2e8a47e3ed137e84dc80a13d network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-ppfcq f1c3604be8f15d6a1d9bdf49f85c8f42a33f497a2e8a47e3ed137e84dc80a13d network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:01:cb:c1:b9:21\",\"name\":\"f1c3604be8f15d6\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9eb81df3-b500-48bc-a2b9-f588657b60a3\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:29.191163 3881 cni.go:403] [openshift-dns/dns-default-4x8xk e20e76fb059dd132d0bc2f6ad943f19291541fc25a53368ae1edd663d0e977fb network default NAD default] ADD starting CNI request [openshift-dns/dns-default-4x8xk e20e76fb059dd132d0bc2f6ad943f19291541fc25a53368ae1edd663d0e977fb network default NAD default] I0327 23:02:29.199838 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-4x8xk, hostIfaceName: e20e76fb059dd13, network: default, NAD default, SandboxID: "e20e76fb059dd132d0bc2f6ad943f19291541fc25a53368ae1edd663d0e977fb", PCI device ID: , UID: "a6895adb-17b7-4fd6-af49-1a50bcea25cd", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0327 23:02:29.226306 3881 cni.go:403] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-945sj e5dbc84a9d991c7e5112f33c8c207de0bcb8b88d19cf4ce0de38029b1d8faec5 network default NAD default] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-945sj e5dbc84a9d991c7e5112f33c8c207de0bcb8b88d19cf4ce0de38029b1d8faec5 network default NAD default] I0327 23:02:29.243556 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-6dd897695f-945sj, hostIfaceName: e5dbc84a9d991c7, network: default, NAD default, SandboxID: "e5dbc84a9d991c7e5112f33c8c207de0bcb8b88d19cf4ce0de38029b1d8faec5", PCI device ID: , UID: "3468a8c9-6478-4f08-9eb1-a69bbc4ff4c9", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0327 23:02:29.243752 3881 cni.go:424] [openshift-dns/dns-default-4x8xk e20e76fb059dd132d0bc2f6ad943f19291541fc25a53368ae1edd663d0e977fb network default NAD default] ADD finished CNI request [openshift-dns/dns-default-4x8xk e20e76fb059dd132d0bc2f6ad943f19291541fc25a53368ae1edd663d0e977fb network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:5c:31:5c:d4:ca\",\"name\":\"e20e76fb059dd13\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/51551e23-2e55-4562-90c8-0db20a676e47\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:29.273035 3881 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-mpp7k 534baec6366652ec827d65e22b222ffab8ab953125092dfcae8b9fefdb6c7ba1 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-mpp7k 534baec6366652ec827d65e22b222ffab8ab953125092dfcae8b9fefdb6c7ba1 network default NAD default] I0327 23:02:29.288979 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-mpp7k, hostIfaceName: 534baec6366652e, network: default, NAD default, SandboxID: "534baec6366652ec827d65e22b222ffab8ab953125092dfcae8b9fefdb6c7ba1", PCI device ID: , UID: "b4c305e1-d496-41bf-bcf5-1c7075633f68", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0327 23:02:29.289041 3881 cni.go:424] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-945sj e5dbc84a9d991c7e5112f33c8c207de0bcb8b88d19cf4ce0de38029b1d8faec5 network default NAD default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-945sj e5dbc84a9d991c7e5112f33c8c207de0bcb8b88d19cf4ce0de38029b1d8faec5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:31:81:f7:86:b2\",\"name\":\"e5dbc84a9d991c7\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/166f2cc4-0e9e-4f47-81c3-6cc38658c34c\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:29.308292 3881 cni.go:403] [openshift-ingress/router-default-5fdfb574f5-bfc42 2e17b65a252457003717bc672151dd0a4f2bc6c7369a09b42ad00036d75fd25e network default NAD default] ADD starting CNI request [openshift-ingress/router-default-5fdfb574f5-bfc42 2e17b65a252457003717bc672151dd0a4f2bc6c7369a09b42ad00036d75fd25e network default NAD default] I0327 23:02:29.321887 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5fdfb574f5-bfc42, hostIfaceName: 2e17b65a2524570, network: default, NAD default, SandboxID: "2e17b65a252457003717bc672151dd0a4f2bc6c7369a09b42ad00036d75fd25e", PCI device ID: , UID: "bd86c6aa-66ae-48fe-b803-9cda6a6789f6", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0327 23:02:29.337196 3881 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-mpp7k 534baec6366652ec827d65e22b222ffab8ab953125092dfcae8b9fefdb6c7ba1 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-mpp7k 534baec6366652ec827d65e22b222ffab8ab953125092dfcae8b9fefdb6c7ba1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:f0:c2:7a:fb:e5\",\"name\":\"534baec6366652e\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1c63b4f4-bcdd-4beb-97de-6f0b5006c495\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:29.338269 3881 cni.go:403] [openshift-ingress/router-default-75d6fd4fb5-7jfmc 14b1d9523fb3e0f2d99656a87b3595d3c1c51bc77ff97eb6650df0afdde94590 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-75d6fd4fb5-7jfmc 14b1d9523fb3e0f2d99656a87b3595d3c1c51bc77ff97eb6650df0afdde94590 network default NAD default] I0327 23:02:29.348899 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-75d6fd4fb5-7jfmc, hostIfaceName: 14b1d9523fb3e0f, network: default, NAD default, SandboxID: "14b1d9523fb3e0f2d99656a87b3595d3c1c51bc77ff97eb6650df0afdde94590", PCI device ID: , UID: "edfd5e46-3de1-4b9f-a797-82b93836af05", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0327 23:02:29.369398 3881 cni.go:424] [openshift-ingress/router-default-5fdfb574f5-bfc42 2e17b65a252457003717bc672151dd0a4f2bc6c7369a09b42ad00036d75fd25e network default NAD default] ADD finished CNI request [openshift-ingress/router-default-5fdfb574f5-bfc42 2e17b65a252457003717bc672151dd0a4f2bc6c7369a09b42ad00036d75fd25e network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:ad:7f:e7:8b:75\",\"name\":\"2e17b65a2524570\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a23a7038-850e-49e2-baca-f6974e85f44d\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:29.398687 3881 cni.go:424] [openshift-ingress/router-default-75d6fd4fb5-7jfmc 14b1d9523fb3e0f2d99656a87b3595d3c1c51bc77ff97eb6650df0afdde94590 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-75d6fd4fb5-7jfmc 14b1d9523fb3e0f2d99656a87b3595d3c1c51bc77ff97eb6650df0afdde94590 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:02:c5:a2:f9:bb\",\"name\":\"14b1d9523fb3e0f\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d6a7d56f-3c72-413e-a667-e3f2a37e57c0\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:29.400607 3881 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-wqpfj 88d19f6f46ea9ffca80ba229366a117ab3c890eb38157e6a788df94f6b1cef1d network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-wqpfj 88d19f6f46ea9ffca80ba229366a117ab3c890eb38157e6a788df94f6b1cef1d network default NAD default] I0327 23:02:29.420068 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-wqpfj, hostIfaceName: 88d19f6f46ea9ff, network: default, NAD default, SandboxID: "88d19f6f46ea9ffca80ba229366a117ab3c890eb38157e6a788df94f6b1cef1d", PCI device ID: , UID: "3a7e46bd-aff6-4c9b-b71d-d51931c725b8", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0327 23:02:29.462843 3881 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-wqpfj 88d19f6f46ea9ffca80ba229366a117ab3c890eb38157e6a788df94f6b1cef1d network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-wqpfj 88d19f6f46ea9ffca80ba229366a117ab3c890eb38157e6a788df94f6b1cef1d network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:78:ee:b5:97:e0\",\"name\":\"88d19f6f46ea9ff\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/58ce345d-7d52-4ff2-8af5-e8e7c525bb08\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:33.013271 3881 cni.go:403] [openshift-multus/network-metrics-daemon-wq5tj 74b65a9084610bbd959b9599b194eb0670fe865923f5ca9db467f5aaae33bf44 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-wq5tj 74b65a9084610bbd959b9599b194eb0670fe865923f5ca9db467f5aaae33bf44 network default NAD default] I0327 23:02:33.022413 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-wq5tj, hostIfaceName: 74b65a9084610bb, network: default, NAD default, SandboxID: "74b65a9084610bbd959b9599b194eb0670fe865923f5ca9db467f5aaae33bf44", PCI device ID: , UID: "79b72cb0-73dc-4c64-9317-5d19e0d3c402", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0327 23:02:33.070133 3881 cni.go:424] [openshift-multus/network-metrics-daemon-wq5tj 74b65a9084610bbd959b9599b194eb0670fe865923f5ca9db467f5aaae33bf44 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-wq5tj 74b65a9084610bbd959b9599b194eb0670fe865923f5ca9db467f5aaae33bf44 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:3f:89:17:ff:4d\",\"name\":\"74b65a9084610bb\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1c219b38-3c51-4e9b-b2b3-4c94899de85f\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:02:33.144067 3881 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 23:02:33.144088 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0327 23:02:33.191638 3881 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 23:02:33.253908 3881 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-2jvnw] creating logical port openshift-console_downloads-56d45c498b-2jvnw for pod on switch ip-10-0-2-103.ec2.internal I0327 23:02:33.254080 3881 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-2jvnw W0327 23:02:33.275621 3881 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 23:02:33.277717 3881 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 23:02:33.287553 3881 pod.go:62] [openshift-console/downloads-56d45c498b-2jvnw] pod update took 33.501622ms I0327 23:02:33.287573 3881 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-2jvnw] addLogicalPort annotation time took 33.525655ms I0327 23:02:33.288828 3881 pods.go:271] [openshift-console/downloads-56d45c498b-2jvnw] addLogicalPort took 34.938261ms, libovsdb time 884.768µs W0327 23:02:33.288903 3881 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-d7f86bd67-qjxfj IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-d7f86bd67-qjxfj: no pod IPs found W0327 23:02:33.291121 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics I0327 23:02:33.736047 3881 base_network_controller_pods.go:487] [default/openshift-console/console-8fdb84b49-nspfx] creating logical port openshift-console_console-8fdb84b49-nspfx for pod on switch ip-10-0-2-103.ec2.internal I0327 23:02:33.736168 3881 kube.go:256] Updating pod openshift-console/console-8fdb84b49-nspfx I0327 23:02:33.754947 3881 pod.go:62] [openshift-console/console-8fdb84b49-nspfx] pod update took 18.802928ms I0327 23:02:33.754964 3881 base_network_controller_pods.go:937] [default/openshift-console/console-8fdb84b49-nspfx] addLogicalPort annotation time took 18.826013ms I0327 23:02:33.756089 3881 pods.go:271] [openshift-console/console-8fdb84b49-nspfx] addLogicalPort took 20.068574ms, libovsdb time 766.227µs I0327 23:02:34.144016 3881 cni.go:403] [openshift-console/console-8fdb84b49-nspfx 403a436f403c256bc819d6d5c37f35d6506dad6a1c00cf339130f3928306a0ed network default NAD default] ADD starting CNI request [openshift-console/console-8fdb84b49-nspfx 403a436f403c256bc819d6d5c37f35d6506dad6a1c00cf339130f3928306a0ed network default NAD default] I0327 23:02:34.152832 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-8fdb84b49-nspfx, hostIfaceName: 403a436f403c256, network: default, NAD default, SandboxID: "403a436f403c256bc819d6d5c37f35d6506dad6a1c00cf339130f3928306a0ed", PCI device ID: , UID: "31b3d90e-cca4-48f7-89de-c5e06077bdb5", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0327 23:02:34.195372 3881 cni.go:424] [openshift-console/console-8fdb84b49-nspfx 403a436f403c256bc819d6d5c37f35d6506dad6a1c00cf339130f3928306a0ed network default NAD default] ADD finished CNI request [openshift-console/console-8fdb84b49-nspfx 403a436f403c256bc819d6d5c37f35d6506dad6a1c00cf339130f3928306a0ed network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:25:d6:93:99:24\",\"name\":\"403a436f403c256\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5cf75f99-be6f-498f-921e-19331f83eefb\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:34.309949 3881 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-trbm7] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-trbm7 for pod on switch ip-10-0-2-103.ec2.internal I0327 23:02:34.310085 3881 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-trbm7 I0327 23:02:34.325534 3881 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-trbm7] pod update took 15.481425ms I0327 23:02:34.325557 3881 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-trbm7] addLogicalPort annotation time took 15.50588ms I0327 23:02:34.326679 3881 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-trbm7] addLogicalPort took 16.756503ms, libovsdb time 770.879µs I0327 23:02:34.523226 3881 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-182.ec2.internal I0327 23:02:34.523258 3881 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-182.ec2.internal: 41.95µs I0327 23:02:34.544163 3881 cni.go:403] [openshift-console/downloads-56d45c498b-2jvnw b8d8ca23f532131f873cfebe674b81eabd1557869f8d9043bd3bb26756fe4d14 network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-2jvnw b8d8ca23f532131f873cfebe674b81eabd1557869f8d9043bd3bb26756fe4d14 network default NAD default] I0327 23:02:34.552525 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-2jvnw, hostIfaceName: b8d8ca23f532131, network: default, NAD default, SandboxID: "b8d8ca23f532131f873cfebe674b81eabd1557869f8d9043bd3bb26756fe4d14", PCI device ID: , UID: "f966255a-a162-43ef-aaee-975b6c2e3f49", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0327 23:02:34.595906 3881 cni.go:424] [openshift-console/downloads-56d45c498b-2jvnw b8d8ca23f532131f873cfebe674b81eabd1557869f8d9043bd3bb26756fe4d14 network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-2jvnw b8d8ca23f532131f873cfebe674b81eabd1557869f8d9043bd3bb26756fe4d14 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:8e:f4:cb:a6:f3\",\"name\":\"b8d8ca23f532131\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/79df5609-10e7-47a5-9063-65fdae9d40f5\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:02:35.304215 3881 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-trbm7 d2bf6a00757416ffdc1d4c67e9b146eab72dfcfe5bf30f8fc38e82f128919d3e network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-trbm7 d2bf6a00757416ffdc1d4c67e9b146eab72dfcfe5bf30f8fc38e82f128919d3e network default NAD default] I0327 23:02:35.313427 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-trbm7, hostIfaceName: d2bf6a00757416f, network: default, NAD default, SandboxID: "d2bf6a00757416ffdc1d4c67e9b146eab72dfcfe5bf30f8fc38e82f128919d3e", PCI device ID: , UID: "33249e99-ed74-421f-81e5-3645febb37e3", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] W0327 23:02:35.326939 3881 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 23:02:35.362435 3881 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-trbm7 d2bf6a00757416ffdc1d4c67e9b146eab72dfcfe5bf30f8fc38e82f128919d3e network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-trbm7 d2bf6a00757416ffdc1d4c67e9b146eab72dfcfe5bf30f8fc38e82f128919d3e network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:25:23:63:9e:e3\",\"name\":\"d2bf6a00757416f\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/863e4717-02ab-47e4-a06a-6ac46578f8c3\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:02:37.322260 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 23:02:38.382613 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 23:02:38.481997 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0327 23:02:38.506787 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0327 23:02:38.534599 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 23:02:38.567950 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 23:02:41.926547 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 23:02:43.461883 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 23:02:43.461905 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0327 23:02:43.461911 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 23:02:44.563911 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 23:02:49.405102 3881 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:02:49.405123 3881 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:02:49.405129 3881 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:02:49.686608 3881 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-77454c6899-jhjrs IP address from the namespace address-set, err: pod openshift-console/console-77454c6899-jhjrs: no pod IPs found W0327 23:02:50.542338 3881 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0327 23:02:50.570308 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0327 23:02:53.409018 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0327 23:02:53.434533 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 23:02:53.478902 3881 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 23:02:53.478923 3881 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0327 23:02:54.194650 3881 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0327 23:02:55.442834 3881 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:02:55.442858 3881 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:02:55.442865 3881 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:02:55.442869 3881 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:02:55.675241 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 23:02:55.719923 3881 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:02:55.719941 3881 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:02:55.719949 3881 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:02:55.838671 3881 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 23:02:55.838688 3881 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0327 23:02:58.755945 3881 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5b6869f466-7xmk7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:58.756455 3881 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5b6869f466-7xmk7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:59.702188 3881 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5b6869f466-7xmk7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:02:59.702208 3881 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5b6869f466-7xmk7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:02:59.721890 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5b6869f466-7xmk7 I0327 23:02:59.721917 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5b6869f466-7xmk7 I0327 23:02:59.878060 3881 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390bb8np of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:59.878619 3881 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390bb8np of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:59.878647 3881 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390bb8np of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:02:59.878657 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0327 23:02:59.878671 3881 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390bb8np of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:02:59.878677 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... W0327 23:03:00.731859 3881 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:03:00.731877 3881 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:03:00.731882 3881 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:03:00.731887 3881 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier I0327 23:03:00.906208 3881 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390bb8np of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:00.906226 3881 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390bb8np of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:00.906241 3881 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390bb8np of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:00.906250 3881 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390bb8np of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:03.884821 3881 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 23:03:03.894353 3881 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 23:03:07.256860 3881 obj_retry.go:464] Detected object openshift-console/console-8fdb84b49-x7vvz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:03:07.257430 3881 obj_retry.go:464] Detected object openshift-console/console-8fdb84b49-x7vvz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:03:07.759148 3881 obj_retry.go:458] Detected object openshift-console/console-8fdb84b49-x7vvz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:07.759169 3881 obj_retry.go:458] Detected object openshift-console/console-8fdb84b49-x7vvz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:07.767221 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-8fdb84b49-x7vvz I0327 23:03:07.767246 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-8fdb84b49-x7vvz I0327 23:03:14.944207 3881 obj_retry.go:464] Detected object openshift-console/console-77454c6899-kjrcp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:03:14.944704 3881 obj_retry.go:464] Detected object openshift-console/console-77454c6899-kjrcp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:03:15.531294 3881 obj_retry.go:458] Detected object openshift-console/console-77454c6899-kjrcp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:15.531317 3881 obj_retry.go:458] Detected object openshift-console/console-77454c6899-kjrcp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:15.537396 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-77454c6899-kjrcp I0327 23:03:15.537420 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-77454c6899-kjrcp I0327 23:03:21.557476 3881 cni.go:403] [openshift-image-registry/image-registry-d7f86bd67-mwdgd b54436a81d35d8addb883da21ddb907a2a520dd879e54d4502c387dd174d6e56 network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-d7f86bd67-mwdgd b54436a81d35d8addb883da21ddb907a2a520dd879e54d4502c387dd174d6e56 network default NAD default] I0327 23:03:21.624993 3881 cni.go:424] [openshift-image-registry/image-registry-d7f86bd67-mwdgd b54436a81d35d8addb883da21ddb907a2a520dd879e54d4502c387dd174d6e56 network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-d7f86bd67-mwdgd b54436a81d35d8addb883da21ddb907a2a520dd879e54d4502c387dd174d6e56 network default NAD default], result "{}", err W0327 23:03:21.682010 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 23:03:21.682007 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 23:03:21.682187 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0327 23:03:21.697903 3881 obj_retry.go:464] Detected object openshift-image-registry/image-registry-d7f86bd67-mwdgd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:03:21.698001 3881 pods.go:176] Deleting pod: openshift-image-registry/image-registry-d7f86bd67-mwdgd I0327 23:03:21.699220 3881 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-d7f86bd67-mwdgd, ips: 10.128.0.26 I0327 23:03:21.699246 3881 obj_retry.go:464] Detected object openshift-image-registry/image-registry-d7f86bd67-mwdgd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 23:03:22.697661 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0327 23:03:22.704525 3881 obj_retry.go:458] Detected object openshift-image-registry/image-registry-d7f86bd67-mwdgd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:22.704550 3881 obj_retry.go:458] Detected object openshift-image-registry/image-registry-d7f86bd67-mwdgd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:22.712664 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-d7f86bd67-mwdgd I0327 23:03:22.712687 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-d7f86bd67-mwdgd W0327 23:03:27.730844 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0327 23:03:27.730901 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0327 23:03:27.730899 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0327 23:03:28.729159 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0327 23:03:36.755035 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 23:03:36.755039 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 23:03:36.755090 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 23:03:37.762551 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0327 23:03:40.202319 3881 cni.go:403] [openshift-ingress/router-default-5fdfb574f5-bfc42 2e17b65a252457003717bc672151dd0a4f2bc6c7369a09b42ad00036d75fd25e network default NAD default] DEL starting CNI request [openshift-ingress/router-default-5fdfb574f5-bfc42 2e17b65a252457003717bc672151dd0a4f2bc6c7369a09b42ad00036d75fd25e network default NAD default] I0327 23:03:40.276110 3881 cni.go:424] [openshift-ingress/router-default-5fdfb574f5-bfc42 2e17b65a252457003717bc672151dd0a4f2bc6c7369a09b42ad00036d75fd25e network default NAD default] DEL finished CNI request [openshift-ingress/router-default-5fdfb574f5-bfc42 2e17b65a252457003717bc672151dd0a4f2bc6c7369a09b42ad00036d75fd25e network default NAD default], result "{}", err I0327 23:03:40.351595 3881 obj_retry.go:464] Detected object openshift-ingress/router-default-5fdfb574f5-bfc42 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:03:40.351625 3881 pods.go:176] Deleting pod: openshift-ingress/router-default-5fdfb574f5-bfc42 I0327 23:03:40.352854 3881 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-5fdfb574f5-bfc42, ips: 10.128.0.19 I0327 23:03:40.352880 3881 obj_retry.go:464] Detected object openshift-ingress/router-default-5fdfb574f5-bfc42 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:03:40.773064 3881 obj_retry.go:458] Detected object openshift-ingress/router-default-5fdfb574f5-bfc42 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:40.773100 3881 obj_retry.go:458] Detected object openshift-ingress/router-default-5fdfb574f5-bfc42 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:40.776227 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-5fdfb574f5-bfc42 I0327 23:03:40.776321 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-5fdfb574f5-bfc42 I0327 23:03:43.037644 3881 cni.go:403] [openshift-console/console-8fdb84b49-nspfx 403a436f403c256bc819d6d5c37f35d6506dad6a1c00cf339130f3928306a0ed network default NAD default] DEL starting CNI request [openshift-console/console-8fdb84b49-nspfx 403a436f403c256bc819d6d5c37f35d6506dad6a1c00cf339130f3928306a0ed network default NAD default] I0327 23:03:43.108125 3881 cni.go:424] [openshift-console/console-8fdb84b49-nspfx 403a436f403c256bc819d6d5c37f35d6506dad6a1c00cf339130f3928306a0ed network default NAD default] DEL finished CNI request [openshift-console/console-8fdb84b49-nspfx 403a436f403c256bc819d6d5c37f35d6506dad6a1c00cf339130f3928306a0ed network default NAD default], result "{}", err I0327 23:03:43.185419 3881 obj_retry.go:464] Detected object openshift-console/console-8fdb84b49-nspfx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:03:43.185452 3881 pods.go:176] Deleting pod: openshift-console/console-8fdb84b49-nspfx I0327 23:03:43.186678 3881 pods.go:236] Attempting to release IPs for pod: openshift-console/console-8fdb84b49-nspfx, ips: 10.128.0.28 I0327 23:03:43.186700 3881 obj_retry.go:464] Detected object openshift-console/console-8fdb84b49-nspfx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:03:43.789690 3881 obj_retry.go:458] Detected object openshift-console/console-8fdb84b49-nspfx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:43.789840 3881 obj_retry.go:458] Detected object openshift-console/console-8fdb84b49-nspfx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:03:43.795350 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-8fdb84b49-nspfx I0327 23:03:43.795375 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-8fdb84b49-nspfx I0327 23:04:06.788522 3881 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:06.789100 3881 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:07.042076 3881 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:07.042100 3881 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:07.056134 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0327 23:04:07.056160 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0327 23:04:11.128838 3881 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:11.129366 3881 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:12.070907 3881 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:12.070929 3881 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:12.078189 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0327 23:04:12.078219 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0327 23:04:13.046544 3881 base_network_controller_pods.go:487] [default/openshift-console/console-df959d948-tkwxt] creating logical port openshift-console_console-df959d948-tkwxt for pod on switch ip-10-0-2-103.ec2.internal I0327 23:04:13.046669 3881 kube.go:256] Updating pod openshift-console/console-df959d948-tkwxt I0327 23:04:13.069422 3881 pod.go:62] [openshift-console/console-df959d948-tkwxt] pod update took 22.789827ms I0327 23:04:13.069442 3881 base_network_controller_pods.go:937] [default/openshift-console/console-df959d948-tkwxt] addLogicalPort annotation time took 22.815466ms I0327 23:04:13.071047 3881 pods.go:271] [openshift-console/console-df959d948-tkwxt] addLogicalPort took 24.524044ms, libovsdb time 1.196738ms I0327 23:04:13.404840 3881 cni.go:403] [openshift-console/console-df959d948-tkwxt c0b01e47f8fbc69c5cb2339ce8a19eb37f34db4d2341ce71f852ffc70d32213f network default NAD default] ADD starting CNI request [openshift-console/console-df959d948-tkwxt c0b01e47f8fbc69c5cb2339ce8a19eb37f34db4d2341ce71f852ffc70d32213f network default NAD default] I0327 23:04:13.413408 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-df959d948-tkwxt, hostIfaceName: c0b01e47f8fbc69, network: default, NAD default, SandboxID: "c0b01e47f8fbc69c5cb2339ce8a19eb37f34db4d2341ce71f852ffc70d32213f", PCI device ID: , UID: "24e50807-29a0-4f81-9ac0-9fdad6729872", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0327 23:04:13.456522 3881 cni.go:424] [openshift-console/console-df959d948-tkwxt c0b01e47f8fbc69c5cb2339ce8a19eb37f34db4d2341ce71f852ffc70d32213f network default NAD default] ADD finished CNI request [openshift-console/console-df959d948-tkwxt c0b01e47f8fbc69c5cb2339ce8a19eb37f34db4d2341ce71f852ffc70d32213f network default NAD default], result "{\"interfaces\":[{\"mac\":\"76:92:0a:b9:d8:ca\",\"name\":\"c0b01e47f8fbc69\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/40a10d60-3fbd-426b-802e-a9a6f8adc403\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:04:23.994213 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 23:04:38.341909 3881 obj_retry.go:464] Detected object openshift-console/console-86b5bc979f-sdjlx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:38.342583 3881 obj_retry.go:464] Detected object openshift-console/console-86b5bc979f-sdjlx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:38.817263 3881 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:38.817789 3881 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:38.866187 3881 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:38.866207 3881 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:38.874438 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0327 23:04:38.874462 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0327 23:04:39.218503 3881 obj_retry.go:458] Detected object openshift-console/console-86b5bc979f-sdjlx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:39.218520 3881 obj_retry.go:458] Detected object openshift-console/console-86b5bc979f-sdjlx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:39.226803 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86b5bc979f-sdjlx I0327 23:04:39.226821 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86b5bc979f-sdjlx I0327 23:04:49.316546 3881 obj_retry.go:464] Detected object openshift-console/console-86b5bc979f-7trzh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:49.317172 3881 obj_retry.go:464] Detected object openshift-console/console-86b5bc979f-7trzh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:49.908082 3881 obj_retry.go:458] Detected object openshift-console/console-86b5bc979f-7trzh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:49.908107 3881 obj_retry.go:458] Detected object openshift-console/console-86b5bc979f-7trzh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:49.915356 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86b5bc979f-7trzh I0327 23:04:49.915392 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86b5bc979f-7trzh I0327 23:05:01.158921 3881 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x 37f53b9c053919c1ac1d38409d1ddf0710682d4bd8985db55c29202af609063c network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x 37f53b9c053919c1ac1d38409d1ddf0710682d4bd8985db55c29202af609063c network default NAD default] I0327 23:05:01.230805 3881 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x 37f53b9c053919c1ac1d38409d1ddf0710682d4bd8985db55c29202af609063c network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x 37f53b9c053919c1ac1d38409d1ddf0710682d4bd8985db55c29202af609063c network default NAD default], result "{}", err I0327 23:05:01.276423 3881 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:05:01.276449 3881 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x I0327 23:05:01.277805 3881 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x, ips: 10.128.0.22 I0327 23:05:01.277834 3881 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:05:02.171147 3881 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:05:02.171172 3881 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:05:14.700464 3881 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:05:14.701091 3881 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:05:15.011439 3881 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:05:15.011464 3881 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:05:15.020000 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0327 23:05:15.020029 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0327 23:06:14.036584 3881 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 23:06:46.871637 3881 cni.go:403] [openshift-console/console-df959d948-tkwxt c0b01e47f8fbc69c5cb2339ce8a19eb37f34db4d2341ce71f852ffc70d32213f network default NAD default] DEL starting CNI request [openshift-console/console-df959d948-tkwxt c0b01e47f8fbc69c5cb2339ce8a19eb37f34db4d2341ce71f852ffc70d32213f network default NAD default] I0327 23:06:46.938490 3881 cni.go:424] [openshift-console/console-df959d948-tkwxt c0b01e47f8fbc69c5cb2339ce8a19eb37f34db4d2341ce71f852ffc70d32213f network default NAD default] DEL finished CNI request [openshift-console/console-df959d948-tkwxt c0b01e47f8fbc69c5cb2339ce8a19eb37f34db4d2341ce71f852ffc70d32213f network default NAD default], result "{}", err I0327 23:06:46.989789 3881 obj_retry.go:464] Detected object openshift-console/console-df959d948-tkwxt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:46.989825 3881 pods.go:176] Deleting pod: openshift-console/console-df959d948-tkwxt I0327 23:06:46.991018 3881 pods.go:236] Attempting to release IPs for pod: openshift-console/console-df959d948-tkwxt, ips: 10.128.0.30 I0327 23:06:46.991045 3881 obj_retry.go:464] Detected object openshift-console/console-df959d948-tkwxt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:46.998381 3881 base_network_controller_pods.go:487] [default/openshift-console/console-7ff69bd975-kq45h] creating logical port openshift-console_console-7ff69bd975-kq45h for pod on switch ip-10-0-2-103.ec2.internal I0327 23:06:46.998487 3881 kube.go:256] Updating pod openshift-console/console-7ff69bd975-kq45h I0327 23:06:47.017744 3881 pod.go:62] [openshift-console/console-7ff69bd975-kq45h] pod update took 19.282592ms I0327 23:06:47.017776 3881 base_network_controller_pods.go:937] [default/openshift-console/console-7ff69bd975-kq45h] addLogicalPort annotation time took 19.301786ms I0327 23:06:47.018823 3881 pods.go:271] [openshift-console/console-7ff69bd975-kq45h] addLogicalPort took 20.454951ms, libovsdb time 764.494µs I0327 23:06:47.356188 3881 cni.go:403] [openshift-console/console-7ff69bd975-kq45h 50e87b49041608221612e4a34b0df6651ccc272ec04a7e284f4a2ca5a309c664 network default NAD default] ADD starting CNI request [openshift-console/console-7ff69bd975-kq45h 50e87b49041608221612e4a34b0df6651ccc272ec04a7e284f4a2ca5a309c664 network default NAD default] I0327 23:06:47.364419 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-7ff69bd975-kq45h, hostIfaceName: 50e87b490416082, network: default, NAD default, SandboxID: "50e87b49041608221612e4a34b0df6651ccc272ec04a7e284f4a2ca5a309c664", PCI device ID: , UID: "103afd3b-a884-427f-b476-03c4e0200649", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0327 23:06:47.409571 3881 cni.go:424] [openshift-console/console-7ff69bd975-kq45h 50e87b49041608221612e4a34b0df6651ccc272ec04a7e284f4a2ca5a309c664 network default NAD default] ADD finished CNI request [openshift-console/console-7ff69bd975-kq45h 50e87b49041608221612e4a34b0df6651ccc272ec04a7e284f4a2ca5a309c664 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:47:25:26:f9:34\",\"name\":\"50e87b490416082\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/50cf8021-446c-4282-b01f-73fb21f81193\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:06:47.616423 3881 obj_retry.go:458] Detected object openshift-console/console-df959d948-tkwxt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:47.616448 3881 obj_retry.go:458] Detected object openshift-console/console-df959d948-tkwxt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:47.621968 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-df959d948-tkwxt I0327 23:06:47.621995 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-df959d948-tkwxt I0327 23:06:51.089028 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=64 I0327 23:06:57.648668 3881 obj_retry.go:464] Detected object openshift-console/console-df959d948-pn2c6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:57.649234 3881 obj_retry.go:464] Detected object openshift-console/console-df959d948-pn2c6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 23:06:57.677453 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 23:06:58.076025 3881 obj_retry.go:458] Detected object openshift-console/console-df959d948-pn2c6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:58.076052 3881 obj_retry.go:458] Detected object openshift-console/console-df959d948-pn2c6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:58.082042 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-df959d948-pn2c6 I0327 23:06:58.082062 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-df959d948-pn2c6 I0327 23:06:58.191321 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0327 23:07:01.176016 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0327 23:07:17.087492 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=140 I0327 23:07:58.087854 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0327 23:08:05.416282 3881 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xc5kp] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-xc5kp for pod on switch ip-10-0-2-103.ec2.internal I0327 23:08:05.416430 3881 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xc5kp I0327 23:08:05.429819 3881 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xc5kp] pod update took 13.423743ms I0327 23:08:05.429841 3881 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xc5kp] addLogicalPort annotation time took 13.451677ms I0327 23:08:05.431149 3881 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xc5kp] addLogicalPort took 14.888577ms, libovsdb time 957.273µs I0327 23:08:05.775247 3881 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xc5kp 566b79325b92b1e9aaceebcd0c6861959abafc20d1794fc8a0251cc3ae37ac5e network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xc5kp 566b79325b92b1e9aaceebcd0c6861959abafc20d1794fc8a0251cc3ae37ac5e network default NAD default] I0327 23:08:05.783716 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-xc5kp, hostIfaceName: 566b79325b92b1e, network: default, NAD default, SandboxID: "566b79325b92b1e9aaceebcd0c6861959abafc20d1794fc8a0251cc3ae37ac5e", PCI device ID: , UID: "ec51121c-a4cf-4902-873c-0bc8dabd931a", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0327 23:08:05.827245 3881 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xc5kp 566b79325b92b1e9aaceebcd0c6861959abafc20d1794fc8a0251cc3ae37ac5e network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xc5kp 566b79325b92b1e9aaceebcd0c6861959abafc20d1794fc8a0251cc3ae37ac5e network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:81:dc:bc:d0:ca\",\"name\":\"566b79325b92b1e\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9e0a8114-76e7-4e1c-90b4-70df17683a3d\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:08:08.990487 3881 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 23:08:11.833164 3881 egressfirewall.go:628] Syncing node "ip-10-0-0-182.ec2.internal" for egress firewall I0327 23:08:11.833172 3881 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-182.ec2.internal I0327 23:08:11.833205 3881 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-182.ec2.internal: 37.865µs I0327 23:08:17.180872 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 W0327 23:08:19.996446 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 23:08:20.044877 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0327 23:08:20.044938 3881 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-f9wf5 IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-f9wf5: no pod IPs found I0327 23:08:35.085993 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=15 I0327 23:08:46.166058 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0327 23:08:53.156012 3881 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0327 23:09:10.186726 3881 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0327 23:09:18.155643 3881 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0327 23:09:25.171684 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0327 23:10:00.211159 3881 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29577550-wpvq7 for pod on switch ip-10-0-2-103.ec2.internal I0327 23:10:00.211286 3881 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 I0327 23:10:00.238326 3881 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7] pod update took 27.066202ms I0327 23:10:00.238351 3881 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7] addLogicalPort annotation time took 27.094027ms I0327 23:10:00.239641 3881 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7] addLogicalPort took 28.501099ms, libovsdb time 932.302µs I0327 23:10:00.581916 3881 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 a462625fe7ad1f1269fce76f5b044848c5eeb208677f11e0a752512fe15cad9e network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 a462625fe7ad1f1269fce76f5b044848c5eeb208677f11e0a752512fe15cad9e network default NAD default] I0327 23:10:00.590200 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29577550-wpvq7, hostIfaceName: a462625fe7ad1f1, network: default, NAD default, SandboxID: "a462625fe7ad1f1269fce76f5b044848c5eeb208677f11e0a752512fe15cad9e", PCI device ID: , UID: "2483c42f-7f4c-4773-b1a3-727fa5e8c887", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0327 23:10:00.633870 3881 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 a462625fe7ad1f1269fce76f5b044848c5eeb208677f11e0a752512fe15cad9e network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 a462625fe7ad1f1269fce76f5b044848c5eeb208677f11e0a752512fe15cad9e network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:82:a1:35:3c:c0\",\"name\":\"a462625fe7ad1f1\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f4f20af5-36e2-483b-aad1-93f790d04118\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:10:03.507934 3881 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 a462625fe7ad1f1269fce76f5b044848c5eeb208677f11e0a752512fe15cad9e network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 a462625fe7ad1f1269fce76f5b044848c5eeb208677f11e0a752512fe15cad9e network default NAD default] I0327 23:10:03.581007 3881 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 a462625fe7ad1f1269fce76f5b044848c5eeb208677f11e0a752512fe15cad9e network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 a462625fe7ad1f1269fce76f5b044848c5eeb208677f11e0a752512fe15cad9e network default NAD default], result "{}", err I0327 23:10:03.630637 3881 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:03.630667 3881 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 I0327 23:10:03.631803 3881 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7, ips: 10.128.0.33 I0327 23:10:03.631829 3881 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:04.517224 3881 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:04.517255 3881 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-wpvq7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:42.192339 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0327 23:10:52.161446 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0327 23:10:59.559099 3881 namespace.go:98] [tsf] adding namespace I0327 23:10:59.560001 3881 namespace.go:102] [tsf] adding namespace took 881.579µs I0327 23:10:59.575847 3881 namespace.go:142] [tsf] updating namespace I0327 23:10:59.589932 3881 namespace.go:142] [tsf] updating namespace I0327 23:11:03.093887 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=720 I0327 23:11:03.438978 3881 namespace.go:98] [konflux-ui] adding namespace I0327 23:11:03.439916 3881 namespace.go:102] [konflux-ui] adding namespace took 912.152µs I0327 23:11:03.442875 3881 namespace.go:98] [rhtpa-operator] adding namespace I0327 23:11:03.442904 3881 namespace.go:98] [cert-manager-operator] adding namespace I0327 23:11:03.443613 3881 namespace.go:102] [rhtpa-operator] adding namespace took 721.109µs I0327 23:11:03.444210 3881 namespace.go:102] [cert-manager-operator] adding namespace took 1.28713ms I0327 23:11:03.444914 3881 namespace.go:98] [tsf-tpa] adding namespace I0327 23:11:03.444968 3881 namespace.go:98] [tsf-keycloak] adding namespace I0327 23:11:03.445087 3881 namespace.go:98] [konflux-operator] adding namespace I0327 23:11:03.445316 3881 namespace.go:98] [rhbk-operator] adding namespace I0327 23:11:03.445707 3881 namespace.go:102] [tsf-tpa] adding namespace took 773.445µs I0327 23:11:03.446344 3881 namespace.go:102] [tsf-keycloak] adding namespace took 1.361982ms I0327 23:11:03.446941 3881 namespace.go:102] [konflux-operator] adding namespace took 1.843722ms I0327 23:11:03.446969 3881 namespace.go:98] [tsf-tas] adding namespace I0327 23:11:03.447518 3881 namespace.go:102] [tsf-tas] adding namespace took 539.73µs I0327 23:11:03.448055 3881 namespace.go:102] [rhbk-operator] adding namespace took 2.723047ms I0327 23:11:03.459700 3881 namespace.go:142] [konflux-ui] updating namespace I0327 23:11:03.483865 3881 namespace.go:142] [konflux-ui] updating namespace I0327 23:11:03.491733 3881 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:11:03.491787 3881 namespace.go:142] [tsf-tpa] updating namespace I0327 23:11:03.496375 3881 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:11:03.496469 3881 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:11:03.509562 3881 namespace.go:142] [rhbk-operator] updating namespace I0327 23:11:03.512330 3881 namespace.go:142] [tsf-tas] updating namespace I0327 23:11:03.512397 3881 namespace.go:142] [konflux-operator] updating namespace I0327 23:11:03.512519 3881 namespace.go:142] [konflux-ui] updating namespace I0327 23:11:03.556218 3881 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:11:03.571580 3881 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:11:03.586920 3881 namespace.go:142] [tsf-tpa] updating namespace I0327 23:11:03.596665 3881 namespace.go:142] [tsf-tpa] updating namespace I0327 23:11:03.618597 3881 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:11:03.635641 3881 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:11:03.644243 3881 namespace.go:142] [konflux-operator] updating namespace I0327 23:11:03.652645 3881 namespace.go:142] [konflux-operator] updating namespace I0327 23:11:03.669575 3881 namespace.go:142] [tsf-tas] updating namespace I0327 23:11:03.680900 3881 namespace.go:142] [tsf-tas] updating namespace I0327 23:11:03.693496 3881 namespace.go:142] [rhbk-operator] updating namespace I0327 23:11:03.705838 3881 namespace.go:142] [rhbk-operator] updating namespace I0327 23:11:03.721544 3881 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:11:03.731265 3881 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:11:04.740908 3881 namespace.go:142] [tsf-tpa] updating namespace I0327 23:11:04.752083 3881 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:11:10.977404 3881 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf459k9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:10.977987 3881 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf459k9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:10.978009 3881 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf459k9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:11:10.978017 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:11:10.978028 3881 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf459k9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:11:10.978035 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:11:10.997592 3881 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d17zh2w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:10.998012 3881 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d17zh2w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:10.998029 3881 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d17zh2w of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:11:10.998035 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:11:10.998045 3881 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d17zh2w of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:11:10.998050 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:11:12.002894 3881 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf459k9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:12.002914 3881 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf459k9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:12.002929 3881 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf459k9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:12.002941 3881 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf459k9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:12.026902 3881 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d17zh2w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:12.026923 3881 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d17zh2w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:12.026935 3881 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d17zh2w of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:12.026949 3881 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d17zh2w of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:17.029546 3881 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafbld4m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:17.030074 3881 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafbld4m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:17.030098 3881 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafbld4m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:11:17.030107 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:11:17.030121 3881 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafbld4m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:11:17.030128 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:11:17.054596 3881 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8cflz4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:17.054994 3881 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8cflz4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:17.055014 3881 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8cflz4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:11:17.055022 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:11:17.055031 3881 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8cflz4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:11:17.055035 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:11:17.086530 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0327 23:11:18.082824 3881 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafbld4m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:18.082849 3881 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafbld4m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:18.082861 3881 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafbld4m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:18.082874 3881 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafbld4m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:18.102835 3881 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8cflz4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:18.102854 3881 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8cflz4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:18.102865 3881 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8cflz4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:18.102875 3881 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8cflz4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:19.048148 3881 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7sqlzd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:19.048719 3881 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7sqlzd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:19.048746 3881 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7sqlzd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:11:19.048756 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:11:19.048788 3881 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7sqlzd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:11:19.048795 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:11:19.067377 3881 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5zrmzw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:19.067812 3881 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5zrmzw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:19.067837 3881 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5zrmzw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:11:19.067846 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:11:19.067859 3881 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5zrmzw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:11:19.067863 3881 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:11:19.673593 3881 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-8468f77bc5-br5kt] creating logical port rhbk-operator_rhbk-operator-8468f77bc5-br5kt for pod on switch ip-10-0-2-103.ec2.internal I0327 23:11:19.673730 3881 kube.go:256] Updating pod rhbk-operator/rhbk-operator-8468f77bc5-br5kt I0327 23:11:19.692519 3881 pod.go:62] [rhbk-operator/rhbk-operator-8468f77bc5-br5kt] pod update took 18.817995ms I0327 23:11:19.692537 3881 base_network_controller_pods.go:937] [default/rhbk-operator/rhbk-operator-8468f77bc5-br5kt] addLogicalPort annotation time took 18.840015ms I0327 23:11:19.693685 3881 pods.go:271] [rhbk-operator/rhbk-operator-8468f77bc5-br5kt] addLogicalPort took 20.104399ms, libovsdb time 788.683µs I0327 23:11:20.044460 3881 cni.go:403] [rhbk-operator/rhbk-operator-8468f77bc5-br5kt c33ade5b0e6a1bfa6a80a726641cdd5ef3ce9b30ae32fc84fa4ea5d1e0f21b96 network default NAD default] ADD starting CNI request [rhbk-operator/rhbk-operator-8468f77bc5-br5kt c33ade5b0e6a1bfa6a80a726641cdd5ef3ce9b30ae32fc84fa4ea5d1e0f21b96 network default NAD default] I0327 23:11:20.052880 3881 helper_linux.go:450] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-8468f77bc5-br5kt, hostIfaceName: c33ade5b0e6a1bf, network: default, NAD default, SandboxID: "c33ade5b0e6a1bfa6a80a726641cdd5ef3ce9b30ae32fc84fa4ea5d1e0f21b96", PCI device ID: , UID: "2b6a6e66-5423-4f15-a8f5-dc02322bbf0a", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0327 23:11:20.081956 3881 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7sqlzd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:20.081977 3881 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7sqlzd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:20.081991 3881 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7sqlzd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:20.082007 3881 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7sqlzd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:20.096485 3881 cni.go:424] [rhbk-operator/rhbk-operator-8468f77bc5-br5kt c33ade5b0e6a1bfa6a80a726641cdd5ef3ce9b30ae32fc84fa4ea5d1e0f21b96 network default NAD default] ADD finished CNI request [rhbk-operator/rhbk-operator-8468f77bc5-br5kt c33ade5b0e6a1bfa6a80a726641cdd5ef3ce9b30ae32fc84fa4ea5d1e0f21b96 network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:85:94:a4:2f:3a\",\"name\":\"c33ade5b0e6a1bf\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/53419f32-3b09-4047-b230-3214308d947d\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:11:20.098913 3881 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5zrmzw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:20.098934 3881 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5zrmzw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:20.098950 3881 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5zrmzw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:20.098965 3881 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5zrmzw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:29.206923 3881 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gfml9] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-58df4cf46c-gfml9 for pod on switch ip-10-0-2-103.ec2.internal I0327 23:11:29.207059 3881 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gfml9 I0327 23:11:29.230159 3881 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gfml9] pod update took 23.13439ms I0327 23:11:29.230181 3881 base_network_controller_pods.go:937] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gfml9] addLogicalPort annotation time took 23.158216ms I0327 23:11:29.231441 3881 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gfml9] addLogicalPort took 24.53238ms, libovsdb time 857.82µs W0327 23:11:29.453720 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 23:11:29.492940 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 23:11:29.501844 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 23:11:29.514804 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 23:11:29.522433 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0327 23:11:29.563366 3881 cni.go:403] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gfml9 e22630872db94b2135be4f55c7794a3c6c5cbe61dc3452305c1eed5e4ac8dced network default NAD default] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gfml9 e22630872db94b2135be4f55c7794a3c6c5cbe61dc3452305c1eed5e4ac8dced network default NAD default] I0327 23:11:29.572143 3881 helper_linux.go:450] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-58df4cf46c-gfml9, hostIfaceName: e22630872db94b2, network: default, NAD default, SandboxID: "e22630872db94b2135be4f55c7794a3c6c5cbe61dc3452305c1eed5e4ac8dced", PCI device ID: , UID: "98f9e591-074e-47fc-86e4-49a150bff1f3", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0327 23:11:29.621965 3881 cni.go:424] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gfml9 e22630872db94b2135be4f55c7794a3c6c5cbe61dc3452305c1eed5e4ac8dced network default NAD default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gfml9 e22630872db94b2135be4f55c7794a3c6c5cbe61dc3452305c1eed5e4ac8dced network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:a5:10:fe:b6:7a\",\"name\":\"e22630872db94b2\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0568ee0d-43be-4f02-bcc2-995317f51bf6\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:11:30.718021 3881 namespace.go:142] [openshift-operators] updating namespace I0327 23:11:30.743008 3881 namespace.go:142] [openshift-operators] updating namespace I0327 23:11:34.704669 3881 namespace.go:98] [cert-manager] adding namespace I0327 23:11:34.705496 3881 namespace.go:102] [cert-manager] adding namespace took 803.212µs I0327 23:11:34.726516 3881 namespace.go:142] [cert-manager] updating namespace I0327 23:11:34.736973 3881 namespace.go:142] [cert-manager] updating namespace I0327 23:11:36.650899 3881 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9xwpn] creating logical port openshift-operators_rhtas-operator-controller-manager-6987b7f98c-9xwpn for pod on switch ip-10-0-2-103.ec2.internal I0327 23:11:36.651020 3881 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9xwpn I0327 23:11:36.684023 3881 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9xwpn] pod update took 33.031722ms I0327 23:11:36.684043 3881 base_network_controller_pods.go:937] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9xwpn] addLogicalPort annotation time took 33.0564ms I0327 23:11:36.685332 3881 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9xwpn] addLogicalPort took 34.450493ms, libovsdb time 953.561µs I0327 23:11:37.006440 3881 cni.go:403] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9xwpn d14e248f4376107e06829432b75c34ccaa0e9081da36af325dfbc79decdd6dd6 network default NAD default] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9xwpn d14e248f4376107e06829432b75c34ccaa0e9081da36af325dfbc79decdd6dd6 network default NAD default] I0327 23:11:37.015057 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-6987b7f98c-9xwpn, hostIfaceName: d14e248f4376107, network: default, NAD default, SandboxID: "d14e248f4376107e06829432b75c34ccaa0e9081da36af325dfbc79decdd6dd6", PCI device ID: , UID: "262a90f1-7a29-447a-b397-7a8527c6b19e", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0327 23:11:37.059072 3881 cni.go:424] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9xwpn d14e248f4376107e06829432b75c34ccaa0e9081da36af325dfbc79decdd6dd6 network default NAD default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9xwpn d14e248f4376107e06829432b75c34ccaa0e9081da36af325dfbc79decdd6dd6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:e4:12:2e:fb:00\",\"name\":\"d14e248f4376107\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7c1fd3d2-fc96-4655-ab1e-787840acb6df\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:11:39.087265 3881 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5b4bf68748-qqm6j] creating logical port cert-manager_cert-manager-cainjector-5b4bf68748-qqm6j for pod on switch ip-10-0-2-103.ec2.internal I0327 23:11:39.087367 3881 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5b4bf68748-qqm6j I0327 23:11:39.143669 3881 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5b4bf68748-qqm6j I0327 23:11:39.168048 3881 pod.go:62] [cert-manager/cert-manager-cainjector-5b4bf68748-qqm6j] pod update took 80.701038ms I0327 23:11:39.168068 3881 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-cainjector-5b4bf68748-qqm6j] addLogicalPort annotation time took 80.728194ms I0327 23:11:39.169390 3881 pods.go:271] [cert-manager/cert-manager-cainjector-5b4bf68748-qqm6j] addLogicalPort took 82.148257ms, libovsdb time 953.694µs I0327 23:11:39.444976 3881 cni.go:403] [cert-manager/cert-manager-cainjector-5b4bf68748-qqm6j d0942c272437be7f4ad0a60728a1c9466871b7f85c7faebc55c1e3a2734373df network default NAD default] ADD starting CNI request [cert-manager/cert-manager-cainjector-5b4bf68748-qqm6j d0942c272437be7f4ad0a60728a1c9466871b7f85c7faebc55c1e3a2734373df network default NAD default] I0327 23:11:39.454602 3881 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5b4bf68748-qqm6j, hostIfaceName: d0942c272437be7, network: default, NAD default, SandboxID: "d0942c272437be7f4ad0a60728a1c9466871b7f85c7faebc55c1e3a2734373df", PCI device ID: , UID: "057bbe7d-3d74-46b0-8fba-345813b4f296", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0327 23:11:39.499349 3881 cni.go:424] [cert-manager/cert-manager-cainjector-5b4bf68748-qqm6j d0942c272437be7f4ad0a60728a1c9466871b7f85c7faebc55c1e3a2734373df network default NAD default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5b4bf68748-qqm6j d0942c272437be7f4ad0a60728a1c9466871b7f85c7faebc55c1e3a2734373df network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:52:bb:aa:1f:d8\",\"name\":\"d0942c272437be7\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5da61f11-3337-4612-b261-02efd12d2379\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:11:39.708744 3881 namespace.go:98] [trusted-artifact-signer] adding namespace I0327 23:11:39.709586 3881 namespace.go:102] [trusted-artifact-signer] adding namespace took 824.148µs I0327 23:11:39.746877 3881 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:11:39.758712 3881 namespace.go:142] [trusted-artifact-signer] updating namespace W0327 23:11:39.984733 3881 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 23:11:42.547389 3881 namespace.go:142] [openshift-operators] updating namespace W0327 23:11:42.565248 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0327 23:11:42.565269 3881 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0327 23:11:43.419647 3881 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6684678d86-gtvg2] creating logical port openshift-operators_openshift-pipelines-operator-6684678d86-gtvg2 for pod on switch ip-10-0-2-103.ec2.internal I0327 23:11:43.419757 3881 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6684678d86-gtvg2 I0327 23:11:43.462675 3881 pod.go:62] [openshift-operators/openshift-pipelines-operator-6684678d86-gtvg2] pod update took 42.941618ms I0327 23:11:43.462699 3881 base_network_controller_pods.go:937] [default/openshift-operators/openshift-pipelines-operator-6684678d86-gtvg2] addLogicalPort annotation time took 42.971563ms I0327 23:11:43.464024 3881 pods.go:271] [openshift-operators/openshift-pipelines-operator-6684678d86-gtvg2] addLogicalPort took 44.395436ms, libovsdb time 902.399µs I0327 23:11:43.778297 3881 cni.go:403] [openshift-operators/openshift-pipelines-operator-6684678d86-gtvg2 9318660cb90807a8c021e9a3c3d9f03cace1347761f6e66135105c8e5896029d network default NAD default] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-gtvg2 9318660cb90807a8c021e9a3c3d9f03cace1347761f6e66135105c8e5896029d network default NAD default] I0327 23:11:43.788008 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6684678d86-gtvg2, hostIfaceName: 9318660cb90807a, network: default, NAD default, SandboxID: "9318660cb90807a8c021e9a3c3d9f03cace1347761f6e66135105c8e5896029d", PCI device ID: , UID: "250dc781-6120-4efb-8eab-e7e4086504a3", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0327 23:11:43.834065 3881 cni.go:424] [openshift-operators/openshift-pipelines-operator-6684678d86-gtvg2 9318660cb90807a8c021e9a3c3d9f03cace1347761f6e66135105c8e5896029d network default NAD default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-gtvg2 9318660cb90807a8c021e9a3c3d9f03cace1347761f6e66135105c8e5896029d network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:6c:05:07:4b:c6\",\"name\":\"9318660cb90807a\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e55ed7fb-6321-4a2b-9ecc-2b31880813c2\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:11:48.141158 3881 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8tkfd] creating logical port tsf-tpa_rhtpa-operator-controller-manager-746944f6d7-8tkfd for pod on switch ip-10-0-2-103.ec2.internal I0327 23:11:48.141287 3881 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8tkfd I0327 23:11:48.205808 3881 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8tkfd I0327 23:11:48.240693 3881 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8tkfd] pod update took 99.441497ms I0327 23:11:48.240713 3881 base_network_controller_pods.go:937] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8tkfd] addLogicalPort annotation time took 99.46633ms I0327 23:11:48.241926 3881 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8tkfd] addLogicalPort took 100.781479ms, libovsdb time 788.502µs I0327 23:11:48.488164 3881 cni.go:403] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8tkfd d28fa5cb756dd371ffbee1ebe67352475025b24c2c01bfbe6c6eea2e44a8d35e network default NAD default] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8tkfd d28fa5cb756dd371ffbee1ebe67352475025b24c2c01bfbe6c6eea2e44a8d35e network default NAD default] I0327 23:11:48.497455 3881 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-746944f6d7-8tkfd, hostIfaceName: d28fa5cb756dd37, network: default, NAD default, SandboxID: "d28fa5cb756dd371ffbee1ebe67352475025b24c2c01bfbe6c6eea2e44a8d35e", PCI device ID: , UID: "d801a36e-fb9c-4352-8a26-572982faa30f", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0327 23:11:48.542882 3881 cni.go:424] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8tkfd d28fa5cb756dd371ffbee1ebe67352475025b24c2c01bfbe6c6eea2e44a8d35e network default NAD default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8tkfd d28fa5cb756dd371ffbee1ebe67352475025b24c2c01bfbe6c6eea2e44a8d35e network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:f9:83:38:1f:d3\",\"name\":\"d28fa5cb756dd37\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/81887573-2837-4068-b2f9-aa7b734cc06c\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:11:50.980416 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0327 23:11:51.023091 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 23:11:58.821541 3881 namespace.go:98] [openshift-pipelines] adding namespace I0327 23:11:58.822487 3881 namespace.go:102] [openshift-pipelines] adding namespace took 926.538µs I0327 23:11:58.851950 3881 namespace.go:142] [openshift-pipelines] updating namespace I0327 23:11:59.532693 3881 namespace.go:142] [cert-manager] updating namespace I0327 23:11:59.734412 3881 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:12:00.133828 3881 namespace.go:142] [konflux-operator] updating namespace I0327 23:12:00.334959 3881 namespace.go:142] [konflux-ui] updating namespace I0327 23:12:00.535623 3881 namespace.go:142] [open-cluster-management-2pabkugc0r4kj7d3r2omus9hg3vasti0] updating namespace I0327 23:12:00.938310 3881 namespace.go:142] [rhbk-operator] updating namespace I0327 23:12:01.134587 3881 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:12:01.336803 3881 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:12:01.536191 3881 namespace.go:142] [tsf] updating namespace I0327 23:12:01.735977 3881 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:12:01.944327 3881 namespace.go:142] [tsf-tas] updating namespace W0327 23:12:02.029062 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 23:12:02.139336 3881 namespace.go:142] [tsf-tpa] updating namespace I0327 23:12:02.269787 3881 namespace.go:142] [cert-manager] updating namespace I0327 23:12:02.279529 3881 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:12:02.288074 3881 namespace.go:142] [default] updating namespace I0327 23:12:02.298363 3881 namespace.go:142] [konflux-operator] updating namespace I0327 23:12:02.306821 3881 namespace.go:142] [konflux-ui] updating namespace I0327 23:12:02.318332 3881 namespace.go:142] [open-cluster-management-2pabkugc0r4kj7d3r2omus9hg3vasti0] updating namespace I0327 23:12:02.327185 3881 namespace.go:142] [openshift] updating namespace I0327 23:12:02.341455 3881 namespace.go:142] [rhbk-operator] updating namespace I0327 23:12:02.351707 3881 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:12:02.360864 3881 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:12:02.372791 3881 namespace.go:142] [tsf] updating namespace I0327 23:12:02.381629 3881 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:12:02.428285 3881 namespace.go:142] [tsf-tas] updating namespace I0327 23:12:02.441979 3881 namespace.go:142] [tsf-tpa] updating namespace I0327 23:12:02.525885 3881 namespace.go:142] [cert-manager] updating namespace I0327 23:12:02.622846 3881 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:12:02.721241 3881 namespace.go:142] [default] updating namespace I0327 23:12:02.825345 3881 namespace.go:142] [konflux-operator] updating namespace I0327 23:12:02.925743 3881 namespace.go:142] [konflux-ui] updating namespace I0327 23:12:03.024035 3881 namespace.go:142] [open-cluster-management-2pabkugc0r4kj7d3r2omus9hg3vasti0] updating namespace I0327 23:12:03.127825 3881 namespace.go:142] [openshift] updating namespace I0327 23:12:03.222166 3881 namespace.go:142] [rhbk-operator] updating namespace I0327 23:12:03.324998 3881 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:12:03.424416 3881 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:12:03.521688 3881 namespace.go:142] [tsf] updating namespace I0327 23:12:03.625952 3881 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:12:03.721819 3881 namespace.go:142] [tsf-tas] updating namespace I0327 23:12:03.823656 3881 namespace.go:142] [tsf-tpa] updating namespace I0327 23:12:04.019624 3881 namespace.go:142] [openshift-pipelines] updating namespace I0327 23:12:06.272642 3881 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:12:06.273176 3881 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:12:06.296671 3881 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:12:06.296691 3881 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:12:06.302514 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0327 23:12:06.302538 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0327 23:12:11.444998 3881 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-hkjqx] creating logical port openshift-pipelines_tekton-pipelines-controller-84d6d58f5c-hkjqx for pod on switch ip-10-0-2-103.ec2.internal I0327 23:12:11.445105 3881 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-hkjqx I0327 23:12:11.460660 3881 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-hkjqx] pod update took 15.583932ms I0327 23:12:11.460684 3881 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-hkjqx] addLogicalPort annotation time took 15.613139ms I0327 23:12:11.461967 3881 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-hkjqx] addLogicalPort took 16.987161ms, libovsdb time 878.759µs I0327 23:12:11.818238 3881 cni.go:403] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-hkjqx e4e4f8104268845f6cfc039a9ddd9665a4d7354010948bf88d23d450b0ca1d32 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-hkjqx e4e4f8104268845f6cfc039a9ddd9665a4d7354010948bf88d23d450b0ca1d32 network default NAD default] I0327 23:12:11.827054 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-84d6d58f5c-hkjqx, hostIfaceName: e4e4f8104268845, network: default, NAD default, SandboxID: "e4e4f8104268845f6cfc039a9ddd9665a4d7354010948bf88d23d450b0ca1d32", PCI device ID: , UID: "d1a1eb93-1b30-4d0d-9903-46a621e0a16e", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0327 23:12:11.873947 3881 cni.go:424] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-hkjqx e4e4f8104268845f6cfc039a9ddd9665a4d7354010948bf88d23d450b0ca1d32 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-hkjqx e4e4f8104268845f6cfc039a9ddd9665a4d7354010948bf88d23d450b0ca1d32 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:7e:34:da:e5:a0\",\"name\":\"e4e4f8104268845\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a93dc1af-b89a-4dda-8d6b-248296576848\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:12:16.156519 3881 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 23:12:16.156540 3881 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 23:12:16.156546 3881 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 23:12:16.695076 3881 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:12:16.695099 3881 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:12:16.695106 3881 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:12:16.695110 3881 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:12:16.730967 3881 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 23:12:16.730983 3881 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 23:12:16.730988 3881 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 23:12:18.230341 3881 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0327 23:12:18.260518 3881 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0327 23:12:27.165565 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 23:12:27.165586 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 23:12:27.165594 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0327 23:12:34.994420 3881 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-28nkm] creating logical port openshift-pipelines_tekton-triggers-webhook-55d94ff6c6-28nkm for pod on switch ip-10-0-2-103.ec2.internal I0327 23:12:34.994523 3881 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-28nkm I0327 23:12:35.011395 3881 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-28nkm] pod update took 16.890858ms I0327 23:12:35.011417 3881 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-28nkm] addLogicalPort annotation time took 16.915386ms I0327 23:12:35.012592 3881 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-28nkm] addLogicalPort took 18.185602ms, libovsdb time 780.494µs I0327 23:12:35.093568 3881 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-6blbj] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-8bb674894-6blbj for pod on switch ip-10-0-2-103.ec2.internal I0327 23:12:35.093657 3881 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-6blbj I0327 23:12:35.108703 3881 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-6blbj] pod update took 15.075312ms I0327 23:12:35.108722 3881 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-6blbj] addLogicalPort annotation time took 15.0997ms I0327 23:12:35.109986 3881 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-6blbj] addLogicalPort took 16.429512ms, libovsdb time 858.264µs I0327 23:12:35.357514 3881 cni.go:403] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-28nkm ebdd345f84a78291661bb35afa961e7393ef67700c54a3d297ce7bdb4f2ae5dd network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-28nkm ebdd345f84a78291661bb35afa961e7393ef67700c54a3d297ce7bdb4f2ae5dd network default NAD default] I0327 23:12:35.366723 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-55d94ff6c6-28nkm, hostIfaceName: ebdd345f84a7829, network: default, NAD default, SandboxID: "ebdd345f84a78291661bb35afa961e7393ef67700c54a3d297ce7bdb4f2ae5dd", PCI device ID: , UID: "f1d93525-98e4-4b27-b5e9-d2fa3e8f312e", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0327 23:12:35.411584 3881 cni.go:424] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-28nkm ebdd345f84a78291661bb35afa961e7393ef67700c54a3d297ce7bdb4f2ae5dd network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-28nkm ebdd345f84a78291661bb35afa961e7393ef67700c54a3d297ce7bdb4f2ae5dd network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:c6:8d:63:b3:fb\",\"name\":\"ebdd345f84a7829\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7bd70ecd-77f8-467e-b173-168d30f293ea\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:12:35.453565 3881 cni.go:403] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-6blbj cc6250c9c1e36da7b247f4301d88ea3f4740be92f41bb6e56523b4e6bad76ca4 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-6blbj cc6250c9c1e36da7b247f4301d88ea3f4740be92f41bb6e56523b4e6bad76ca4 network default NAD default] I0327 23:12:35.462287 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-8bb674894-6blbj, hostIfaceName: cc6250c9c1e36da, network: default, NAD default, SandboxID: "cc6250c9c1e36da7b247f4301d88ea3f4740be92f41bb6e56523b4e6bad76ca4", PCI device ID: , UID: "2e3d9201-0ec8-4261-b63b-2887b98eecf6", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0327 23:12:35.506874 3881 cni.go:424] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-6blbj cc6250c9c1e36da7b247f4301d88ea3f4740be92f41bb6e56523b4e6bad76ca4 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-6blbj cc6250c9c1e36da7b247f4301d88ea3f4740be92f41bb6e56523b4e6bad76ca4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:13:f4:7a:ce:33\",\"name\":\"cc6250c9c1e36da\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/55c005ce-05f9-43f8-be96-1bda2aa75359\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:12:37.090896 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=55 W0327 23:12:39.272237 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0327 23:12:41.484595 3881 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:12:41.485184 3881 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:12:41.503408 3881 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:12:41.503429 3881 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:12:41.510249 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0327 23:12:41.510268 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0327 23:12:42.638437 3881 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-103.ec2.internal I0327 23:12:42.638644 3881 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0327 23:12:42.653115 3881 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 14.547495ms I0327 23:12:42.653171 3881 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 14.586998ms I0327 23:12:42.654851 3881 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 16.438211ms, libovsdb time 1.052703ms I0327 23:12:43.001715 3881 cni.go:403] [tsf-keycloak/keycloak-0 17a2c1f287a288ff3ebcab5c4d4daad0d747439a08cc539a671488b0ab0d8e10 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 17a2c1f287a288ff3ebcab5c4d4daad0d747439a08cc539a671488b0ab0d8e10 network default NAD default] I0327 23:12:43.010153 3881 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 17a2c1f287a288f, network: default, NAD default, SandboxID: "17a2c1f287a288ff3ebcab5c4d4daad0d747439a08cc539a671488b0ab0d8e10", PCI device ID: , UID: "c05c3e12-a71c-4c97-9e34-9a3c587d87f4", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0327 23:12:43.056130 3881 cni.go:424] [tsf-keycloak/keycloak-0 17a2c1f287a288ff3ebcab5c4d4daad0d747439a08cc539a671488b0ab0d8e10 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 17a2c1f287a288ff3ebcab5c4d4daad0d747439a08cc539a671488b0ab0d8e10 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:4d:70:24:94:e6\",\"name\":\"17a2c1f287a288f\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f0884f9e-125a-4573-9d0a-473927262f03\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:12:49.329667 3881 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:12:49.329687 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:12:49.336391 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 23:12:50.289315 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0327 23:12:56.311932 3881 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0327 23:13:08.268198 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:13:08.268217 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:13:08.271723 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 23:13:12.780321 3881 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-ghn4q] creating logical port tsf-keycloak_tsf-iam-ghn4q for pod on switch ip-10-0-2-103.ec2.internal I0327 23:13:12.780419 3881 kube.go:256] Updating pod tsf-keycloak/tsf-iam-ghn4q I0327 23:13:12.794633 3881 pod.go:62] [tsf-keycloak/tsf-iam-ghn4q] pod update took 14.243658ms I0327 23:13:12.794660 3881 base_network_controller_pods.go:937] [default/tsf-keycloak/tsf-iam-ghn4q] addLogicalPort annotation time took 14.27533ms I0327 23:13:12.796481 3881 pods.go:271] [tsf-keycloak/tsf-iam-ghn4q] addLogicalPort took 16.171994ms, libovsdb time 1.184785ms I0327 23:13:13.148494 3881 cni.go:403] [tsf-keycloak/tsf-iam-ghn4q 9a8c4d68e7411a92ab6e94db9195ad948bf1ba6a5e2abe76c59d4bd4bb80bd59 network default NAD default] ADD starting CNI request [tsf-keycloak/tsf-iam-ghn4q 9a8c4d68e7411a92ab6e94db9195ad948bf1ba6a5e2abe76c59d4bd4bb80bd59 network default NAD default] I0327 23:13:13.156982 3881 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-ghn4q, hostIfaceName: 9a8c4d68e7411a9, network: default, NAD default, SandboxID: "9a8c4d68e7411a92ab6e94db9195ad948bf1ba6a5e2abe76c59d4bd4bb80bd59", PCI device ID: , UID: "c5ed2a85-7468-4f20-ac29-4c9985005f36", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0327 23:13:13.202834 3881 cni.go:424] [tsf-keycloak/tsf-iam-ghn4q 9a8c4d68e7411a92ab6e94db9195ad948bf1ba6a5e2abe76c59d4bd4bb80bd59 network default NAD default] ADD finished CNI request [tsf-keycloak/tsf-iam-ghn4q 9a8c4d68e7411a92ab6e94db9195ad948bf1ba6a5e2abe76c59d4bd4bb80bd59 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:23:84:54:7e:99\",\"name\":\"9a8c4d68e7411a9\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e887a0cd-daf5-4c71-81f7-39d3d2ea295f\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:13:16.419387 3881 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:16.420011 3881 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:16.463263 3881 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:16.463286 3881 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:16.468549 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 23:13:16.468575 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam W0327 23:13:18.983313 3881 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 23:13:18.983434 3881 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 23:13:18.983473 3881 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0327 23:13:19.194548 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0327 23:13:19.967008 3881 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-plgzg-4lhgq] creating logical port tsf-tas_rekor-createtree-job-plgzg-4lhgq for pod on switch ip-10-0-2-103.ec2.internal I0327 23:13:19.967127 3881 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-plgzg-4lhgq I0327 23:13:19.995728 3881 pod.go:62] [tsf-tas/rekor-createtree-job-plgzg-4lhgq] pod update took 28.633264ms I0327 23:13:19.995776 3881 base_network_controller_pods.go:937] [default/tsf-tas/rekor-createtree-job-plgzg-4lhgq] addLogicalPort annotation time took 28.671081ms I0327 23:13:19.997406 3881 pods.go:271] [tsf-tas/rekor-createtree-job-plgzg-4lhgq] addLogicalPort took 30.420674ms, libovsdb time 1.073297ms I0327 23:13:20.368442 3881 cni.go:403] [tsf-tas/rekor-createtree-job-plgzg-4lhgq 6808a675d69ad53b4db9c518ac33f008663718e6bb74ebbd9a7f02e12d6890ed network default NAD default] ADD starting CNI request [tsf-tas/rekor-createtree-job-plgzg-4lhgq 6808a675d69ad53b4db9c518ac33f008663718e6bb74ebbd9a7f02e12d6890ed network default NAD default] I0327 23:13:20.377702 3881 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-plgzg-4lhgq, hostIfaceName: 6808a675d69ad53, network: default, NAD default, SandboxID: "6808a675d69ad53b4db9c518ac33f008663718e6bb74ebbd9a7f02e12d6890ed", PCI device ID: , UID: "da222246-21f5-4002-9e8b-7a8a0ef54bbd", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0327 23:13:20.423706 3881 cni.go:424] [tsf-tas/rekor-createtree-job-plgzg-4lhgq 6808a675d69ad53b4db9c518ac33f008663718e6bb74ebbd9a7f02e12d6890ed network default NAD default] ADD finished CNI request [tsf-tas/rekor-createtree-job-plgzg-4lhgq 6808a675d69ad53b4db9c518ac33f008663718e6bb74ebbd9a7f02e12d6890ed network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:05:0d:ce:55:06\",\"name\":\"6808a675d69ad53\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/18a581fb-45d4-409d-8db5-134df3d70314\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:13:26.643113 3881 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0327 23:13:29.035416 3881 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0327 23:13:29.035434 3881 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0327 23:13:30.651758 3881 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0327 23:13:31.571259 3881 cni.go:403] [tsf-keycloak/tsf-iam-ghn4q 9a8c4d68e7411a92ab6e94db9195ad948bf1ba6a5e2abe76c59d4bd4bb80bd59 network default NAD default] DEL starting CNI request [tsf-keycloak/tsf-iam-ghn4q 9a8c4d68e7411a92ab6e94db9195ad948bf1ba6a5e2abe76c59d4bd4bb80bd59 network default NAD default] I0327 23:13:31.650961 3881 cni.go:424] [tsf-keycloak/tsf-iam-ghn4q 9a8c4d68e7411a92ab6e94db9195ad948bf1ba6a5e2abe76c59d4bd4bb80bd59 network default NAD default] DEL finished CNI request [tsf-keycloak/tsf-iam-ghn4q 9a8c4d68e7411a92ab6e94db9195ad948bf1ba6a5e2abe76c59d4bd4bb80bd59 network default NAD default], result "{}", err I0327 23:13:31.705373 3881 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-ghn4q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:31.705404 3881 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-ghn4q I0327 23:13:31.706742 3881 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-ghn4q, ips: 10.128.0.44 I0327 23:13:31.706779 3881 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-ghn4q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 23:13:32.054526 3881 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0327 23:13:32.738867 3881 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-ghn4q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:32.738894 3881 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-ghn4q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:32.975614 3881 cni.go:403] [tsf-keycloak/keycloak-0 17a2c1f287a288ff3ebcab5c4d4daad0d747439a08cc539a671488b0ab0d8e10 network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 17a2c1f287a288ff3ebcab5c4d4daad0d747439a08cc539a671488b0ab0d8e10 network default NAD default] I0327 23:13:33.053083 3881 cni.go:424] [tsf-keycloak/keycloak-0 17a2c1f287a288ff3ebcab5c4d4daad0d747439a08cc539a671488b0ab0d8e10 network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 17a2c1f287a288ff3ebcab5c4d4daad0d747439a08cc539a671488b0ab0d8e10 network default NAD default], result "{}", err I0327 23:13:33.111122 3881 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:33.111153 3881 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0327 23:13:33.112535 3881 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.128.0.43 I0327 23:13:33.112569 3881 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:33.573307 3881 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:33.573390 3881 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:33.576721 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 23:13:33.576749 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0327 23:13:33.607910 3881 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-103.ec2.internal I0327 23:13:33.608032 3881 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0327 23:13:33.623643 3881 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 15.648916ms I0327 23:13:33.623670 3881 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 15.681477ms I0327 23:13:33.624999 3881 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 17.107878ms, libovsdb time 917.297µs I0327 23:13:33.966858 3881 cni.go:403] [tsf-keycloak/keycloak-0 8f116f6dd03fcfd133a5acffc3aa45438ab4cfe18fc20680e840ddbddeb500e0 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 8f116f6dd03fcfd133a5acffc3aa45438ab4cfe18fc20680e840ddbddeb500e0 network default NAD default] I0327 23:13:33.975689 3881 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 8f116f6dd03fcfd, network: default, NAD default, SandboxID: "8f116f6dd03fcfd133a5acffc3aa45438ab4cfe18fc20680e840ddbddeb500e0", PCI device ID: , UID: "a141fa1d-27e5-4e10-b550-3bd03f9af907", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0327 23:13:34.022413 3881 cni.go:424] [tsf-keycloak/keycloak-0 8f116f6dd03fcfd133a5acffc3aa45438ab4cfe18fc20680e840ddbddeb500e0 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 8f116f6dd03fcfd133a5acffc3aa45438ab4cfe18fc20680e840ddbddeb500e0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:b1:fb:58:42:01\",\"name\":\"8f116f6dd03fcfd\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8401361d-392a-43d2-a4f7-3da8af990caa\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:13:35.196968 3881 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-vk94d-lx8rl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:35.197485 3881 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-vk94d-lx8rl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:36.218705 3881 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-vk94d-lx8rl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:36.218728 3881 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-vk94d-lx8rl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 23:13:37.606781 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:13:37.606813 3881 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:13:37.610902 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 23:13:37.762776 3881 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-zqxtc] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-zqxtc for pod on switch ip-10-0-2-103.ec2.internal I0327 23:13:37.762986 3881 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-zqxtc I0327 23:13:37.777576 3881 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-zqxtc] pod update took 14.610762ms I0327 23:13:37.777603 3881 base_network_controller_pods.go:937] [default/openshift-pipelines/tkn-cli-serve-784b566968-zqxtc] addLogicalPort annotation time took 14.644354ms I0327 23:13:37.779238 3881 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-zqxtc] addLogicalPort took 16.50093ms, libovsdb time 1.011568ms I0327 23:13:38.135879 3881 cni.go:403] [openshift-pipelines/tkn-cli-serve-784b566968-zqxtc b3408245c57bde8b1caccb1332b5dfb68a79b221b4e8fe00a1029ce44b754a18 network default NAD default] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-zqxtc b3408245c57bde8b1caccb1332b5dfb68a79b221b4e8fe00a1029ce44b754a18 network default NAD default] I0327 23:13:38.147961 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-zqxtc, hostIfaceName: b3408245c57bde8, network: default, NAD default, SandboxID: "b3408245c57bde8b1caccb1332b5dfb68a79b221b4e8fe00a1029ce44b754a18", PCI device ID: , UID: "eaa4614d-b2ba-452f-84dc-82fb52184fc9", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0327 23:13:38.211114 3881 cni.go:424] [openshift-pipelines/tkn-cli-serve-784b566968-zqxtc b3408245c57bde8b1caccb1332b5dfb68a79b221b4e8fe00a1029ce44b754a18 network default NAD default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-zqxtc b3408245c57bde8b1caccb1332b5dfb68a79b221b4e8fe00a1029ce44b754a18 network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:54:6e:25:e0:65\",\"name\":\"b3408245c57bde8\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/32dd5ace-2960-426b-bfd5-2a0c85597d10\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:13:51.322542 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:13:51.322571 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:13:51.327907 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 23:14:01.395411 3881 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-c5vkn] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-c5vkn for pod on switch ip-10-0-2-103.ec2.internal I0327 23:14:01.395522 3881 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-c5vkn I0327 23:14:01.409886 3881 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-c5vkn] pod update took 14.385361ms I0327 23:14:01.409911 3881 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-c5vkn] addLogicalPort annotation time took 14.414851ms I0327 23:14:01.411246 3881 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-c5vkn] addLogicalPort took 15.85031ms, libovsdb time 940.287µs I0327 23:14:01.759806 3881 cni.go:403] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-c5vkn f95314405e83f44a0b6a833d58245f44fd8f94f66ddbc5080bca14dd562d352b network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-c5vkn f95314405e83f44a0b6a833d58245f44fd8f94f66ddbc5080bca14dd562d352b network default NAD default] I0327 23:14:01.768383 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-c5vkn, hostIfaceName: f95314405e83f44, network: default, NAD default, SandboxID: "f95314405e83f44a0b6a833d58245f44fd8f94f66ddbc5080bca14dd562d352b", PCI device ID: , UID: "389bad19-0ed9-4a52-82c2-f52ba9262f9f", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0327 23:14:01.819729 3881 cni.go:424] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-c5vkn f95314405e83f44a0b6a833d58245f44fd8f94f66ddbc5080bca14dd562d352b network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-c5vkn f95314405e83f44a0b6a833d58245f44fd8f94f66ddbc5080bca14dd562d352b network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:12:ba:86:b3:a0\",\"name\":\"f95314405e83f44\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b20ae31a-8d1a-4dd9-903a-8798ef903460\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:14:04.781423 3881 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 23:14:04.781451 3881 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 23:14:04.807812 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 23:14:04.807829 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8082: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 23:14:05.265113 3881 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0327 23:14:11.003451 3881 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:11.004036 3881 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:11.024213 3881 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:11.024231 3881 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:11.029831 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0327 23:14:11.029856 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0327 23:14:14.838917 3881 cni.go:403] [tsf-tas/rekor-createtree-job-plgzg-4lhgq 6808a675d69ad53b4db9c518ac33f008663718e6bb74ebbd9a7f02e12d6890ed network default NAD default] DEL starting CNI request [tsf-tas/rekor-createtree-job-plgzg-4lhgq 6808a675d69ad53b4db9c518ac33f008663718e6bb74ebbd9a7f02e12d6890ed network default NAD default] I0327 23:14:14.911029 3881 cni.go:424] [tsf-tas/rekor-createtree-job-plgzg-4lhgq 6808a675d69ad53b4db9c518ac33f008663718e6bb74ebbd9a7f02e12d6890ed network default NAD default] DEL finished CNI request [tsf-tas/rekor-createtree-job-plgzg-4lhgq 6808a675d69ad53b4db9c518ac33f008663718e6bb74ebbd9a7f02e12d6890ed network default NAD default], result "{}", err I0327 23:14:15.062513 3881 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-plgzg-4lhgq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:15.062550 3881 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-plgzg-4lhgq I0327 23:14:15.063816 3881 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-plgzg-4lhgq, ips: 10.128.0.45 I0327 23:14:15.063850 3881 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-plgzg-4lhgq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:15.851906 3881 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-plgzg-4lhgq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:15.851928 3881 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-plgzg-4lhgq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:17.360326 3881 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-98jkh] creating logical port openshift-pipelines_pipelines-console-plugin-79cb8f5447-98jkh for pod on switch ip-10-0-2-103.ec2.internal I0327 23:14:17.360412 3881 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-79cb8f5447-98jkh I0327 23:14:17.375997 3881 pod.go:62] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-98jkh] pod update took 15.606486ms I0327 23:14:17.376017 3881 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-98jkh] addLogicalPort annotation time took 15.632022ms I0327 23:14:17.377234 3881 pods.go:271] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-98jkh] addLogicalPort took 16.925288ms, libovsdb time 821.75µs I0327 23:14:17.719624 3881 cni.go:403] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-98jkh 2b728d88a6b7f7fde5420537e9803542f305bd77ed6acad1b8097091b5e63011 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-98jkh 2b728d88a6b7f7fde5420537e9803542f305bd77ed6acad1b8097091b5e63011 network default NAD default] I0327 23:14:17.728607 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-79cb8f5447-98jkh, hostIfaceName: 2b728d88a6b7f7f, network: default, NAD default, SandboxID: "2b728d88a6b7f7fde5420537e9803542f305bd77ed6acad1b8097091b5e63011", PCI device ID: , UID: "e09a204e-7e3f-4c35-8546-6fe678346a8d", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0327 23:14:17.980683 3881 cni.go:424] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-98jkh 2b728d88a6b7f7fde5420537e9803542f305bd77ed6acad1b8097091b5e63011 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-98jkh 2b728d88a6b7f7fde5420537e9803542f305bd77ed6acad1b8097091b5e63011 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:71:2a:4f:7e:36\",\"name\":\"2b728d88a6b7f7f\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/366318c4-bf1a-4ffa-a9d7-0baab96302ed\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:14:18.049810 3881 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-6wkdj-vsqfb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:18.050290 3881 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-6wkdj-vsqfb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:18.942999 3881 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-6wkdj-vsqfb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:18.943024 3881 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-6wkdj-vsqfb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:19.066033 3881 obj_retry.go:464] Detected object tsf/patch-tekton-config-v78tr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:19.066565 3881 obj_retry.go:464] Detected object tsf/patch-tekton-config-v78tr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:20.086934 3881 obj_retry.go:458] Detected object tsf/patch-tekton-config-v78tr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:20.086963 3881 obj_retry.go:458] Detected object tsf/patch-tekton-config-v78tr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:21.123577 3881 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-6c88675596-cvw9f] creating logical port tsf-tas_rekor-redis-6c88675596-cvw9f for pod on switch ip-10-0-2-103.ec2.internal I0327 23:14:21.123731 3881 kube.go:256] Updating pod tsf-tas/rekor-redis-6c88675596-cvw9f I0327 23:14:21.137691 3881 pod.go:62] [tsf-tas/rekor-redis-6c88675596-cvw9f] pod update took 13.983765ms I0327 23:14:21.137714 3881 base_network_controller_pods.go:937] [default/tsf-tas/rekor-redis-6c88675596-cvw9f] addLogicalPort annotation time took 14.011843ms I0327 23:14:21.139296 3881 pods.go:271] [tsf-tas/rekor-redis-6c88675596-cvw9f] addLogicalPort took 15.736942ms, libovsdb time 961.175µs I0327 23:14:22.956940 3881 cni.go:403] [tsf-tas/rekor-redis-6c88675596-cvw9f 9de477d748780717beb1457002c43e6a8d2363865af3e738f6a8a16970397cbb network default NAD default] ADD starting CNI request [tsf-tas/rekor-redis-6c88675596-cvw9f 9de477d748780717beb1457002c43e6a8d2363865af3e738f6a8a16970397cbb network default NAD default] I0327 23:14:22.966345 3881 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-6c88675596-cvw9f, hostIfaceName: 9de477d74878071, network: default, NAD default, SandboxID: "9de477d748780717beb1457002c43e6a8d2363865af3e738f6a8a16970397cbb", PCI device ID: , UID: "d1e80b4e-ef0a-414a-9fbb-bd09ea22cb03", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0327 23:14:23.016479 3881 cni.go:424] [tsf-tas/rekor-redis-6c88675596-cvw9f 9de477d748780717beb1457002c43e6a8d2363865af3e738f6a8a16970397cbb network default NAD default] ADD finished CNI request [tsf-tas/rekor-redis-6c88675596-cvw9f 9de477d748780717beb1457002c43e6a8d2363865af3e738f6a8a16970397cbb network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:dc:a9:1e:3f:b0\",\"name\":\"9de477d74878071\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a0b4cc0b-a7cc-4f94-9e1f-a90c8f792ac0\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:14:23.995559 3881 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0327 23:14:24.341126 3881 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0327 23:14:24.902006 3881 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0327 23:14:25.112334 3881 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-p8t58 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:25.112888 3881 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-p8t58 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:26.137894 3881 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-p8t58 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:26.137920 3881 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-p8t58 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:26.180795 3881 obj_retry.go:458] Detected object tsf/patch-tekton-config-v78tr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:26.180819 3881 obj_retry.go:458] Detected object tsf/patch-tekton-config-v78tr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:26.184670 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-v78tr I0327 23:14:26.184692 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-v78tr I0327 23:14:34.160373 3881 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 W0327 23:14:35.051069 3881 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0327 23:14:36.196392 3881 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:36.197018 3881 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:36.221302 3881 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:36.221323 3881 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:36.227598 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0327 23:14:36.227620 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines W0327 23:14:36.949404 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis I0327 23:14:40.220341 3881 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:40.220999 3881 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:40.241299 3881 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:40.241318 3881 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:40.248397 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0327 23:14:40.248416 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0327 23:14:41.553361 3881 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-l5kzq] creating logical port tsf-tpa_create-db-l5kzq for pod on switch ip-10-0-2-103.ec2.internal I0327 23:14:41.553455 3881 kube.go:256] Updating pod tsf-tpa/create-db-l5kzq I0327 23:14:41.566252 3881 pod.go:62] [tsf-tpa/create-db-l5kzq] pod update took 12.820727ms I0327 23:14:41.566272 3881 base_network_controller_pods.go:937] [default/tsf-tpa/create-db-l5kzq] addLogicalPort annotation time took 12.845475ms I0327 23:14:41.567532 3881 pods.go:271] [tsf-tpa/create-db-l5kzq] addLogicalPort took 14.190637ms, libovsdb time 787.57µs I0327 23:14:41.915000 3881 cni.go:403] [tsf-tpa/create-db-l5kzq dcd600da307efcdc6fcf9789bd739b6643b5147d24f4f489bf459af42b12ae31 network default NAD default] ADD starting CNI request [tsf-tpa/create-db-l5kzq dcd600da307efcdc6fcf9789bd739b6643b5147d24f4f489bf459af42b12ae31 network default NAD default] I0327 23:14:41.924273 3881 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-db-l5kzq, hostIfaceName: dcd600da307efcd, network: default, NAD default, SandboxID: "dcd600da307efcdc6fcf9789bd739b6643b5147d24f4f489bf459af42b12ae31", PCI device ID: , UID: "9f2cc7b6-452c-45be-b152-bb43e567a43d", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0327 23:14:41.971273 3881 cni.go:424] [tsf-tpa/create-db-l5kzq dcd600da307efcdc6fcf9789bd739b6643b5147d24f4f489bf459af42b12ae31 network default NAD default] ADD finished CNI request [tsf-tpa/create-db-l5kzq dcd600da307efcdc6fcf9789bd739b6643b5147d24f4f489bf459af42b12ae31 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:59:22:2e:98:3d\",\"name\":\"dcd600da307efcd\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/733b5b00-7953-4724-b414-7f72057f2f07\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:14:43.091560 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=69 I0327 23:14:46.029786 3881 cni.go:403] [tsf-tpa/create-db-l5kzq dcd600da307efcdc6fcf9789bd739b6643b5147d24f4f489bf459af42b12ae31 network default NAD default] DEL starting CNI request [tsf-tpa/create-db-l5kzq dcd600da307efcdc6fcf9789bd739b6643b5147d24f4f489bf459af42b12ae31 network default NAD default] I0327 23:14:46.124367 3881 cni.go:424] [tsf-tpa/create-db-l5kzq dcd600da307efcdc6fcf9789bd739b6643b5147d24f4f489bf459af42b12ae31 network default NAD default] DEL finished CNI request [tsf-tpa/create-db-l5kzq dcd600da307efcdc6fcf9789bd739b6643b5147d24f4f489bf459af42b12ae31 network default NAD default], result "{}", err I0327 23:14:46.174163 3881 obj_retry.go:464] Detected object tsf-tpa/create-db-l5kzq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:46.174202 3881 pods.go:176] Deleting pod: tsf-tpa/create-db-l5kzq I0327 23:14:46.175597 3881 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-l5kzq, ips: 10.128.0.51 I0327 23:14:46.175628 3881 obj_retry.go:464] Detected object tsf-tpa/create-db-l5kzq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:47.196478 3881 obj_retry.go:458] Detected object tsf-tpa/create-db-l5kzq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:47.196524 3881 obj_retry.go:458] Detected object tsf-tpa/create-db-l5kzq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:47.315780 3881 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-czpsf] creating logical port tsf-tpa_migrate-db-czpsf for pod on switch ip-10-0-2-103.ec2.internal I0327 23:14:47.315922 3881 kube.go:256] Updating pod tsf-tpa/migrate-db-czpsf I0327 23:14:47.330792 3881 pod.go:62] [tsf-tpa/migrate-db-czpsf] pod update took 14.898274ms I0327 23:14:47.330811 3881 base_network_controller_pods.go:937] [default/tsf-tpa/migrate-db-czpsf] addLogicalPort annotation time took 14.921534ms I0327 23:14:47.332096 3881 pods.go:271] [tsf-tpa/migrate-db-czpsf] addLogicalPort took 16.351737ms, libovsdb time 879.014µs I0327 23:14:47.674669 3881 cni.go:403] [tsf-tpa/migrate-db-czpsf 4b2394cb9894bee3d0965beb613d107d8fd01e504ebcfa8b29b903be9d62102b network default NAD default] ADD starting CNI request [tsf-tpa/migrate-db-czpsf 4b2394cb9894bee3d0965beb613d107d8fd01e504ebcfa8b29b903be9d62102b network default NAD default] I0327 23:14:47.683219 3881 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-czpsf, hostIfaceName: 4b2394cb9894bee, network: default, NAD default, SandboxID: "4b2394cb9894bee3d0965beb613d107d8fd01e504ebcfa8b29b903be9d62102b", PCI device ID: , UID: "49b431df-06a1-48ba-97ee-08c20856c78a", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0327 23:14:47.729750 3881 cni.go:424] [tsf-tpa/migrate-db-czpsf 4b2394cb9894bee3d0965beb613d107d8fd01e504ebcfa8b29b903be9d62102b network default NAD default] ADD finished CNI request [tsf-tpa/migrate-db-czpsf 4b2394cb9894bee3d0965beb613d107d8fd01e504ebcfa8b29b903be9d62102b network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:2c:21:8e:64:60\",\"name\":\"4b2394cb9894bee\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9f566378-b528-4cab-b6f2-eb27f1591dec\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:14:50.058802 3881 cni.go:403] [tsf-tpa/migrate-db-czpsf 4b2394cb9894bee3d0965beb613d107d8fd01e504ebcfa8b29b903be9d62102b network default NAD default] DEL starting CNI request [tsf-tpa/migrate-db-czpsf 4b2394cb9894bee3d0965beb613d107d8fd01e504ebcfa8b29b903be9d62102b network default NAD default] I0327 23:14:50.149899 3881 cni.go:424] [tsf-tpa/migrate-db-czpsf 4b2394cb9894bee3d0965beb613d107d8fd01e504ebcfa8b29b903be9d62102b network default NAD default] DEL finished CNI request [tsf-tpa/migrate-db-czpsf 4b2394cb9894bee3d0965beb613d107d8fd01e504ebcfa8b29b903be9d62102b network default NAD default], result "{}", err I0327 23:14:50.216172 3881 obj_retry.go:464] Detected object tsf-tpa/migrate-db-czpsf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:50.216201 3881 pods.go:176] Deleting pod: tsf-tpa/migrate-db-czpsf I0327 23:14:50.217490 3881 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-czpsf, ips: 10.128.0.52 I0327 23:14:50.217517 3881 obj_retry.go:464] Detected object tsf-tpa/migrate-db-czpsf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:50.359294 3881 obj_retry.go:458] Detected object tsf-tpa/migrate-db-czpsf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:50.359313 3881 obj_retry.go:458] Detected object tsf-tpa/migrate-db-czpsf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:50.606804 3881 base_network_controller_pods.go:487] [default/tsf-tpa/create-importers-cd6ts] creating logical port tsf-tpa_create-importers-cd6ts for pod on switch ip-10-0-2-103.ec2.internal I0327 23:14:50.606891 3881 kube.go:256] Updating pod tsf-tpa/create-importers-cd6ts I0327 23:14:50.629548 3881 pod.go:62] [tsf-tpa/create-importers-cd6ts] pod update took 22.684836ms I0327 23:14:50.629569 3881 base_network_controller_pods.go:937] [default/tsf-tpa/create-importers-cd6ts] addLogicalPort annotation time took 22.708265ms I0327 23:14:50.630833 3881 pods.go:271] [tsf-tpa/create-importers-cd6ts] addLogicalPort took 24.046594ms, libovsdb time 853.313µs I0327 23:14:50.963171 3881 cni.go:403] [tsf-tpa/create-importers-cd6ts 49a21b72f2c11180657809e263cb3f719e28e2ca0d61a9118a4986666c8bb19d network default NAD default] ADD starting CNI request [tsf-tpa/create-importers-cd6ts 49a21b72f2c11180657809e263cb3f719e28e2ca0d61a9118a4986666c8bb19d network default NAD default] I0327 23:14:50.971570 3881 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-importers-cd6ts, hostIfaceName: 49a21b72f2c1118, network: default, NAD default, SandboxID: "49a21b72f2c11180657809e263cb3f719e28e2ca0d61a9118a4986666c8bb19d", PCI device ID: , UID: "0202bded-5509-40df-82d3-f9d039bb5fa3", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0327 23:14:51.016816 3881 cni.go:424] [tsf-tpa/create-importers-cd6ts 49a21b72f2c11180657809e263cb3f719e28e2ca0d61a9118a4986666c8bb19d network default NAD default] ADD finished CNI request [tsf-tpa/create-importers-cd6ts 49a21b72f2c11180657809e263cb3f719e28e2ca0d61a9118a4986666c8bb19d network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:5a:53:7c:cf:9d\",\"name\":\"49a21b72f2c1118\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f037151e-2487-4cee-8fda-907a67b8779f\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:14:53.083934 3881 cni.go:403] [tsf-tpa/create-importers-cd6ts 49a21b72f2c11180657809e263cb3f719e28e2ca0d61a9118a4986666c8bb19d network default NAD default] DEL starting CNI request [tsf-tpa/create-importers-cd6ts 49a21b72f2c11180657809e263cb3f719e28e2ca0d61a9118a4986666c8bb19d network default NAD default] I0327 23:14:53.170657 3881 cni.go:424] [tsf-tpa/create-importers-cd6ts 49a21b72f2c11180657809e263cb3f719e28e2ca0d61a9118a4986666c8bb19d network default NAD default] DEL finished CNI request [tsf-tpa/create-importers-cd6ts 49a21b72f2c11180657809e263cb3f719e28e2ca0d61a9118a4986666c8bb19d network default NAD default], result "{}", err I0327 23:14:53.234868 3881 obj_retry.go:464] Detected object tsf-tpa/create-importers-cd6ts of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:53.234897 3881 pods.go:176] Deleting pod: tsf-tpa/create-importers-cd6ts I0327 23:14:53.236388 3881 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-importers-cd6ts, ips: 10.128.0.53 I0327 23:14:53.236415 3881 obj_retry.go:464] Detected object tsf-tpa/create-importers-cd6ts of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:54.263077 3881 obj_retry.go:458] Detected object tsf-tpa/create-importers-cd6ts of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:54.263100 3881 obj_retry.go:458] Detected object tsf-tpa/create-importers-cd6ts of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:15:02.184670 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 W0327 23:15:05.278104 3881 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0327 23:15:05.407810 3881 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-42djh-vdd57 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:15:05.408399 3881 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-42djh-vdd57 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:15:06.274454 3881 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-42djh-vdd57 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:15:06.274476 3881 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-42djh-vdd57 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:15:08.180140 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0327 23:15:08.413542 3881 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:15:08.414179 3881 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:15:08.433289 3881 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:15:08.433306 3881 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:15:08.439207 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0327 23:15:08.439236 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0327 23:15:09.115143 3881 namespace.go:98] [image-controller] adding namespace I0327 23:15:09.116220 3881 namespace.go:102] [image-controller] adding namespace took 1.05278ms I0327 23:15:09.124831 3881 namespace.go:98] [integration-service] adding namespace I0327 23:15:09.124897 3881 namespace.go:98] [build-service] adding namespace I0327 23:15:09.125547 3881 namespace.go:102] [integration-service] adding namespace took 699.196µs I0327 23:15:09.126106 3881 namespace.go:102] [build-service] adding namespace took 1.192155ms I0327 23:15:09.139506 3881 namespace.go:142] [image-controller] updating namespace I0327 23:15:09.159364 3881 namespace.go:142] [image-controller] updating namespace I0327 23:15:09.176195 3881 namespace.go:142] [integration-service] updating namespace I0327 23:15:09.207747 3881 namespace.go:142] [integration-service] updating namespace I0327 23:15:09.223881 3881 namespace.go:142] [build-service] updating namespace I0327 23:15:09.241650 3881 namespace.go:142] [build-service] updating namespace I0327 23:15:09.384873 3881 namespace.go:142] [build-service] updating namespace I0327 23:15:09.407686 3881 namespace.go:142] [integration-service] updating namespace I0327 23:15:09.448582 3881 namespace.go:98] [release-service] adding namespace I0327 23:15:09.449292 3881 namespace.go:102] [release-service] adding namespace took 694.996µs I0327 23:15:09.469001 3881 namespace.go:142] [konflux-ui] updating namespace I0327 23:15:09.525848 3881 namespace.go:142] [release-service] updating namespace I0327 23:15:09.544492 3881 namespace.go:142] [release-service] updating namespace I0327 23:15:09.603784 3881 namespace.go:98] [konflux-info] adding namespace I0327 23:15:09.604541 3881 namespace.go:102] [konflux-info] adding namespace took 735.211µs I0327 23:15:09.648797 3881 namespace.go:98] [namespace-lister] adding namespace I0327 23:15:09.649489 3881 namespace.go:102] [namespace-lister] adding namespace took 675.832µs I0327 23:15:09.671936 3881 namespace.go:142] [konflux-info] updating namespace I0327 23:15:09.708547 3881 namespace.go:98] [enterprise-contract-service] adding namespace I0327 23:15:09.708691 3881 namespace.go:142] [image-controller] updating namespace I0327 23:15:09.709264 3881 namespace.go:102] [enterprise-contract-service] adding namespace took 700.835µs I0327 23:15:09.727180 3881 namespace.go:142] [konflux-info] updating namespace I0327 23:15:09.763142 3881 namespace.go:142] [namespace-lister] updating namespace I0327 23:15:09.827627 3881 namespace.go:142] [namespace-lister] updating namespace I0327 23:15:09.900932 3881 namespace.go:98] [default-tenant] adding namespace I0327 23:15:09.901598 3881 namespace.go:102] [default-tenant] adding namespace took 650.35µs I0327 23:15:09.913940 3881 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:15:09.951049 3881 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:15:09.975606 3881 namespace.go:142] [default-tenant] updating namespace I0327 23:15:09.995519 3881 namespace.go:98] [segment-bridge] adding namespace I0327 23:15:09.996235 3881 namespace.go:102] [segment-bridge] adding namespace took 703.482µs I0327 23:15:10.081484 3881 namespace.go:142] [default-tenant] updating namespace I0327 23:15:10.133706 3881 namespace.go:142] [segment-bridge] updating namespace I0327 23:15:10.406693 3881 namespace.go:142] [segment-bridge] updating namespace I0327 23:15:10.721843 3881 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0327 23:15:10.722417 3881 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0327 23:15:10.722431 3881 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0327 23:15:10.725025 3881 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0327 23:15:10.770807 3881 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0327 23:15:10.770826 3881 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0327 23:15:10.771472 3881 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0327 23:15:11.852191 3881 namespace.go:142] [build-service] updating namespace I0327 23:15:12.379851 3881 namespace.go:142] [default-tenant] updating namespace I0327 23:15:12.455013 3881 base_network_controller_pods.go:487] [default/konflux-ui/proxy-788756c547-c5zf7] creating logical port konflux-ui_proxy-788756c547-c5zf7 for pod on switch ip-10-0-2-103.ec2.internal I0327 23:15:12.455106 3881 kube.go:256] Updating pod konflux-ui/proxy-788756c547-c5zf7 I0327 23:15:12.604939 3881 pod.go:62] [konflux-ui/proxy-788756c547-c5zf7] pod update took 149.858504ms I0327 23:15:12.604960 3881 base_network_controller_pods.go:937] [default/konflux-ui/proxy-788756c547-c5zf7] addLogicalPort annotation time took 149.886955ms I0327 23:15:12.606311 3881 pods.go:271] [konflux-ui/proxy-788756c547-c5zf7] addLogicalPort took 151.307119ms, libovsdb time 915.343µs I0327 23:15:12.726633 3881 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:15:12.895111 3881 namespace.go:142] [image-controller] updating namespace I0327 23:15:13.095478 3881 namespace.go:142] [integration-service] updating namespace I0327 23:15:13.299900 3881 namespace.go:142] [konflux-info] updating namespace W0327 23:15:13.325672 3881 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0327 23:15:13.486678 3881 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-zxjxm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:15:13.487226 3881 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-zxjxm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:15:13.498581 3881 namespace.go:142] [namespace-lister] updating namespace I0327 23:15:13.696345 3881 namespace.go:142] [release-service] updating namespace I0327 23:15:13.898883 3881 namespace.go:142] [segment-bridge] updating namespace I0327 23:15:14.012684 3881 namespace.go:142] [build-service] updating namespace I0327 23:15:14.025843 3881 namespace.go:142] [default-tenant] updating namespace I0327 23:15:14.037719 3881 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:15:14.048253 3881 namespace.go:142] [image-controller] updating namespace I0327 23:15:14.059042 3881 namespace.go:142] [integration-service] updating namespace I0327 23:15:14.072056 3881 namespace.go:142] [konflux-info] updating namespace I0327 23:15:14.086125 3881 namespace.go:142] [namespace-lister] updating namespace I0327 23:15:14.096498 3881 namespace.go:142] [release-service] updating namespace I0327 23:15:14.111003 3881 namespace.go:142] [segment-bridge] updating namespace I0327 23:15:14.153595 3881 namespace.go:142] [build-service] updating namespace I0327 23:15:14.243716 3881 namespace.go:142] [default-tenant] updating namespace I0327 23:15:14.341682 3881 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:15:14.437895 3881 namespace.go:142] [image-controller] updating namespace I0327 23:15:14.516466 3881 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-zxjxm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:15:14.516491 3881 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-zxjxm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:15:14.546318 3881 namespace.go:142] [integration-service] updating namespace I0327 23:15:14.639786 3881 namespace.go:142] [konflux-info] updating namespace I0327 23:15:14.673217 3881 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-2-103.ec2.internal I0327 23:15:14.673311 3881 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0327 23:15:14.686729 3881 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 13.442773ms I0327 23:15:14.686746 3881 base_network_controller_pods.go:937] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 13.464408ms I0327 23:15:14.688280 3881 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 15.075003ms, libovsdb time 978.253µs I0327 23:15:14.741518 3881 namespace.go:142] [namespace-lister] updating namespace I0327 23:15:14.843577 3881 namespace.go:142] [release-service] updating namespace I0327 23:15:14.944592 3881 namespace.go:142] [segment-bridge] updating namespace I0327 23:15:15.033672 3881 cni.go:403] [konflux-ui/test-tsf-konflux e07f6ffb6a056d487d58503ab04fdec64233ef24a2d56af5357fc012e1ba91f8 network default NAD default] ADD starting CNI request [konflux-ui/test-tsf-konflux e07f6ffb6a056d487d58503ab04fdec64233ef24a2d56af5357fc012e1ba91f8 network default NAD default] I0327 23:15:15.042110 3881 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: e07f6ffb6a056d4, network: default, NAD default, SandboxID: "e07f6ffb6a056d487d58503ab04fdec64233ef24a2d56af5357fc012e1ba91f8", PCI device ID: , UID: "03b808cb-332e-4114-a665-3a21a33f0851", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0327 23:15:15.089853 3881 cni.go:424] [konflux-ui/test-tsf-konflux e07f6ffb6a056d487d58503ab04fdec64233ef24a2d56af5357fc012e1ba91f8 network default NAD default] ADD finished CNI request [konflux-ui/test-tsf-konflux e07f6ffb6a056d487d58503ab04fdec64233ef24a2d56af5357fc012e1ba91f8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:6f:a0:b9:87:8a\",\"name\":\"e07f6ffb6a056d4\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f13edd90-52e5-4f81-8798-d004609dc92d\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:15:15.590887 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0327 23:15:18.636950 3881 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0327 23:15:19.092994 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=58 I0327 23:15:20.525177 3881 cni.go:403] [konflux-ui/proxy-788756c547-c5zf7 b35cd9923865963b1868241b8abbdf14317ae86959123ebbce2f857a6d9ddb32 network default NAD default] ADD starting CNI request [konflux-ui/proxy-788756c547-c5zf7 b35cd9923865963b1868241b8abbdf14317ae86959123ebbce2f857a6d9ddb32 network default NAD default] I0327 23:15:20.534446 3881 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-788756c547-c5zf7, hostIfaceName: b35cd9923865963, network: default, NAD default, SandboxID: "b35cd9923865963b1868241b8abbdf14317ae86959123ebbce2f857a6d9ddb32", PCI device ID: , UID: "d87f7888-aea8-4d30-8dc7-1cbd74ecde67", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0327 23:15:20.793218 3881 cni.go:424] [konflux-ui/proxy-788756c547-c5zf7 b35cd9923865963b1868241b8abbdf14317ae86959123ebbce2f857a6d9ddb32 network default NAD default] ADD finished CNI request [konflux-ui/proxy-788756c547-c5zf7 b35cd9923865963b1868241b8abbdf14317ae86959123ebbce2f857a6d9ddb32 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:04:16:97:c6:a0\",\"name\":\"b35cd9923865963\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3b57a0ad-a106-4407-b41e-d3bd20735ee7\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 23:15:27.679065 3881 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 23:15:27.680742 3881 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 23:15:27.714303 3881 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 23:15:27.714321 3881 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0327 23:15:27.740644 3881 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 23:15:27.741852 3881 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 23:15:27.765803 3881 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0327 23:15:27.791752 3881 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0327 23:15:27.791774 3881 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0327 23:15:28.308090 3881 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 23:15:28.308111 3881 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0327 23:15:39.074051 3881 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-8jmp7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:15:39.074513 3881 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-8jmp7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:15:39.760067 3881 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-8jmp7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:15:39.760090 3881 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-8jmp7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:15:39.765946 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-8jmp7 I0327 23:15:39.765971 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-8jmp7 I0327 23:15:56.175032 3881 namespace.go:142] [openshift-pipelines] updating namespace W0327 23:16:05.754921 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 23:16:05.754943 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0327 23:16:05.754956 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 23:16:05.754961 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0327 23:16:06.369121 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 23:16:06.369141 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0327 23:16:22.097938 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=634 I0327 23:16:24.165193 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0327 23:16:26.190363 3881 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0327 23:16:36.359039 3881 obj_retry.go:464] Detected object konflux-ui/proxy-9cffd64c8-x6f68 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:16:36.359652 3881 obj_retry.go:464] Detected object konflux-ui/proxy-9cffd64c8-x6f68 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:16:37.069593 3881 obj_retry.go:458] Detected object konflux-ui/proxy-9cffd64c8-x6f68 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:16:37.069624 3881 obj_retry.go:458] Detected object konflux-ui/proxy-9cffd64c8-x6f68 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:16:37.076066 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-9cffd64c8-x6f68 I0327 23:16:37.076098 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-9cffd64c8-x6f68 I0327 23:16:40.196026 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0327 23:16:45.746329 3881 cni.go:403] [konflux-ui/test-tsf-konflux e07f6ffb6a056d487d58503ab04fdec64233ef24a2d56af5357fc012e1ba91f8 network default NAD default] DEL starting CNI request [konflux-ui/test-tsf-konflux e07f6ffb6a056d487d58503ab04fdec64233ef24a2d56af5357fc012e1ba91f8 network default NAD default] I0327 23:16:45.824169 3881 cni.go:424] [konflux-ui/test-tsf-konflux e07f6ffb6a056d487d58503ab04fdec64233ef24a2d56af5357fc012e1ba91f8 network default NAD default] DEL finished CNI request [konflux-ui/test-tsf-konflux e07f6ffb6a056d487d58503ab04fdec64233ef24a2d56af5357fc012e1ba91f8 network default NAD default], result "{}", err I0327 23:16:45.874186 3881 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:16:45.874210 3881 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0327 23:16:45.875506 3881 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.128.0.55 I0327 23:16:45.875529 3881 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:16:45.917365 3881 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:16:45.917387 3881 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:16:45.920219 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0327 23:16:45.920244 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0327 23:17:13.883514 3881 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-wfmlv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:17:13.884857 3881 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-wfmlv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:17:14.265826 3881 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-wfmlv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:17:14.265848 3881 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-wfmlv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:17:14.272680 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-wfmlv I0327 23:17:14.272710 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-wfmlv I0327 23:17:26.612294 3881 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-w5cm7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:17:26.612857 3881 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-w5cm7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:17:27.340610 3881 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-w5cm7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:17:27.340631 3881 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-w5cm7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:17:27.349514 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-w5cm7 I0327 23:17:27.349533 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-w5cm7 I0327 23:17:30.090136 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0327 23:17:35.175674 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0327 23:17:38.169752 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0327 23:17:49.159307 3881 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0327 23:18:12.089920 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=159 I0327 23:19:46.094704 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=124 I0327 23:20:00.258228 3881 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29577560-5z5dx for pod on switch ip-10-0-2-103.ec2.internal I0327 23:20:00.258350 3881 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx I0327 23:20:00.286687 3881 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx] pod update took 28.363861ms I0327 23:20:00.286709 3881 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx] addLogicalPort annotation time took 28.392514ms I0327 23:20:00.288015 3881 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx] addLogicalPort took 29.810097ms, libovsdb time 867.944µs I0327 23:20:00.614134 3881 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx 218c2159426c1fd53873a89d245b6e8025d1cde4e9bd1ad6619fbb7506c3f5b9 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx 218c2159426c1fd53873a89d245b6e8025d1cde4e9bd1ad6619fbb7506c3f5b9 network default NAD default] I0327 23:20:00.622483 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29577560-5z5dx, hostIfaceName: 218c2159426c1fd, network: default, NAD default, SandboxID: "218c2159426c1fd53873a89d245b6e8025d1cde4e9bd1ad6619fbb7506c3f5b9", PCI device ID: , UID: "3b8b661d-bdaa-429b-9a2f-db484cf2cc8d", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0327 23:20:00.670313 3881 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx 218c2159426c1fd53873a89d245b6e8025d1cde4e9bd1ad6619fbb7506c3f5b9 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx 218c2159426c1fd53873a89d245b6e8025d1cde4e9bd1ad6619fbb7506c3f5b9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"be:39:12:91:f7:65\",\"name\":\"218c2159426c1fd\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c6eb1f76-d031-46f9-a67c-18ddaefe174b\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:20:03.882355 3881 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx 218c2159426c1fd53873a89d245b6e8025d1cde4e9bd1ad6619fbb7506c3f5b9 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx 218c2159426c1fd53873a89d245b6e8025d1cde4e9bd1ad6619fbb7506c3f5b9 network default NAD default] I0327 23:20:03.971585 3881 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx 218c2159426c1fd53873a89d245b6e8025d1cde4e9bd1ad6619fbb7506c3f5b9 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx 218c2159426c1fd53873a89d245b6e8025d1cde4e9bd1ad6619fbb7506c3f5b9 network default NAD default], result "{}", err I0327 23:20:04.021130 3881 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:20:04.021159 3881 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx I0327 23:20:04.022495 3881 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx, ips: 10.128.0.56 I0327 23:20:04.022520 3881 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:20:04.892266 3881 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:20:04.892313 3881 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577560-5z5dx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:20:53.163535 3881 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0327 23:21:06.097332 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=38 I0327 23:21:46.168946 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0327 23:21:56.193778 3881 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0327 23:22:00.101707 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=41 I0327 23:22:10.198538 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0327 23:22:15.199508 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0327 23:22:43.188367 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0327 23:22:54.163378 3881 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=5 I0327 23:23:37.184033 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0327 23:23:45.095663 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=22 I0327 23:24:12.173200 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0327 23:24:21.179844 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0327 23:24:47.051280 3881 obj_retry.go:458] Detected object tsf-tpa/create-db-l5kzq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:24:47.051329 3881 obj_retry.go:458] Detected object tsf-tpa/create-db-l5kzq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:24:47.053888 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-l5kzq I0327 23:24:47.053913 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-l5kzq I0327 23:24:50.031442 3881 obj_retry.go:458] Detected object tsf-tpa/migrate-db-czpsf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:24:50.031484 3881 obj_retry.go:458] Detected object tsf-tpa/migrate-db-czpsf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:24:50.037919 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-czpsf I0327 23:24:50.037943 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-czpsf I0327 23:24:54.033231 3881 obj_retry.go:458] Detected object tsf-tpa/create-importers-cd6ts of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:24:54.033281 3881 obj_retry.go:458] Detected object tsf-tpa/create-importers-cd6ts of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:24:54.038131 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-cd6ts I0327 23:24:54.038165 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-cd6ts I0327 23:25:17.093439 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0327 23:25:20.103536 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=8 I0327 23:26:26.166786 3881 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=5 I0327 23:26:47.101035 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=26 I0327 23:27:57.197230 3881 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=5 I0327 23:28:15.990496 3881 namespace.go:98] [default-tenant-managed] adding namespace I0327 23:28:15.991497 3881 namespace.go:102] [default-tenant-managed] adding namespace took 975.093µs I0327 23:28:16.018483 3881 namespace.go:142] [default-tenant-managed] updating namespace I0327 23:28:16.038338 3881 namespace.go:142] [default-tenant-managed] updating namespace I0327 23:28:16.105106 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=13 I0327 23:28:17.083980 3881 namespace.go:142] [default-tenant-managed] updating namespace I0327 23:28:17.235480 3881 namespace.go:142] [default-tenant-managed] updating namespace I0327 23:28:17.315822 3881 namespace.go:142] [default-tenant-managed] updating namespace I0327 23:28:26.202466 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0327 23:29:10.727611 3881 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-xsdlw for pod on switch ip-10-0-2-103.ec2.internal I0327 23:29:10.727727 3881 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw I0327 23:29:10.744617 3881 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw] pod update took 16.919507ms I0327 23:29:10.744642 3881 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw] addLogicalPort annotation time took 16.951142ms I0327 23:29:10.746164 3881 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw] addLogicalPort took 18.562217ms, libovsdb time 1.097689ms I0327 23:29:11.088521 3881 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw d0a7a7263d79337da6ccdbb4a51aebdf9db988cea5ff5129e7c4010556a3862f network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw d0a7a7263d79337da6ccdbb4a51aebdf9db988cea5ff5129e7c4010556a3862f network default NAD default] I0327 23:29:11.098347 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-xsdlw, hostIfaceName: d0a7a7263d79337, network: default, NAD default, SandboxID: "d0a7a7263d79337da6ccdbb4a51aebdf9db988cea5ff5129e7c4010556a3862f", PCI device ID: , UID: "8d4551cc-136a-4111-9132-8b0afe0172f9", MAC: 0a:58:0a:80:00:39, IPs: [10.128.0.57/23] I0327 23:29:11.149852 3881 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw d0a7a7263d79337da6ccdbb4a51aebdf9db988cea5ff5129e7c4010556a3862f network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw d0a7a7263d79337da6ccdbb4a51aebdf9db988cea5ff5129e7c4010556a3862f network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:a3:54:ee:7a:89\",\"name\":\"d0a7a7263d79337\"},{\"mac\":\"0a:58:0a:80:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0830a329-f524-49d0-aa87-5879a3cd61cf\"}],\"ips\":[{\"address\":\"10.128.0.57/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:29:16.221713 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:29:16.222302 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:29:24.098791 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=13 W0327 23:29:24.213651 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:29:24.213670 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:29:24.213676 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:29:24.213680 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:29:24.213694 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:29:24.213700 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:29:24.213706 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:29:24.213711 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:29:25.005121 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:29:25.005141 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:29:25.005148 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:29:25.005153 3881 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0327 23:29:42.217959 3881 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod] creating logical port default-tenant_tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod for pod on switch ip-10-0-2-103.ec2.internal I0327 23:29:42.218135 3881 kube.go:256] Updating pod default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod I0327 23:29:42.249413 3881 pod.go:62] [default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod] pod update took 31.326936ms I0327 23:29:42.249436 3881 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod] addLogicalPort annotation time took 31.353095ms I0327 23:29:42.250660 3881 pods.go:271] [default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod] addLogicalPort took 32.720037ms, libovsdb time 820.929µs I0327 23:29:42.425387 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:29:42.426040 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:29:42.583605 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod bf56735d90a9ff8769847ba6867e6bee8317795378cdedc678deec0ab221f988 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod bf56735d90a9ff8769847ba6867e6bee8317795378cdedc678deec0ab221f988 network default NAD default] I0327 23:29:42.592045 3881 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod, hostIfaceName: bf56735d90a9ff8, network: default, NAD default, SandboxID: "bf56735d90a9ff8769847ba6867e6bee8317795378cdedc678deec0ab221f988", PCI device ID: , UID: "25baca7f-7a24-48f4-b470-f86b41c65bc2", MAC: 0a:58:0a:80:00:3a, IPs: [10.128.0.58/23] I0327 23:29:42.640728 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod bf56735d90a9ff8769847ba6867e6bee8317795378cdedc678deec0ab221f988 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod bf56735d90a9ff8769847ba6867e6bee8317795378cdedc678deec0ab221f988 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:fb:37:e0:c5:17\",\"name\":\"bf56735d90a9ff8\"},{\"mac\":\"0a:58:0a:80:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/443c1e0b-ae5f-49bc-bcd2-2cc9fc7e927d\"}],\"ips\":[{\"address\":\"10.128.0.58/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:29:46.099101 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=6 I0327 23:29:51.183590 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=5 I0327 23:30:00.189877 3881 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29577570-h5zxl for pod on switch ip-10-0-2-103.ec2.internal I0327 23:30:00.189974 3881 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl I0327 23:30:00.204529 3881 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl] pod update took 14.577063ms I0327 23:30:00.204554 3881 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl] addLogicalPort annotation time took 14.606859ms I0327 23:30:00.205833 3881 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl] addLogicalPort took 15.971168ms, libovsdb time 863.481µs I0327 23:30:00.551710 3881 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl 0072c1ac3b789b472ab2688bc5587f010581c92a6a49ad06c0023ac6196f402c network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl 0072c1ac3b789b472ab2688bc5587f010581c92a6a49ad06c0023ac6196f402c network default NAD default] I0327 23:30:00.560964 3881 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29577570-h5zxl, hostIfaceName: 0072c1ac3b789b4, network: default, NAD default, SandboxID: "0072c1ac3b789b472ab2688bc5587f010581c92a6a49ad06c0023ac6196f402c", PCI device ID: , UID: "4527bd1c-4a8d-4186-ab04-f02805d29044", MAC: 0a:58:0a:80:00:3b, IPs: [10.128.0.59/23] I0327 23:30:00.610815 3881 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl 0072c1ac3b789b472ab2688bc5587f010581c92a6a49ad06c0023ac6196f402c network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl 0072c1ac3b789b472ab2688bc5587f010581c92a6a49ad06c0023ac6196f402c network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:dd:2f:a0:f3:5b\",\"name\":\"0072c1ac3b789b4\"},{\"mac\":\"0a:58:0a:80:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c988b853-7070-4458-932f-e13ead367497\"}],\"ips\":[{\"address\":\"10.128.0.59/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:30:02.194559 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0327 23:30:02.299060 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod bf56735d90a9ff8769847ba6867e6bee8317795378cdedc678deec0ab221f988 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod bf56735d90a9ff8769847ba6867e6bee8317795378cdedc678deec0ab221f988 network default NAD default] I0327 23:30:02.383704 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod bf56735d90a9ff8769847ba6867e6bee8317795378cdedc678deec0ab221f988 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod bf56735d90a9ff8769847ba6867e6bee8317795378cdedc678deec0ab221f988 network default NAD default], result "{}", err I0327 23:30:02.501632 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:30:02.501676 3881 pods.go:176] Deleting pod: default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod I0327 23:30:02.503191 3881 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod, ips: 10.128.0.58 I0327 23:30:02.503217 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:30:04.202742 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0327 23:30:04.315699 3881 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl 0072c1ac3b789b472ab2688bc5587f010581c92a6a49ad06c0023ac6196f402c network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl 0072c1ac3b789b472ab2688bc5587f010581c92a6a49ad06c0023ac6196f402c network default NAD default] I0327 23:30:04.409664 3881 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl 0072c1ac3b789b472ab2688bc5587f010581c92a6a49ad06c0023ac6196f402c network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl 0072c1ac3b789b472ab2688bc5587f010581c92a6a49ad06c0023ac6196f402c network default NAD default], result "{}", err I0327 23:30:04.460932 3881 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:30:04.460972 3881 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl I0327 23:30:04.462334 3881 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl, ips: 10.128.0.59 I0327 23:30:04.462364 3881 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:30:05.321320 3881 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:30:05.321348 3881 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577570-h5zxl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:30:05.360738 3881 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:30:05.360777 3881 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:30:05.365687 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x I0327 23:30:05.365715 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-2l24x I0327 23:30:51.107595 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0327 23:31:12.172869 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=11 I0327 23:31:56.170023 3881 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0327 23:32:00.167467 3881 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=11 I0327 23:32:17.909174 3881 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod] creating logical port default-tenant_tsf-comp-uyub-on-push-n7csp-build-image-index-pod for pod on switch ip-10-0-2-103.ec2.internal I0327 23:32:17.909314 3881 kube.go:256] Updating pod default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod I0327 23:32:17.934930 3881 pod.go:62] [default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod] pod update took 25.650035ms I0327 23:32:17.934951 3881 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod] addLogicalPort annotation time took 25.676263ms I0327 23:32:17.936238 3881 pods.go:271] [default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod] addLogicalPort took 27.082528ms, libovsdb time 889.049µs I0327 23:32:18.273406 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod b5b74754530d9768473f97e5d871fc06540e5931a209f947b67a8442fbaebe90 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod b5b74754530d9768473f97e5d871fc06540e5931a209f947b67a8442fbaebe90 network default NAD default] I0327 23:32:18.281956 3881 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-uyub-on-push-n7csp-build-image-index-pod, hostIfaceName: b5b74754530d976, network: default, NAD default, SandboxID: "b5b74754530d9768473f97e5d871fc06540e5931a209f947b67a8442fbaebe90", PCI device ID: , UID: "725ee1ff-5fd4-485e-b21d-de546572918e", MAC: 0a:58:0a:80:00:3c, IPs: [10.128.0.60/23] I0327 23:32:18.329186 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod b5b74754530d9768473f97e5d871fc06540e5931a209f947b67a8442fbaebe90 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod b5b74754530d9768473f97e5d871fc06540e5931a209f947b67a8442fbaebe90 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:ec:0e:fb:a6:cb\",\"name\":\"b5b74754530d976\"},{\"mac\":\"0a:58:0a:80:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c95698ef-b58f-4ca2-a8e2-cb166d6c4a76\"}],\"ips\":[{\"address\":\"10.128.0.60/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:32:18.425953 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:32:18.426509 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:32:46.176603 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0327 23:33:00.306689 3881 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod] creating logical port default-tenant_tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod for pod on switch ip-10-0-2-103.ec2.internal I0327 23:33:00.306987 3881 kube.go:256] Updating pod default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod I0327 23:33:00.336815 3881 pod.go:62] [default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod] pod update took 29.884801ms I0327 23:33:00.336835 3881 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod] addLogicalPort annotation time took 29.910233ms I0327 23:33:00.338194 3881 pods.go:271] [default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod] addLogicalPort took 31.523106ms, libovsdb time 961.972µs I0327 23:33:00.372561 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod b5b74754530d9768473f97e5d871fc06540e5931a209f947b67a8442fbaebe90 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod b5b74754530d9768473f97e5d871fc06540e5931a209f947b67a8442fbaebe90 network default NAD default] I0327 23:33:00.458883 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod b5b74754530d9768473f97e5d871fc06540e5931a209f947b67a8442fbaebe90 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod b5b74754530d9768473f97e5d871fc06540e5931a209f947b67a8442fbaebe90 network default NAD default], result "{}", err I0327 23:33:00.535279 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:33:00.535309 3881 pods.go:176] Deleting pod: default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod I0327 23:33:00.536872 3881 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod, ips: 10.128.0.60 I0327 23:33:00.536898 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:33:00.672136 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod 3b409fb93c648ea4b037cc1872fde7be85933135e20f76a6fd5747a834ccc17f network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod 3b409fb93c648ea4b037cc1872fde7be85933135e20f76a6fd5747a834ccc17f network default NAD default] I0327 23:33:00.680866 3881 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod, hostIfaceName: 3b409fb93c648ea, network: default, NAD default, SandboxID: "3b409fb93c648ea4b037cc1872fde7be85933135e20f76a6fd5747a834ccc17f", PCI device ID: , UID: "13aff1bd-2a78-42b3-8cd1-0135e5351c37", MAC: 0a:58:0a:80:00:3d, IPs: [10.128.0.61/23] I0327 23:33:00.728331 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod 3b409fb93c648ea4b037cc1872fde7be85933135e20f76a6fd5747a834ccc17f network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod 3b409fb93c648ea4b037cc1872fde7be85933135e20f76a6fd5747a834ccc17f network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:68:32:28:c8:7e\",\"name\":\"3b409fb93c648ea\"},{\"mac\":\"0a:58:0a:80:00:3d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1ecaab64-7a08-4de0-83a5-65cbcebf70e2\"}],\"ips\":[{\"address\":\"10.128.0.61/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:33:01.379707 3881 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod] creating logical port default-tenant_tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod for pod on switch ip-10-0-2-103.ec2.internal I0327 23:33:01.379873 3881 kube.go:256] Updating pod default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod I0327 23:33:01.402314 3881 pod.go:62] [default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod] pod update took 22.470947ms I0327 23:33:01.402333 3881 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod] addLogicalPort annotation time took 22.495211ms I0327 23:33:01.403633 3881 pods.go:271] [default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod] addLogicalPort took 23.940669ms, libovsdb time 920.602µs I0327 23:33:01.744296 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod a0d6ab4f1c2be60d0d93ba06c9826e681438489f0c78a5373bc6cc390b442fee network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod a0d6ab4f1c2be60d0d93ba06c9826e681438489f0c78a5373bc6cc390b442fee network default NAD default] I0327 23:33:01.753340 3881 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod, hostIfaceName: a0d6ab4f1c2be60, network: default, NAD default, SandboxID: "a0d6ab4f1c2be60d0d93ba06c9826e681438489f0c78a5373bc6cc390b442fee", PCI device ID: , UID: "f9ab57c3-c23b-44c9-a08a-946cc9e5ef02", MAC: 0a:58:0a:80:00:3e, IPs: [10.128.0.62/23] I0327 23:33:01.782501 3881 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod] creating logical port default-tenant_tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod for pod on switch ip-10-0-2-103.ec2.internal I0327 23:33:01.782625 3881 kube.go:256] Updating pod default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod I0327 23:33:01.803825 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod a0d6ab4f1c2be60d0d93ba06c9826e681438489f0c78a5373bc6cc390b442fee network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod a0d6ab4f1c2be60d0d93ba06c9826e681438489f0c78a5373bc6cc390b442fee network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:95:b5:16:c7:5c\",\"name\":\"a0d6ab4f1c2be60\"},{\"mac\":\"0a:58:0a:80:00:3e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6bfb1d0d-acc9-419f-a43d-850c6c64de55\"}],\"ips\":[{\"address\":\"10.128.0.62/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:33:01.817837 3881 pod.go:62] [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod] pod update took 35.245928ms I0327 23:33:01.817859 3881 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod] addLogicalPort annotation time took 35.273817ms I0327 23:33:01.819148 3881 pods.go:271] [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod] addLogicalPort took 36.665243ms, libovsdb time 894.492µs I0327 23:33:02.057715 3881 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod] creating logical port default-tenant_tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod for pod on switch ip-10-0-2-103.ec2.internal I0327 23:33:02.057854 3881 kube.go:256] Updating pod default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod I0327 23:33:02.097893 3881 pod.go:62] [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod] pod update took 40.07698ms I0327 23:33:02.097912 3881 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod] addLogicalPort annotation time took 40.100767ms I0327 23:33:02.099310 3881 pods.go:271] [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod] addLogicalPort took 41.611435ms, libovsdb time 971.473µs I0327 23:33:02.147992 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod 26f79dfc34cf9cd99e977cea8e4d94c9654457cd690817a60dad2ef873de484b network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod 26f79dfc34cf9cd99e977cea8e4d94c9654457cd690817a60dad2ef873de484b network default NAD default] I0327 23:33:02.156550 3881 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod, hostIfaceName: 26f79dfc34cf9cd, network: default, NAD default, SandboxID: "26f79dfc34cf9cd99e977cea8e4d94c9654457cd690817a60dad2ef873de484b", PCI device ID: , UID: "cebe73c9-a2b7-486a-81c5-4bb2557eb876", MAC: 0a:58:0a:80:00:3f, IPs: [10.128.0.63/23] I0327 23:33:02.206611 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod 26f79dfc34cf9cd99e977cea8e4d94c9654457cd690817a60dad2ef873de484b network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod 26f79dfc34cf9cd99e977cea8e4d94c9654457cd690817a60dad2ef873de484b network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:99:c5:9e:ab:cd\",\"name\":\"26f79dfc34cf9cd\"},{\"mac\":\"0a:58:0a:80:00:3f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d1bad0e8-d353-4e58-8e35-7c97666e840b\"}],\"ips\":[{\"address\":\"10.128.0.63/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:33:02.423459 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod 8184f39d415af98930bac43e45978b0ced0cbc6fdbdfee1758596747a0b7f0b6 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod 8184f39d415af98930bac43e45978b0ced0cbc6fdbdfee1758596747a0b7f0b6 network default NAD default] I0327 23:33:02.434135 3881 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod, hostIfaceName: 8184f39d415af98, network: default, NAD default, SandboxID: "8184f39d415af98930bac43e45978b0ced0cbc6fdbdfee1758596747a0b7f0b6", PCI device ID: , UID: "989653ae-5a6c-49c0-9edc-f95ae6792698", MAC: 0a:58:0a:80:00:40, IPs: [10.128.0.64/23] I0327 23:33:02.484495 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod 8184f39d415af98930bac43e45978b0ced0cbc6fdbdfee1758596747a0b7f0b6 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod 8184f39d415af98930bac43e45978b0ced0cbc6fdbdfee1758596747a0b7f0b6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:29:2b:fa:0b:16\",\"name\":\"8184f39d415af98\"},{\"mac\":\"0a:58:0a:80:00:40\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5db7f28d-8c55-4ed9-b751-6aeb3529064a\"}],\"ips\":[{\"address\":\"10.128.0.64/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:33:02.695054 3881 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod] creating logical port default-tenant_tsf-comp-uyub-on-push-n7csp-apply-tags-pod for pod on switch ip-10-0-2-103.ec2.internal I0327 23:33:02.695189 3881 kube.go:256] Updating pod default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod I0327 23:33:02.713366 3881 pod.go:62] [default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod] pod update took 18.204539ms I0327 23:33:02.713386 3881 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod] addLogicalPort annotation time took 18.22756ms I0327 23:33:02.714793 3881 pods.go:271] [default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod] addLogicalPort took 19.750353ms, libovsdb time 873.943µs I0327 23:33:03.065866 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod f9101f38a609cc5f2b73eaf2647533de2dbc79a297b0769173d1e90bce7f8f78 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod f9101f38a609cc5f2b73eaf2647533de2dbc79a297b0769173d1e90bce7f8f78 network default NAD default] I0327 23:33:03.076371 3881 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-uyub-on-push-n7csp-apply-tags-pod, hostIfaceName: f9101f38a609cc5, network: default, NAD default, SandboxID: "f9101f38a609cc5f2b73eaf2647533de2dbc79a297b0769173d1e90bce7f8f78", PCI device ID: , UID: "adad8bc2-c817-4af4-9778-687496dab1ae", MAC: 0a:58:0a:80:00:41, IPs: [10.128.0.65/23] I0327 23:33:03.135966 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod f9101f38a609cc5f2b73eaf2647533de2dbc79a297b0769173d1e90bce7f8f78 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod f9101f38a609cc5f2b73eaf2647533de2dbc79a297b0769173d1e90bce7f8f78 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:95:47:33:66:61\",\"name\":\"f9101f38a609cc5\"},{\"mac\":\"0a:58:0a:80:00:41\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/43dd4b30-4dc0-4243-8c64-57eb0926e57a\"}],\"ips\":[{\"address\":\"10.128.0.65/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:33:03.539688 3881 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod] creating logical port default-tenant_tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod for pod on switch ip-10-0-2-103.ec2.internal I0327 23:33:03.539907 3881 kube.go:256] Updating pod default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod I0327 23:33:03.559160 3881 pod.go:62] [default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod] pod update took 19.321399ms I0327 23:33:03.559183 3881 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod] addLogicalPort annotation time took 19.348085ms I0327 23:33:03.560618 3881 pods.go:271] [default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod] addLogicalPort took 20.947856ms, libovsdb time 915.679µs I0327 23:33:03.695279 3881 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod] creating logical port default-tenant_tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod for pod on switch ip-10-0-2-103.ec2.internal I0327 23:33:03.695484 3881 kube.go:256] Updating pod default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod I0327 23:33:03.715587 3881 pod.go:62] [default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod] pod update took 20.14287ms I0327 23:33:03.715608 3881 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod] addLogicalPort annotation time took 20.168692ms I0327 23:33:03.716979 3881 pods.go:271] [default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod] addLogicalPort took 21.721636ms, libovsdb time 870.674µs I0327 23:33:03.937214 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod 88856ea7fcd1a30fe4c996e6b7a7618fe455295c33c03e1fd79eaa3ce6eed19f network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod 88856ea7fcd1a30fe4c996e6b7a7618fe455295c33c03e1fd79eaa3ce6eed19f network default NAD default] I0327 23:33:03.947328 3881 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod, hostIfaceName: 88856ea7fcd1a30, network: default, NAD default, SandboxID: "88856ea7fcd1a30fe4c996e6b7a7618fe455295c33c03e1fd79eaa3ce6eed19f", PCI device ID: , UID: "0d55895a-bf99-4b7a-8a03-e7ea4916b65e", MAC: 0a:58:0a:80:00:42, IPs: [10.128.0.66/23] I0327 23:33:04.007359 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod 88856ea7fcd1a30fe4c996e6b7a7618fe455295c33c03e1fd79eaa3ce6eed19f network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod 88856ea7fcd1a30fe4c996e6b7a7618fe455295c33c03e1fd79eaa3ce6eed19f network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:7c:c4:3a:a4:bc\",\"name\":\"88856ea7fcd1a30\"},{\"mac\":\"0a:58:0a:80:00:42\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/beb7e548-0b31-4644-9ddd-b08027b9ddb0\"}],\"ips\":[{\"address\":\"10.128.0.66/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:33:04.096687 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod 7e316c4b1448c3cd8df63131eba506aa2a3955cc86ffe32cd46852f1488f698b network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod 7e316c4b1448c3cd8df63131eba506aa2a3955cc86ffe32cd46852f1488f698b network default NAD default] I0327 23:33:04.109342 3881 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod, hostIfaceName: 7e316c4b1448c3c, network: default, NAD default, SandboxID: "7e316c4b1448c3cd8df63131eba506aa2a3955cc86ffe32cd46852f1488f698b", PCI device ID: , UID: "b0e4e19d-4f4d-4fdf-91b3-2022c62e8426", MAC: 0a:58:0a:80:00:43, IPs: [10.128.0.67/23] I0327 23:33:04.174169 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod 7e316c4b1448c3cd8df63131eba506aa2a3955cc86ffe32cd46852f1488f698b network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod 7e316c4b1448c3cd8df63131eba506aa2a3955cc86ffe32cd46852f1488f698b network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:94:42:c9:8c:59\",\"name\":\"7e316c4b1448c3c\"},{\"mac\":\"0a:58:0a:80:00:43\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cd77c086-c939-4a73-a81e-98e5a1751e52\"}],\"ips\":[{\"address\":\"10.128.0.67/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:33:24.188132 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0327 23:34:00.005783 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod a0d6ab4f1c2be60d0d93ba06c9826e681438489f0c78a5373bc6cc390b442fee network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod a0d6ab4f1c2be60d0d93ba06c9826e681438489f0c78a5373bc6cc390b442fee network default NAD default] I0327 23:34:00.007462 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod f9101f38a609cc5f2b73eaf2647533de2dbc79a297b0769173d1e90bce7f8f78 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod f9101f38a609cc5f2b73eaf2647533de2dbc79a297b0769173d1e90bce7f8f78 network default NAD default] I0327 23:34:00.124008 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod a0d6ab4f1c2be60d0d93ba06c9826e681438489f0c78a5373bc6cc390b442fee network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod a0d6ab4f1c2be60d0d93ba06c9826e681438489f0c78a5373bc6cc390b442fee network default NAD default], result "{}", err I0327 23:34:00.144572 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod f9101f38a609cc5f2b73eaf2647533de2dbc79a297b0769173d1e90bce7f8f78 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod f9101f38a609cc5f2b73eaf2647533de2dbc79a297b0769173d1e90bce7f8f78 network default NAD default], result "{}", err I0327 23:34:00.261816 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:00.261930 3881 pods.go:176] Deleting pod: default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod I0327 23:34:00.263497 3881 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod, ips: 10.128.0.62 I0327 23:34:00.263525 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-coverity-availability-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:00.298324 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:00.298362 3881 pods.go:176] Deleting pod: default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod I0327 23:34:00.299779 3881 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod, ips: 10.128.0.65 I0327 23:34:00.299803 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-apply-tags-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:02.033707 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod 26f79dfc34cf9cd99e977cea8e4d94c9654457cd690817a60dad2ef873de484b network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod 26f79dfc34cf9cd99e977cea8e4d94c9654457cd690817a60dad2ef873de484b network default NAD default] I0327 23:34:02.123007 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod 26f79dfc34cf9cd99e977cea8e4d94c9654457cd690817a60dad2ef873de484b network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod 26f79dfc34cf9cd99e977cea8e4d94c9654457cd690817a60dad2ef873de484b network default NAD default], result "{}", err I0327 23:34:02.244741 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:02.244856 3881 pods.go:176] Deleting pod: default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod I0327 23:34:02.246868 3881 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod, ips: 10.128.0.63 I0327 23:34:02.246955 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-sast-snyk-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:04.258314 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:04.258883 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:15.148652 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:15.149373 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:15.166567 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod 7e316c4b1448c3cd8df63131eba506aa2a3955cc86ffe32cd46852f1488f698b network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod 7e316c4b1448c3cd8df63131eba506aa2a3955cc86ffe32cd46852f1488f698b network default NAD default] I0327 23:34:15.166651 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod 8184f39d415af98930bac43e45978b0ced0cbc6fdbdfee1758596747a0b7f0b6 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod 8184f39d415af98930bac43e45978b0ced0cbc6fdbdfee1758596747a0b7f0b6 network default NAD default] I0327 23:34:15.290389 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:15.291196 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:15.291821 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod 7e316c4b1448c3cd8df63131eba506aa2a3955cc86ffe32cd46852f1488f698b network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod 7e316c4b1448c3cd8df63131eba506aa2a3955cc86ffe32cd46852f1488f698b network default NAD default], result "{}", err I0327 23:34:15.300289 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod 8184f39d415af98930bac43e45978b0ced0cbc6fdbdfee1758596747a0b7f0b6 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod 8184f39d415af98930bac43e45978b0ced0cbc6fdbdfee1758596747a0b7f0b6 network default NAD default], result "{}", err I0327 23:34:15.476779 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:15.476814 3881 pods.go:176] Deleting pod: default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod I0327 23:34:15.478603 3881 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod, ips: 10.128.0.67 I0327 23:34:15.478630 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-push-dockerfile-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:15.511989 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:15.512054 3881 pods.go:176] Deleting pod: default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod I0327 23:34:15.514025 3881 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod, ips: 10.128.0.64 I0327 23:34:15.514078 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:50.444886 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod 88856ea7fcd1a30fe4c996e6b7a7618fe455295c33c03e1fd79eaa3ce6eed19f network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod 88856ea7fcd1a30fe4c996e6b7a7618fe455295c33c03e1fd79eaa3ce6eed19f network default NAD default] I0327 23:34:50.530482 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod 88856ea7fcd1a30fe4c996e6b7a7618fe455295c33c03e1fd79eaa3ce6eed19f network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod 88856ea7fcd1a30fe4c996e6b7a7618fe455295c33c03e1fd79eaa3ce6eed19f network default NAD default], result "{}", err I0327 23:34:50.723868 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:50.723902 3881 pods.go:176] Deleting pod: default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod I0327 23:34:50.725226 3881 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod, ips: 10.128.0.66 I0327 23:34:50.725253 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:54.484728 3881 cni.go:403] [default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod 3b409fb93c648ea4b037cc1872fde7be85933135e20f76a6fd5747a834ccc17f network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod 3b409fb93c648ea4b037cc1872fde7be85933135e20f76a6fd5747a834ccc17f network default NAD default] I0327 23:34:54.584227 3881 cni.go:424] [default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod 3b409fb93c648ea4b037cc1872fde7be85933135e20f76a6fd5747a834ccc17f network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod 3b409fb93c648ea4b037cc1872fde7be85933135e20f76a6fd5747a834ccc17f network default NAD default], result "{}", err I0327 23:34:54.691845 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:54.691875 3881 pods.go:176] Deleting pod: default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod I0327 23:34:54.693330 3881 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod, ips: 10.128.0.61 I0327 23:34:54.693364 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-ecosystem-cert-preflight-checks-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:34:55.096834 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 W0327 23:34:56.475908 3881 duration.go:461] Config duration recorder: measurement expired for pod/tsf-tpa/create-importers-cd6ts I0327 23:35:11.942894 3881 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw d0a7a7263d79337da6ccdbb4a51aebdf9db988cea5ff5129e7c4010556a3862f network default NAD default] DEL starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw d0a7a7263d79337da6ccdbb4a51aebdf9db988cea5ff5129e7c4010556a3862f network default NAD default] I0327 23:35:12.029733 3881 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw d0a7a7263d79337da6ccdbb4a51aebdf9db988cea5ff5129e7c4010556a3862f network default NAD default] DEL finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw d0a7a7263d79337da6ccdbb4a51aebdf9db988cea5ff5129e7c4010556a3862f network default NAD default], result "{}", err I0327 23:35:12.107203 3881 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:35:12.107234 3881 pods.go:176] Deleting pod: openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw I0327 23:35:12.108923 3881 pods.go:236] Attempting to release IPs for pod: openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw, ips: 10.128.0.57 I0327 23:35:12.108958 3881 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:35:12.905677 3881 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:35:12.905700 3881 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:35:12.909010 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw I0327 23:35:12.909036 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-xsdlw I0327 23:35:24.102369 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=10 I0327 23:35:48.207182 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=5 I0327 23:35:49.202425 3881 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0327 23:36:01.108990 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=233 I0327 23:36:11.508016 3881 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-22pqh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:36:11.508551 3881 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-22pqh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:36:11.669016 3881 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-22pqh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:36:11.669045 3881 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-22pqh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:36:11.675625 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-22pqh I0327 23:36:11.675647 3881 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-22pqh I0327 23:36:19.104539 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=54 I0327 23:36:27.066272 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:36:27.066855 3881 obj_retry.go:464] Detected object default-tenant/tsf-comp-uyub-on-push-n7csp-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:36:49.965602 3881 base_network_controller_pods.go:487] [default/default-tenant/my-integration-test-iaxd-hdhqk-verify-pod] creating logical port default-tenant_my-integration-test-iaxd-hdhqk-verify-pod for pod on switch ip-10-0-2-103.ec2.internal I0327 23:36:49.965804 3881 kube.go:256] Updating pod default-tenant/my-integration-test-iaxd-hdhqk-verify-pod I0327 23:36:50.004492 3881 pod.go:62] [default-tenant/my-integration-test-iaxd-hdhqk-verify-pod] pod update took 38.75481ms I0327 23:36:50.004520 3881 base_network_controller_pods.go:937] [default/default-tenant/my-integration-test-iaxd-hdhqk-verify-pod] addLogicalPort annotation time took 38.788372ms I0327 23:36:50.005842 3881 pods.go:271] [default-tenant/my-integration-test-iaxd-hdhqk-verify-pod] addLogicalPort took 40.258225ms, libovsdb time 882.647µs I0327 23:36:50.334204 3881 cni.go:403] [default-tenant/my-integration-test-iaxd-hdhqk-verify-pod a3c4223f5aa9ae17725dc24ba4c484f855e73935ac3903c304a9c8e200b91e33 network default NAD default] ADD starting CNI request [default-tenant/my-integration-test-iaxd-hdhqk-verify-pod a3c4223f5aa9ae17725dc24ba4c484f855e73935ac3903c304a9c8e200b91e33 network default NAD default] I0327 23:36:50.343075 3881 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: my-integration-test-iaxd-hdhqk-verify-pod, hostIfaceName: a3c4223f5aa9ae1, network: default, NAD default, SandboxID: "a3c4223f5aa9ae17725dc24ba4c484f855e73935ac3903c304a9c8e200b91e33", PCI device ID: , UID: "4d41b320-4ec5-4aa2-ad2d-ee25c601ba93", MAC: 0a:58:0a:80:00:44, IPs: [10.128.0.68/23] I0327 23:36:50.390988 3881 cni.go:424] [default-tenant/my-integration-test-iaxd-hdhqk-verify-pod a3c4223f5aa9ae17725dc24ba4c484f855e73935ac3903c304a9c8e200b91e33 network default NAD default] ADD finished CNI request [default-tenant/my-integration-test-iaxd-hdhqk-verify-pod a3c4223f5aa9ae17725dc24ba4c484f855e73935ac3903c304a9c8e200b91e33 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:19:df:c9:34:b2\",\"name\":\"a3c4223f5aa9ae1\"},{\"mac\":\"0a:58:0a:80:00:44\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/97643191-8590-4cdf-bd66-bfd2a1bb7f28\"}],\"ips\":[{\"address\":\"10.128.0.68/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 23:36:52.103213 3881 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=9 I0327 23:37:06.203590 3881 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10