+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-98.ec2.internal ++ [[ -n ip-10-0-1-98.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-98.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0514 23:14:09.888869159 - disable conntrack on geneve port' I0514 23:14:09.888869159 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0514 23:14:09.905041085 - starting ovnkube-node' I0514 23:14:09.905041085 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-98.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-98.ec2.internal --init-node ip-10-0-1-98.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-98.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0514 23:14:09.946974 3702 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0514 23:14:09.947043 3702 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-bdaf609cfa.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0514 23:14:09.948585 3702 kube.go:431] Waiting for certificate I0514 23:14:09.948611 3702 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0514 23:14:09.948643 3702 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0514 23:14:09.949518 3702 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0514 23:14:09.985370 3702 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0514 23:14:09.985399 3702 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0514 23:14:09.989092 3702 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0514 23:14:09.989199 3702 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-gb7sj" I0514 23:14:09.993997 3702 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-gb7sj" I0514 23:14:09.994081 3702 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0514 23:14:10.949659 3702 kube.go:438] Certificate found I0514 23:14:10.950344 3702 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0514 23:14:10.951431 3702 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0514 23:14:11.013649 3702 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0514 23:14:11.032683 3702 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0514 23:14:11.032752 3702 factory.go:561] Starting watch factory I0514 23:14:11.032843 3702 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.032853 3702 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.032859 3702 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.032871 3702 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.032874 3702 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.032880 3702 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.032880 3702 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.032899 3702 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.032852 3702 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.032975 3702 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.033082 3702 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.033141 3702 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.034279 3702 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0514 23:14:11.034487 3702 server.go:101] MetricServer registers OVS metrics I0514 23:14:11.034969 3702 server.go:105] MetricServer registers OVN DB metrics I0514 23:14:11.037151 3702 controller_manager.go:370] Starting the ovnkube controller I0514 23:14:11.037160 3702 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-98.ec2.internal I0514 23:14:11.037217 3702 controller_manager.go:395] NBDB zone sync took: 50.273µs I0514 23:14:11.037226 3702 factory.go:561] Starting watch factory I0514 23:14:11.039536 3702 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0514 23:14:11.039568 3702 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0514 23:14:11.043659 3702 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-05-15 23:09:09 +0000 UTC" deadline="2026-05-15 19:24:34.090926438 +0000 UTC" I0514 23:14:11.043676 3702 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h10m23.047256673s" I0514 23:14:11.052036 3702 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.053309 3702 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0514 23:14:11.053328 3702 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0514 23:14:11.057030 3702 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0514 23:14:11.057064 3702 server.go:109] MetricServer registers OVN Controller metrics I0514 23:14:11.057545 3702 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.059907 3702 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.061443 3702 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.063125 3702 server.go:113] MetricServer registers OVN Northd metrics I0514 23:14:11.063461 3702 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.066003 3702 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:11.068743 3702 metrics.go:519] OVN Metrics Server starts to run ... I0514 23:14:11.138783 3702 factory.go:1977] *v1.Pod informer cache synced successfully I0514 23:14:11.138809 3702 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0514 23:14:11.138813 3702 factory.go:1977] *v1.Namespace informer cache synced successfully I0514 23:14:11.138817 3702 factory.go:1977] *v1.Node informer cache synced successfully I0514 23:14:11.138820 3702 factory.go:1977] *v1.Service informer cache synced successfully I0514 23:14:11.138823 3702 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0514 23:14:11.138932 3702 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:11.138962 3702 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:11.139397 3702 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:11.139414 3702 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:11.141556 3702 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:11.141579 3702 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:11.143482 3702 factory.go:1977] *v1.Node informer cache synced successfully I0514 23:14:11.143500 3702 factory.go:1977] *v1.Service informer cache synced successfully I0514 23:14:11.143505 3702 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0514 23:14:11.143510 3702 factory.go:1977] *v1.Pod informer cache synced successfully I0514 23:14:11.143515 3702 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0514 23:14:11.143518 3702 factory.go:1977] *v1.Namespace informer cache synced successfully I0514 23:14:11.144634 3702 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0514 23:14:11.144641 3702 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0514 23:14:11.144695 3702 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.144707 3702 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.146653 3702 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.148678 3702 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0514 23:14:11.148690 3702 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0514 23:14:11.150203 3702 factory.go:1977] *v1.EgressIP informer cache synced successfully I0514 23:14:11.150262 3702 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.150275 3702 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.152360 3702 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.154024 3702 factory.go:1977] *v1.EgressIP informer cache synced successfully I0514 23:14:11.156226 3702 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0514 23:14:11.156282 3702 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.156293 3702 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.158511 3702 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.159468 3702 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0514 23:14:11.162241 3702 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0514 23:14:11.162292 3702 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.162302 3702 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.164311 3702 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.165326 3702 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0514 23:14:11.167991 3702 factory.go:1977] *v1.EgressService informer cache synced successfully I0514 23:14:11.168041 3702 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.168051 3702 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.170144 3702 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.171045 3702 factory.go:1977] *v1.EgressService informer cache synced successfully I0514 23:14:11.173796 3702 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0514 23:14:11.173985 3702 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0514 23:14:11.174005 3702 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0514 23:14:11.176077 3702 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0514 23:14:11.176273 3702 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0514 23:14:11.178913 3702 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0514 23:14:11.178970 3702 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.178982 3702 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.179042 3702 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.179062 3702 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.180932 3702 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.180983 3702 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:11.181095 3702 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0514 23:14:11.184800 3702 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0514 23:14:11.184810 3702 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0514 23:14:11.184817 3702 factory.go:680] Watch Factory start up complete, took: 152.068137ms I0514 23:14:11.184898 3702 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0514 23:14:11.186200 3702 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0514 23:14:11.186209 3702 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0514 23:14:11.186216 3702 factory.go:680] Watch Factory start up complete, took: 148.988945ms I0514 23:14:11.186223 3702 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-1-98.ec2.internal" zone I0514 23:14:11.227575 3702 default_node_network_controller.go:710] Initializing the default node network controller I0514 23:14:11.302903 3702 udn_isolation.go:101] Starting UDN host isolation manager I0514 23:14:11.309822 3702 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0514 23:14:11.365099 3702 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0514 23:14:11.365191 3702 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0514 23:14:11.365209 3702 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0514 23:14:11.397390 3702 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0514 23:14:11.397465 3702 default_node_network_controller.go:809] Node ip-10-0-1-98.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0514 23:14:11.402970 3702 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.98"] k8s.ovn.org/zone-name:ip-10-0-1-98.ec2.internal] on node ip-10-0-1-98.ec2.internal I0514 23:14:11.423750 3702 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0514 23:14:11.430046 3702 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0514 23:14:11.430158 3702 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0514 23:14:11.430296 3702 gateway_init.go:263] Preparing Gateway I0514 23:14:11.430308 3702 gateway_shared_intf.go:1687] Creating new gateway I0514 23:14:11.436414 3702 controller_manager.go:429] Waiting for node in zone sync took: 250.180562ms I0514 23:14:11.442764 3702 controller_manager.go:611] Switching to transit router for layer2 networks I0514 23:14:11.442777 3702 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-98.ec2.internal I0514 23:14:11.458709 3702 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0514 23:14:11.465397 3702 controller_manager.go:290] SCTP support detected in OVN I0514 23:14:11.466291 3702 services_controller.go:69] Creating services controller for network=default I0514 23:14:11.466390 3702 udn_enabled_svc.go:80] Waiting for service informer to sync I0514 23:14:11.466493 3702 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0514 23:14:11.466532 3702 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0514 23:14:11.466552 3702 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0514 23:14:11.466564 3702 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0514 23:14:11.466830 3702 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0514 23:14:11.466922 3702 udn_enabled_svc.go:95] Performing full resync I0514 23:14:11.466942 3702 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0514 23:14:11.466970 3702 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 55.497µs I0514 23:14:11.466986 3702 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 34.679µs I0514 23:14:11.466995 3702 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 52.677µs I0514 23:14:11.467002 3702 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0514 23:14:11.467010 3702 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0514 23:14:11.467022 3702 default_network_controller.go:325] Starting the default network controller I0514 23:14:11.467143 3702 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 17.149µs I0514 23:14:11.467323 3702 udn_enabled_svc.go:99] Waiting for handler to sync I0514 23:14:11.467336 3702 udn_enabled_svc.go:104] Starting worker I0514 23:14:11.477192 3702 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0514 23:14:11.477267 3702 repair.go:33] Syncing exgw routes took 59.54µs I0514 23:14:11.477293 3702 default_network_controller.go:414] Starting all the Watchers... I0514 23:14:11.477459 3702 namespace.go:98] [openshift-image-registry] adding namespace I0514 23:14:11.477466 3702 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0514 23:14:11.477486 3702 namespace.go:98] [openshift-operators] adding namespace I0514 23:14:11.477499 3702 namespace.go:98] [openshift-apiserver-operator] adding namespace I0514 23:14:11.477504 3702 namespace.go:98] [kube-system] adding namespace I0514 23:14:11.477521 3702 namespace.go:98] [kube-public] adding namespace I0514 23:14:11.477534 3702 namespace.go:98] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] adding namespace I0514 23:14:11.477542 3702 namespace.go:98] [openshift-dns] adding namespace I0514 23:14:11.477537 3702 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0514 23:14:11.477543 3702 namespace.go:98] [openshift-backplane-csm] adding namespace I0514 23:14:11.477556 3702 namespace.go:98] [openshift-etcd] adding namespace I0514 23:14:11.477564 3702 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0514 23:14:11.477568 3702 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0514 23:14:11.477572 3702 namespace.go:98] [openshift-must-gather-operator] adding namespace I0514 23:14:11.477599 3702 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0514 23:14:11.478252 3702 namespace.go:102] [openshift-image-registry] adding namespace took 780.027µs I0514 23:14:11.478269 3702 namespace.go:98] [openshift-ingress-canary] adding namespace I0514 23:14:11.478917 3702 namespace.go:102] [openshift-ingress-canary] adding namespace took 642.812µs I0514 23:14:11.478936 3702 namespace.go:98] [openshift-host-network] adding namespace I0514 23:14:11.479777 3702 namespace.go:102] [openshift-host-network] adding namespace took 832.35µs I0514 23:14:11.479800 3702 namespace.go:98] [openshift-console] adding namespace I0514 23:14:11.480626 3702 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 3.147784ms I0514 23:14:11.480687 3702 namespace.go:98] [openshift-ingress-operator] adding namespace I0514 23:14:11.481297 3702 namespace.go:102] [openshift-operators] adding namespace took 3.798125ms I0514 23:14:11.481318 3702 namespace.go:98] [openshift-logging] adding namespace W0514 23:14:11.481970 3702 base_network_controller_namespace.go:410] Failed to get IPs for pod kube-system/global-pull-secret-syncer-49qv2: pod kube-system/global-pull-secret-syncer-49qv2: no pod IPs found I0514 23:14:11.481975 3702 namespace.go:102] [openshift-apiserver-operator] adding namespace took 4.469343ms W0514 23:14:11.481980 3702 base_network_controller_namespace.go:410] Failed to get IPs for pod kube-system/global-pull-secret-syncer-p6k99: pod kube-system/global-pull-secret-syncer-p6k99: no pod IPs found I0514 23:14:11.481985 3702 namespace.go:98] [openshift-infra] adding namespace I0514 23:14:11.482541 3702 namespace.go:102] [kube-system] adding namespace took 5.030836ms I0514 23:14:11.482555 3702 namespace.go:98] [openshift-network-node-identity] adding namespace I0514 23:14:11.483051 3702 namespace.go:102] [kube-public] adding namespace took 5.51922ms I0514 23:14:11.483065 3702 namespace.go:98] [openshift-console-operator] adding namespace I0514 23:14:11.483547 3702 namespace.go:102] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] adding namespace took 6.00169ms I0514 23:14:11.483562 3702 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0514 23:14:11.484494 3702 namespace.go:102] [openshift-dns] adding namespace took 6.94293ms I0514 23:14:11.484509 3702 namespace.go:98] [openshift-machine-config-operator] adding namespace I0514 23:14:11.485097 3702 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.539023ms I0514 23:14:11.485113 3702 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0514 23:14:11.485754 3702 namespace.go:102] [openshift-etcd] adding namespace took 8.188516ms I0514 23:14:11.485770 3702 namespace.go:98] [openshift-package-operator] adding namespace I0514 23:14:11.486348 3702 namespace.go:102] [openshift-backplane-csm] adding namespace took 8.78344ms I0514 23:14:11.486370 3702 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0514 23:14:11.486925 3702 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 9.354815ms I0514 23:14:11.486943 3702 namespace.go:98] [openshift-operators-redhat] adding namespace I0514 23:14:11.487552 3702 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 9.978206ms I0514 23:14:11.487567 3702 namespace.go:98] [openshift-authentication] adding namespace I0514 23:14:11.488141 3702 namespace.go:102] [openshift-must-gather-operator] adding namespace took 10.563506ms I0514 23:14:11.488158 3702 namespace.go:98] [openshift-backplane-cse] adding namespace I0514 23:14:11.488699 3702 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 11.092107ms I0514 23:14:11.488713 3702 namespace.go:98] [openshift-network-console] adding namespace I0514 23:14:11.489196 3702 namespace.go:102] [openshift-console] adding namespace took 9.389928ms I0514 23:14:11.489209 3702 namespace.go:98] [openshift] adding namespace I0514 23:14:11.489690 3702 namespace.go:102] [openshift-ingress-operator] adding namespace took 8.991471ms I0514 23:14:11.489703 3702 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0514 23:14:11.490199 3702 namespace.go:102] [openshift-logging] adding namespace took 8.87576ms I0514 23:14:11.490211 3702 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0514 23:14:11.490719 3702 namespace.go:102] [openshift-infra] adding namespace took 8.728544ms I0514 23:14:11.490731 3702 namespace.go:98] [openshift-customer-monitoring] adding namespace I0514 23:14:11.491195 3702 namespace.go:102] [openshift-network-node-identity] adding namespace took 8.634887ms I0514 23:14:11.491207 3702 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0514 23:14:11.491710 3702 namespace.go:102] [openshift-console-operator] adding namespace took 8.640803ms I0514 23:14:11.491721 3702 namespace.go:98] [openshift-multus] adding namespace I0514 23:14:11.492201 3702 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 8.630424ms I0514 23:14:11.492223 3702 namespace.go:98] [kube-node-lease] adding namespace I0514 23:14:11.492662 3702 namespace.go:102] [openshift-machine-config-operator] adding namespace took 8.148497ms I0514 23:14:11.492673 3702 namespace.go:98] [openshift-machine-api] adding namespace I0514 23:14:11.493124 3702 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 8.005212ms I0514 23:14:11.493138 3702 namespace.go:98] [openshift-backplane-tam] adding namespace I0514 23:14:11.493594 3702 namespace.go:102] [openshift-package-operator] adding namespace took 7.819053ms I0514 23:14:11.493607 3702 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0514 23:14:11.494037 3702 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.660591ms I0514 23:14:11.494052 3702 namespace.go:98] [openshift-config-managed] adding namespace I0514 23:14:11.494452 3702 namespace.go:102] [openshift-operators-redhat] adding namespace took 7.503373ms I0514 23:14:11.494467 3702 namespace.go:98] [openshift-node] adding namespace I0514 23:14:11.494837 3702 namespace.go:102] [openshift-authentication] adding namespace took 7.264909ms I0514 23:14:11.494851 3702 namespace.go:98] [openshift-insights] adding namespace I0514 23:14:11.495193 3702 namespace.go:102] [openshift-backplane-cse] adding namespace took 7.031338ms I0514 23:14:11.495203 3702 namespace.go:98] [openshift-ingress] adding namespace I0514 23:14:11.495645 3702 namespace.go:102] [openshift-network-console] adding namespace took 6.927142ms I0514 23:14:11.495656 3702 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0514 23:14:11.496071 3702 namespace.go:102] [openshift] adding namespace took 6.858589ms I0514 23:14:11.496081 3702 namespace.go:98] [openshift-kube-apiserver] adding namespace I0514 23:14:11.496479 3702 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.771068ms I0514 23:14:11.496491 3702 namespace.go:98] [openshift-kube-scheduler] adding namespace I0514 23:14:11.496874 3702 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.657772ms I0514 23:14:11.496891 3702 namespace.go:98] [openshift-backplane-ai-agent] adding namespace I0514 23:14:11.497412 3702 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.676101ms I0514 23:14:11.497426 3702 namespace.go:98] [openshift-controller-manager] adding namespace W0514 23:14:11.497912 3702 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-j7h6h: pod openshift-multus/network-metrics-daemon-j7h6h: no pod IPs found W0514 23:14:11.497924 3702 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-mhnwf: pod openshift-multus/network-metrics-daemon-mhnwf: no pod IPs found I0514 23:14:11.497956 3702 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.74198ms I0514 23:14:11.497974 3702 namespace.go:98] [default] adding namespace I0514 23:14:11.498552 3702 namespace.go:102] [openshift-multus] adding namespace took 6.824297ms I0514 23:14:11.498568 3702 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0514 23:14:11.499102 3702 namespace.go:102] [kube-node-lease] adding namespace took 6.870293ms I0514 23:14:11.499124 3702 namespace.go:98] [openshift-backplane-mobb] adding namespace I0514 23:14:11.499550 3702 namespace.go:102] [openshift-machine-api] adding namespace took 6.870484ms I0514 23:14:11.499567 3702 namespace.go:98] [openshift-service-ca-operator] adding namespace I0514 23:14:11.500074 3702 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.931229ms I0514 23:14:11.500085 3702 namespace.go:98] [openshift-backplane] adding namespace I0514 23:14:11.500555 3702 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.94194ms I0514 23:14:11.500570 3702 namespace.go:98] [openshift-config] adding namespace I0514 23:14:11.500955 3702 namespace.go:102] [openshift-config-managed] adding namespace took 6.896227ms I0514 23:14:11.500971 3702 namespace.go:98] [openshift-network-diagnostics] adding namespace I0514 23:14:11.501362 3702 namespace.go:102] [openshift-node] adding namespace took 6.889901ms I0514 23:14:11.501376 3702 namespace.go:98] [openshift-marketplace] adding namespace I0514 23:14:11.501751 3702 namespace.go:102] [openshift-insights] adding namespace took 6.89399ms I0514 23:14:11.501764 3702 namespace.go:98] [openshift-config-operator] adding namespace I0514 23:14:11.502133 3702 namespace.go:102] [openshift-ingress] adding namespace took 6.924808ms I0514 23:14:11.502145 3702 namespace.go:98] [openshift-apiserver] adding namespace I0514 23:14:11.502574 3702 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.911935ms I0514 23:14:11.502586 3702 namespace.go:98] [dedicated-admin] adding namespace I0514 23:14:11.502957 3702 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.868345ms I0514 23:14:11.503539 3702 namespace.go:102] [openshift-kube-scheduler] adding namespace took 7.041165ms I0514 23:14:11.503557 3702 namespace.go:98] [openshift-backplane-srep] adding namespace I0514 23:14:11.503946 3702 namespace.go:102] [openshift-backplane-ai-agent] adding namespace took 7.047724ms I0514 23:14:11.503963 3702 namespace.go:98] [openshift-dns-operator] adding namespace I0514 23:14:11.504460 3702 namespace.go:102] [openshift-controller-manager] adding namespace took 7.02853ms I0514 23:14:11.504475 3702 namespace.go:98] [openshift-cluster-version] adding namespace I0514 23:14:11.504908 3702 namespace.go:102] [default] adding namespace took 6.926872ms I0514 23:14:11.504923 3702 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0514 23:14:11.505359 3702 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.783087ms I0514 23:14:11.505376 3702 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0514 23:14:11.505740 3702 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.609963ms I0514 23:14:11.505753 3702 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0514 23:14:11.506234 3702 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.658413ms I0514 23:14:11.506257 3702 namespace.go:98] [openshift-backplane-cee] adding namespace I0514 23:14:11.506781 3702 namespace.go:102] [openshift-backplane] adding namespace took 6.688888ms I0514 23:14:11.506802 3702 namespace.go:98] [open-cluster-management-agent-addon] adding namespace W0514 23:14:11.507388 3702 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-snbfj: pod openshift-network-diagnostics/network-check-target-snbfj: no pod IPs found W0514 23:14:11.507407 3702 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-x87mq: pod openshift-network-diagnostics/network-check-target-x87mq: no pod IPs found I0514 23:14:11.507414 3702 namespace.go:102] [openshift-config] adding namespace took 6.836844ms I0514 23:14:11.507428 3702 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0514 23:14:11.507960 3702 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.980301ms I0514 23:14:11.507984 3702 namespace.go:98] [openshift-network-operator] adding namespace I0514 23:14:11.508621 3702 namespace.go:102] [openshift-marketplace] adding namespace took 7.237393ms I0514 23:14:11.508638 3702 namespace.go:98] [openshift-monitoring] adding namespace I0514 23:14:11.509159 3702 namespace.go:102] [openshift-config-operator] adding namespace took 7.387792ms I0514 23:14:11.509187 3702 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0514 23:14:11.509768 3702 namespace.go:102] [openshift-apiserver] adding namespace took 7.617241ms I0514 23:14:11.509783 3702 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0514 23:14:11.510281 3702 namespace.go:102] [dedicated-admin] adding namespace took 7.689005ms I0514 23:14:11.510293 3702 namespace.go:98] [openshift-authentication-operator] adding namespace I0514 23:14:11.510745 3702 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.181367ms I0514 23:14:11.510763 3702 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0514 23:14:11.511304 3702 namespace.go:102] [openshift-dns-operator] adding namespace took 7.33408ms I0514 23:14:11.511320 3702 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0514 23:14:11.511798 3702 namespace.go:102] [openshift-cluster-version] adding namespace took 7.317906ms I0514 23:14:11.512200 3702 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.272793ms I0514 23:14:11.512217 3702 namespace.go:98] [openshift-route-controller-manager] adding namespace I0514 23:14:11.512750 3702 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.368217ms I0514 23:14:11.513252 3702 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 7.494843ms I0514 23:14:11.513816 3702 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.554362ms I0514 23:14:11.514304 3702 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.49648ms I0514 23:14:11.514811 3702 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 7.377865ms I0514 23:14:11.515355 3702 namespace.go:102] [openshift-network-operator] adding namespace took 7.36551ms I0514 23:14:11.515367 3702 namespace.go:98] [openshift-console-user-settings] adding namespace I0514 23:14:11.515808 3702 namespace.go:102] [openshift-monitoring] adding namespace took 7.165384ms I0514 23:14:11.516248 3702 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.055683ms I0514 23:14:11.516732 3702 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.944017ms I0514 23:14:11.517260 3702 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.963189ms I0514 23:14:11.517828 3702 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 7.060622ms I0514 23:14:11.518398 3702 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.070021ms I0514 23:14:11.518816 3702 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.592878ms I0514 23:14:11.519301 3702 namespace.go:102] [openshift-console-user-settings] adding namespace took 3.92863ms W0514 23:14:11.519378 3702 master.go:458] Unable to parse local node ip-10-0-1-98.ec2.internal chassis-id annotation. Chassis may be removed during sync W0514 23:14:11.519386 3702 master.go:471] Unable to parse remote node ip-10-0-2-152.ec2.internal chassis-id annotation. Chassis may be removed during sync I0514 23:14:11.519391 3702 master.go:481] Removing stale chassis with ID/Name: 4ef625d1-0092-48be-add3-31ccb23efd1a, hostname: ip-10-0-1-98.ec2.internal I0514 23:14:11.519812 3702 master.go:548] Adding or Updating local node "ip-10-0-1-98.ec2.internal" for network "default" I0514 23:14:11.519913 3702 obj_retry.go:578] Failed to create *v1.Node ip-10-0-2-152.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-152.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-152.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-152.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-152.ec2.internal I0514 23:14:11.522970 3702 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-98.ec2.internal for the network default I0514 23:14:11.523246 3702 zone_ic_handler.go:195] Time taken to create transit switch: 264.991µs I0514 23:14:11.523824 3702 default_network_controller.go:823] Node add failed for ip-10-0-1-98.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-98.ec2.internal, error getting gateway config for node ip-10-0-1-98.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-98.ec2.internal", failed to update chassis to local for local node ip-10-0-1-98.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-98.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-98.ec2.internal] E0514 23:14:11.523864 3702 obj_retry.go:575] Failed to create *v1.Node ip-10-0-1-98.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-98.ec2.internal, error getting gateway config for node ip-10-0-1-98.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-98.ec2.internal", failed to update chassis to local for local node ip-10-0-1-98.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-98.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-98.ec2.internal] I0514 23:14:11.523901 3702 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0514 23:14:11.523921 3702 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0514 23:14:11.523936 3702 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0514 23:14:11.523950 3702 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-152.ec2.internal in network "default" I0514 23:14:11.523932 3702 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-1-98.ec2.internal", UID:"c7b1ab1b-06af-492c-9600-b9809841907e", APIVersion:"v1", ResourceVersion:"8398", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-98.ec2.internal, error getting gateway config for node ip-10-0-1-98.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-98.ec2.internal", failed to update chassis to local for local node ip-10-0-1-98.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-98.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-98.ec2.internal] I0514 23:14:11.523987 3702 node_tracker.go:235] Node ip-10-0-2-152.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-152.ec2.internal" I0514 23:14:11.524004 3702 node_tracker.go:169] Node ip-10-0-2-152.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:11.524021 3702 services_controller.go:574] Full service sync requested for network=default I0514 23:14:11.524034 3702 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-98.ec2.internal in network "default" I0514 23:14:11.524048 3702 node_tracker.go:235] Node ip-10-0-1-98.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-98.ec2.internal" I0514 23:14:11.524060 3702 node_tracker.go:169] Node ip-10-0-1-98.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:11.524066 3702 services_controller.go:574] Full service sync requested for network=default I0514 23:14:11.554058 3702 iptables.go:108] Creating table: filter chain: FORWARD I0514 23:14:11.560344 3702 iptables.go:108] Creating table: filter chain: OUTPUT I0514 23:14:11.566486 3702 gateway_shared_intf.go:1816] Gateway Creation Complete I0514 23:14:11.566503 3702 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-98.ec2.internal","mac-address":"0a:ff:d9:f0:a2:f5","ip-addresses":["10.0.1.98/24"],"ip-address":"10.0.1.98/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:4ef625d1-0092-48be-add3-31ccb23efd1a k8s.ovn.org/node-encap-ips:["10.0.1.98"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.98/24"} k8s.ovn.org/zone-name:ip-10-0-1-98.ec2.internal] on node ip-10-0-1-98.ec2.internal I0514 23:14:11.586284 3702 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-98.ec2.internal in network "default" I0514 23:14:11.586298 3702 master.go:548] Adding or Updating local node "ip-10-0-1-98.ec2.internal" for network "default" I0514 23:14:11.586345 3702 node_tracker.go:169] Node ip-10-0-1-98.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:11.586359 3702 services_controller.go:574] Full service sync requested for network=default I0514 23:14:11.590144 3702 default_node_network_controller.go:519] Node connection status = connected W0514 23:14:11.591073 3702 gateway.go:207] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-49qv2: pod kube-system/global-pull-secret-syncer-49qv2: no pod IPs found W0514 23:14:11.591092 3702 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-mhnwf: pod openshift-multus/network-metrics-daemon-mhnwf: no pod IPs found W0514 23:14:11.591100 3702 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-x87mq: pod openshift-network-diagnostics/network-check-target-x87mq: no pod IPs found I0514 23:14:11.592310 3702 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-98.ec2.internal for the network default I0514 23:14:11.592532 3702 zone_ic_handler.go:195] Time taken to create transit switch: 198.656µs I0514 23:14:11.624733 3702 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0514 23:14:11.624755 3702 services_controller.go:208] Setting up event handlers for services for network=default I0514 23:14:11.624811 3702 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0514 23:14:11.624852 3702 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0514 23:14:11.624867 3702 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0514 23:14:11.725480 3702 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0514 23:14:11.725502 3702 repair.go:60] Starting repairing loop for services I0514 23:14:11.725595 3702 repair.go:131] Deleted 0 stale service LBs I0514 23:14:11.725616 3702 repair.go:137] Deleted 0 stale Chassis Template Vars I0514 23:14:11.725632 3702 repair.go:62] Finished repairing loop for services: 131.748µs I0514 23:14:11.725657 3702 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0514 23:14:11.725666 3702 services_controller.go:254] Starting workers for network=default I0514 23:14:11.725977 3702 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-mhnwf] creating logical port openshift-multus_network-metrics-daemon-mhnwf for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:11.725990 3702 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-snbfj, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-snbfj: suppressed error logged: pod openshift-network-diagnostics/network-check-target-snbfj: no pod IPs found I0514 23:14:11.726004 3702 obj_retry.go:578] Failed to create *v1.Pod kube-system/global-pull-secret-syncer-p6k99, error: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-p6k99: suppressed error logged: pod kube-system/global-pull-secret-syncer-p6k99: no pod IPs found I0514 23:14:11.726011 3702 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-49qv2] creating logical port kube-system_global-pull-secret-syncer-49qv2 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:11.726029 3702 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-j7h6h, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-j7h6h: suppressed error logged: pod openshift-multus/network-metrics-daemon-j7h6h: no pod IPs found I0514 23:14:11.725979 3702 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-x87mq] creating logical port openshift-network-diagnostics_network-check-target-x87mq for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:11.726155 3702 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-mhnwf I0514 23:14:11.726163 3702 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-49qv2 I0514 23:14:11.726203 3702 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-x87mq I0514 23:14:11.741327 3702 pod.go:62] [kube-system/global-pull-secret-syncer-49qv2] pod update took 15.257851ms I0514 23:14:11.741344 3702 base_network_controller_pods.go:951] [default/kube-system/global-pull-secret-syncer-49qv2] addLogicalPort annotation time took 15.276611ms I0514 23:14:11.742310 3702 pods.go:271] [kube-system/global-pull-secret-syncer-49qv2] addLogicalPort took 16.303448ms, libovsdb time 692.553µs I0514 23:14:11.743850 3702 pod.go:62] [openshift-multus/network-metrics-daemon-mhnwf] pod update took 17.802349ms I0514 23:14:11.743869 3702 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-mhnwf] addLogicalPort annotation time took 17.822416ms I0514 23:14:11.744732 3702 pods.go:271] [openshift-multus/network-metrics-daemon-mhnwf] addLogicalPort took 18.764335ms, libovsdb time 601.669µs I0514 23:14:11.745613 3702 pod.go:62] [openshift-network-diagnostics/network-check-target-x87mq] pod update took 19.503428ms I0514 23:14:11.745634 3702 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-x87mq] addLogicalPort annotation time took 19.528373ms I0514 23:14:11.746493 3702 pods.go:271] [openshift-network-diagnostics/network-check-target-x87mq] addLogicalPort took 20.525626ms, libovsdb time 598.775µs I0514 23:14:11.746858 3702 admin_network_policy_controller.go:227] Starting controller default-network-controller I0514 23:14:11.746881 3702 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0514 23:14:11.746902 3702 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0514 23:14:11.746910 3702 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0514 23:14:11.747096 3702 repair.go:29] Repairing admin network policies took 179.283µs I0514 23:14:11.747218 3702 repair.go:92] Repairing baseline admin network policies took 112.961µs I0514 23:14:11.747518 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0514 23:14:11.747535 3702 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0514 23:14:11.749532 3702 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0514 23:14:11.749561 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0514 23:14:11.749571 3702 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0514 23:14:11.751357 3702 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0514 23:14:11.751379 3702 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0514 23:14:11.751390 3702 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0514 23:14:11.753526 3702 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0514 23:14:11.753555 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0514 23:14:11.753563 3702 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0514 23:14:11.753835 3702 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0514 23:14:11.753853 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0514 23:14:11.754262 3702 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0514 23:14:11.754275 3702 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0514 23:14:11.756202 3702 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0514 23:14:11.756228 3702 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0514 23:14:11.756237 3702 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0514 23:14:11.757254 3702 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0514 23:14:11.757273 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0514 23:14:11.757281 3702 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0514 23:14:11.757743 3702 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0514 23:14:11.757762 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0514 23:14:11.757774 3702 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0514 23:14:11.758220 3702 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0514 23:14:11.758237 3702 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0514 23:14:11.758246 3702 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0514 23:14:11.760167 3702 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0514 23:14:11.760202 3702 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0514 23:14:11.760210 3702 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0514 23:14:11.761476 3702 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0514 23:14:11.761493 3702 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0514 23:14:11.761502 3702 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0514 23:14:11.762381 3702 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0514 23:14:11.762399 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0514 23:14:11.762405 3702 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0514 23:14:11.762700 3702 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0514 23:14:11.762717 3702 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0514 23:14:11.762725 3702 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0514 23:14:11.763336 3702 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0514 23:14:11.763353 3702 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0514 23:14:11.763364 3702 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0514 23:14:11.764942 3702 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0514 23:14:11.764965 3702 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0514 23:14:11.764974 3702 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0514 23:14:11.765307 3702 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0514 23:14:11.765326 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0514 23:14:11.765334 3702 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0514 23:14:11.765818 3702 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0514 23:14:11.765833 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0514 23:14:11.765840 3702 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0514 23:14:11.766384 3702 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0514 23:14:11.766402 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0514 23:14:11.766409 3702 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0514 23:14:11.766953 3702 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0514 23:14:11.766969 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0514 23:14:11.766988 3702 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0514 23:14:11.767654 3702 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0514 23:14:11.767673 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0514 23:14:11.767680 3702 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0514 23:14:11.768829 3702 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0514 23:14:11.768846 3702 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0514 23:14:11.768856 3702 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0514 23:14:11.769873 3702 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0514 23:14:11.769891 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0514 23:14:11.769899 3702 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0514 23:14:11.770548 3702 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0514 23:14:11.770564 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0514 23:14:11.770571 3702 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0514 23:14:11.771044 3702 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0514 23:14:11.771060 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0514 23:14:11.771067 3702 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0514 23:14:11.771523 3702 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0514 23:14:11.771539 3702 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0514 23:14:11.771547 3702 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0514 23:14:11.772338 3702 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0514 23:14:11.772355 3702 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0514 23:14:11.772361 3702 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0514 23:14:11.772834 3702 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0514 23:14:11.772851 3702 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0514 23:14:11.772858 3702 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0514 23:14:11.773350 3702 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0514 23:14:11.773366 3702 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0514 23:14:11.773376 3702 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0514 23:14:11.774328 3702 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0514 23:14:11.774347 3702 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0514 23:14:11.774357 3702 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0514 23:14:11.775955 3702 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0514 23:14:11.780343 3702 egressfirewall.go:374] Starting EgressFirewall controller I0514 23:14:11.780369 3702 controller.go:133] Adding controller egress-firewall-controller event handlers I0514 23:14:11.780379 3702 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0514 23:14:11.780391 3702 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0514 23:14:11.780401 3702 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0514 23:14:11.780420 3702 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0514 23:14:11.780433 3702 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0514 23:14:11.780676 3702 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0514 23:14:11.780688 3702 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0514 23:14:11.780693 3702 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0514 23:14:11.780701 3702 egressqos.go:193] Setting up event handlers for EgressQoS I0514 23:14:11.780708 3702 egressfirewall.go:723] Syncing node "ip-10-0-1-98.ec2.internal" for egress firewall I0514 23:14:11.780715 3702 egressfirewall.go:723] Syncing node "ip-10-0-2-152.ec2.internal" for egress firewall I0514 23:14:11.780799 3702 egressqos.go:245] Starting EgressQoS Controller I0514 23:14:11.780808 3702 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0514 23:14:11.780813 3702 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0514 23:14:11.780819 3702 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0514 23:14:11.780823 3702 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0514 23:14:11.780828 3702 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0514 23:14:11.780832 3702 shared_informer.go:356] "Caches are synced" controller="egressqos" I0514 23:14:11.780836 3702 egressqos.go:259] Repairing EgressQoSes I0514 23:14:11.780840 3702 egressqos.go:399] Starting repairing loop for egressqos I0514 23:14:11.780918 3702 egressqos.go:401] Finished repairing loop for egressqos: 78.008µs I0514 23:14:11.780936 3702 egressservice_zone.go:132] Setting up event handlers for Egress Services I0514 23:14:11.780955 3702 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-98.ec2.internal I0514 23:14:11.780997 3702 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-98.ec2.internal : 42.577µs I0514 23:14:11.781109 3702 egressservice_zone.go:210] Starting Egress Services Controller I0514 23:14:11.781121 3702 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0514 23:14:11.781128 3702 shared_informer.go:356] "Caches are synced" controller="egressservices" I0514 23:14:11.781135 3702 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0514 23:14:11.781142 3702 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0514 23:14:11.781150 3702 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0514 23:14:11.781158 3702 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0514 23:14:11.781166 3702 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0514 23:14:11.781187 3702 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0514 23:14:11.781194 3702 egressservice_zone.go:228] Repairing Egress Services I0514 23:14:11.781492 3702 master_controller.go:88] Starting Admin Policy Based Route Controller I0514 23:14:11.781502 3702 external_controller.go:278] Starting Admin Policy Based Route Controller I0514 23:14:11.781567 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-98.ec2.internal I0514 23:14:11.781579 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-98.ec2.internal: 14.456µs I0514 23:14:11.781592 3702 default_network_controller.go:585] Completing all the Watchers took 304.290457ms I0514 23:14:11.781608 3702 default_network_controller.go:589] Starting unidling controllers I0514 23:14:11.781612 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-152.ec2.internal I0514 23:14:11.781616 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-152.ec2.internal: 29.471µs I0514 23:14:11.781620 3702 unidle.go:46] Registering OVN SB ControllerEvent handler I0514 23:14:11.781626 3702 unidle.go:63] Populating Initial ContollerEvent events I0514 23:14:11.781640 3702 unidle.go:79] Setting up event handlers for services I0514 23:14:11.781692 3702 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0514 23:14:12.090868 3702 default_node_network_controller.go:519] Node connection status = connected I0514 23:14:12.590447 3702 default_node_network_controller.go:519] Node connection status = connected I0514 23:14:13.090596 3702 default_node_network_controller.go:519] Node connection status = connected I0514 23:14:13.590432 3702 default_node_network_controller.go:519] Node connection status = connected I0514 23:14:13.907516 3702 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0514 23:14:14.091121 3702 default_node_network_controller.go:519] Node connection status = connected I0514 23:14:14.108891 3702 bridgeconfig.go:537] Gateway is ready I0514 23:14:14.108910 3702 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0514 23:14:14.120621 3702 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.98. Updating OVN encap IP. I0514 23:14:14.125813 3702 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.98 - it is already configured I0514 23:14:14.125828 3702 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.98/24:{} 172.20.0.1/32:{}]. Updating annotations. I0514 23:14:14.126054 3702 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.98/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-98.ec2.internal","mac-address":"0a:ff:d9:f0:a2:f5","ip-addresses":["10.0.1.98/24"],"ip-address":"10.0.1.98/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:4ef625d1-0092-48be-add3-31ccb23efd1a k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.98/24"}] on node ip-10-0-1-98.ec2.internal I0514 23:14:14.141893 3702 master.go:548] Adding or Updating local node "ip-10-0-1-98.ec2.internal" for network "default" I0514 23:14:14.141933 3702 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-98.ec2.internal in network "default" I0514 23:14:14.141982 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-98.ec2.internal I0514 23:14:14.141994 3702 node_tracker.go:169] Node ip-10-0-1-98.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:14.141999 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-98.ec2.internal: 23.387µs I0514 23:14:14.142010 3702 services_controller.go:574] Full service sync requested for network=default I0514 23:14:14.142019 3702 egressfirewall.go:723] Syncing node "ip-10-0-1-98.ec2.internal" for egress firewall I0514 23:14:14.142078 3702 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.1.98 I0514 23:14:14.142164 3702 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-98.ec2.internal I0514 23:14:14.147094 3702 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-98.ec2.internal. Updating no re-route policies I0514 23:14:14.147207 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-5bqft I0514 23:14:14.147215 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-bbpmr I0514 23:14:14.147225 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-5bqft I0514 23:14:14.147226 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-tdhgn I0514 23:14:14.147238 3702 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-p6k99 I0514 23:14:14.147238 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-x87mq I0514 23:14:14.147246 3702 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-p6k99 I0514 23:14:14.147248 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-98.ec2.internal I0514 23:14:14.147250 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-82hjk I0514 23:14:14.147258 3702 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-98.ec2.internal I0514 23:14:14.147262 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-82hjk I0514 23:14:14.147262 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-rglvr I0514 23:14:14.147264 3702 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-98.ec2.internal I0514 23:14:14.147265 3702 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-49qv2 I0514 23:14:14.147265 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-gxx2m I0514 23:14:14.147271 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-rglvr I0514 23:14:14.147278 3702 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-49qv2 I0514 23:14:14.147277 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-j7h6h I0514 23:14:14.147282 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-tbncl I0514 23:14:14.147284 3702 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-98.ec2.internal after 0 failed attempt(s) I0514 23:14:14.147291 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-j7h6h I0514 23:14:14.147268 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-98.ec2.internal I0514 23:14:14.147296 3702 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-rglvr after 0 failed attempt(s) I0514 23:14:14.147280 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-gxx2m I0514 23:14:14.147251 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-x87mq I0514 23:14:14.147306 3702 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-98.ec2.internal after 0 failed attempt(s) I0514 23:14:14.147249 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-mhnwf I0514 23:14:14.147312 3702 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-49qv2] creating logical port kube-system_global-pull-secret-syncer-49qv2 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:14.147315 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-mhnwf I0514 23:14:14.147319 3702 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-gxx2m after 0 failed attempt(s) I0514 23:14:14.147226 3702 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-cd94m I0514 23:14:14.147328 3702 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-x87mq] creating logical port openshift-network-diagnostics_network-check-target-x87mq for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:14.147335 3702 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-cd94m I0514 23:14:14.147230 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-bbpmr I0514 23:14:14.147326 3702 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-j7h6h, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-j7h6h: suppressed error logged: pod openshift-multus/network-metrics-daemon-j7h6h: no pod IPs found I0514 23:14:14.147354 3702 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-cd94m after 0 failed attempt(s) I0514 23:14:14.147241 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-tdhgn I0514 23:14:14.147364 3702 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-bbpmr after 0 failed attempt(s) I0514 23:14:14.147370 3702 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-tdhgn after 0 failed attempt(s) I0514 23:14:14.147252 3702 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-5bqft after 0 failed attempt(s) I0514 23:14:14.147206 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-47rwv I0514 23:14:14.147281 3702 obj_retry.go:409] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-p6k99, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-p6k99: suppressed error logged: pod kube-system/global-pull-secret-syncer-p6k99: no pod IPs found I0514 23:14:14.147299 3702 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-82hjk after 0 failed attempt(s) I0514 23:14:14.147379 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-47rwv I0514 23:14:14.147337 3702 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-mhnwf] creating logical port openshift-multus_network-metrics-daemon-mhnwf for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:14.147392 3702 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-47rwv after 0 failed attempt(s) I0514 23:14:14.147295 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-tbncl I0514 23:14:14.147405 3702 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-tbncl after 0 failed attempt(s) I0514 23:14:14.147235 3702 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-snbfj I0514 23:14:14.147420 3702 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-snbfj I0514 23:14:14.147451 3702 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-snbfj, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-snbfj: suppressed error logged: pod openshift-network-diagnostics/network-check-target-snbfj: no pod IPs found I0514 23:14:14.148037 3702 pods.go:271] [openshift-network-diagnostics/network-check-target-x87mq] addLogicalPort took 715.046µs, libovsdb time 238.753µs I0514 23:14:14.148048 3702 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-x87mq after 0 failed attempt(s) I0514 23:14:14.148345 3702 pods.go:271] [openshift-multus/network-metrics-daemon-mhnwf] addLogicalPort took 1.015839ms, libovsdb time 412.858µs I0514 23:14:14.148355 3702 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-mhnwf after 0 failed attempt(s) I0514 23:14:14.148404 3702 pods.go:271] [kube-system/global-pull-secret-syncer-49qv2] addLogicalPort took 1.101517ms, libovsdb time 429.259µs I0514 23:14:14.148416 3702 obj_retry.go:418] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-49qv2 after 0 failed attempt(s) I0514 23:14:14.162951 3702 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0514 23:14:14.174971 3702 iptables.go:108] Creating table: mangle chain: OUTPUT I0514 23:14:14.178155 3702 iptables.go:108] Creating table: nat chain: OUTPUT I0514 23:14:14.224721 3702 iptables.go:108] Creating table: nat chain: PREROUTING I0514 23:14:14.276093 3702 iptables.go:108] Creating table: filter chain: FORWARD I0514 23:14:14.292114 3702 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0514 23:14:14.292254 3702 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0514 23:14:14.292313 3702 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0514 23:14:14.292402 3702 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0514 23:14:14.292460 3702 gateway_init.go:325] Gateway and management port readiness took 2.706880684s I0514 23:14:14.292487 3702 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0514 23:14:14.292523 3702 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0514 23:14:14.292538 3702 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0514 23:14:14.292551 3702 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0514 23:14:14.292617 3702 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0514 23:14:14.292700 3702 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 54.951µs I0514 23:14:14.292971 3702 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0514 23:14:14.292997 3702 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.65µs I0514 23:14:14.293005 3702 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 34.01µs I0514 23:14:14.293010 3702 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0514 23:14:14.293019 3702 nad_controller.go:379] [node-nad-controller NAD controller]: started I0514 23:14:14.293024 3702 default_node_network_controller.go:899] Starting the default node network controller I0514 23:14:14.293029 3702 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0514 23:14:14.293110 3702 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 15.733µs I0514 23:14:14.293219 3702 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0514 23:14:14.293228 3702 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0514 23:14:14.293252 3702 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a6d0c0 0xc0002d5b20 0xc0002d5b90 0xc0002d5c00 0xc0002d5c70 0xc0002d5ce0 0xc0002d5d50 0xc000be2320 0xc0002d5dc0 0xc0002d5e30 0xc0008e62d0 0xc006155420 0xc0002d5ea0 map[0x2ff9080:0xc002824360 0x2ff95c0:0xc0028244e0 0x2ffa2e0:0xc001b114a0 0x2ffa580:0xc002824300 0x2ffad60:0xc0028243c0 0x2ffb000:0xc002824420 0x2ffb2a0:0xc002824480 0x2ffba80:0xc002824540 0x2ffbd20:0xc0028245a0 0x3026980:0xc001b109c0 0x3026d20:0xc001b10a20 0x3027800:0xc000b9d5c0 0x3029160:0xc001cd2000 0x30298a0:0xc001b11440 0x302a720:0xc00122f200] 0xc0000d2e00 0} I0514 23:14:14.293306 3702 gateway.go:151] Starting gateway service sync I0514 23:14:14.293746 3702 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0514 23:14:14.295907 3702 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0514 23:14:14.324119 3702 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0514 23:14:14.325270 3702 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0514 23:14:14.326363 3702 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0514 23:14:14.359940 3702 gateway.go:168] Gateway service sync done. Time taken: 66.619258ms I0514 23:14:14.360498 3702 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32423 I0514 23:14:14.360773 3702 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a6d0c0 0xc0002d5b20 0xc0002d5b90 0xc0002d5c00 0xc0002d5c70 0xc0002d5ce0 0xc0002d5d50 0xc000be2320 0xc0002d5dc0 0xc0002d5e30 0xc0008e62d0 0xc006155420 0xc0002d5ea0 map[0x2ff9080:0xc002824360 0x2ff95c0:0xc0028244e0 0x2ffa2e0:0xc001b114a0 0x2ffa580:0xc002824300 0x2ffad60:0xc0028243c0 0x2ffb000:0xc002824420 0x2ffb2a0:0xc002824480 0x2ffba80:0xc002824540 0x2ffbd20:0xc0028245a0 0x3026980:0xc001b109c0 0x3026d20:0xc001b10a20 0x3027800:0xc000b9d5c0 0x3029160:0xc001cd2000 0x30298a0:0xc001b11440 0x302a720:0xc00122f200] 0xc0000d2e00 0} I0514 23:14:14.361257 3702 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a6d0c0 0xc0002d5b20 0xc0002d5b90 0xc0002d5c00 0xc0002d5c70 0xc0002d5ce0 0xc0002d5d50 0xc000be2320 0xc0002d5dc0 0xc0002d5e30 0xc0008e62d0 0xc006155420 0xc0002d5ea0 map[0x2ff9080:0xc002824360 0x2ff95c0:0xc0028244e0 0x2ffa2e0:0xc001b114a0 0x2ffa580:0xc002824300 0x2ffad60:0xc0028243c0 0x2ffb000:0xc002824420 0x2ffb2a0:0xc002824480 0x2ffba80:0xc002824540 0x2ffbd20:0xc0028245a0 0x3026980:0xc001b109c0 0x3026d20:0xc001b10a20 0x3027800:0xc000b9d5c0 0x3029160:0xc001cd2000 0x30298a0:0xc001b11440 0x302a720:0xc00122f200] 0xc0000d2e00 0} I0514 23:14:14.361298 3702 gateway.go:532] Reconciling gateway with updates I0514 23:14:14.366881 3702 gateway.go:339] Spawning Conntrack Rule Check Thread I0514 23:14:14.366958 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0514 23:14:14.366971 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0514 23:14:14.366982 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0514 23:14:14.366979 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:14.366989 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0514 23:14:14.366992 3702 default_node_network_controller.go:944] Gateway and management port readiness took 73.757855ms I0514 23:14:14.366998 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:14.366997 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0514 23:14:14.367016 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0514 23:14:14.366964 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0514 23:14:14.367037 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0514 23:14:14.367050 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0514 23:14:14.367055 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0514 23:14:14.367067 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0514 23:14:14.367067 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0514 23:14:14.367074 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0514 23:14:14.367074 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0514 23:14:14.367077 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0514 23:14:14.367087 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0514 23:14:14.367093 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0514 23:14:14.367095 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0514 23:14:14.367098 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0514 23:14:14.367096 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0514 23:14:14.367103 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0514 23:14:14.367115 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0514 23:14:14.367119 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0514 23:14:14.367040 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0514 23:14:14.367127 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0514 23:14:14.367130 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0514 23:14:14.367128 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0514 23:14:14.367141 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0514 23:14:14.367143 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0514 23:14:14.367153 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0514 23:14:14.367154 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0514 23:14:14.367184 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0514 23:14:14.367197 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0514 23:14:14.367197 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0514 23:14:14.367100 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0514 23:14:14.367059 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0514 23:14:14.367214 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0514 23:14:14.367214 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0514 23:14:14.367139 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0514 23:14:14.367222 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0514 23:14:14.367237 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0514 23:14:14.367241 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0514 23:14:14.367153 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0514 23:14:14.367244 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0514 23:14:14.367251 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0514 23:14:14.367257 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0514 23:14:14.367137 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0514 23:14:14.367264 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0514 23:14:14.367273 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0514 23:14:14.367282 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0514 23:14:14.367217 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0514 23:14:14.367191 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0514 23:14:14.367300 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0514 23:14:14.367303 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0514 23:14:14.367302 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:14:14.367312 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0514 23:14:14.367315 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0514 23:14:14.367321 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:14:14.367328 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0514 23:14:14.367023 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0514 23:14:14.367346 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0514 23:14:14.367121 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0514 23:14:14.367308 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0514 23:14:14.367358 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0514 23:14:14.367365 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0514 23:14:14.367286 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0514 23:14:14.367374 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0514 23:14:14.367376 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0514 23:14:14.367383 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0514 23:14:14.367388 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0514 23:14:14.367349 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0514 23:14:14.367398 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0514 23:14:14.367402 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0514 23:14:14.367408 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0514 23:14:14.367367 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0514 23:14:14.367418 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0514 23:14:14.367294 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0514 23:14:14.367439 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0514 23:14:14.367459 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0514 23:14:14.367477 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0514 23:14:14.367494 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0514 23:14:14.367505 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0514 23:14:14.367505 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0514 23:14:14.367511 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0514 23:14:14.367514 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0514 23:14:14.367514 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0514 23:14:14.367525 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0514 23:14:14.367525 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0514 23:14:14.367459 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0514 23:14:14.367525 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0514 23:14:14.367545 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0514 23:14:14.367558 3702 port_claim.go:71] Svc openshift-ingress/router-default: port 31443 is already open, no action needed I0514 23:14:14.367571 3702 port_claim.go:71] Svc openshift-ingress/router-default: port 30875 is already open, no action needed I0514 23:14:14.367580 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0514 23:14:14.367581 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0514 23:14:14.367594 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0514 23:14:14.367596 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0514 23:14:14.367607 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0514 23:14:14.367619 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0514 23:14:14.367581 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0514 23:14:14.367634 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0514 23:14:14.367640 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0514 23:14:14.367627 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0514 23:14:14.367651 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0514 23:14:14.367659 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0514 23:14:14.367668 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0514 23:14:14.367679 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0514 23:14:14.367686 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0514 23:14:14.367685 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0514 23:14:14.367711 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0514 23:14:14.368417 3702 node_ip_handler_linux.go:160] Node IP manager is running W0514 23:14:14.405001 3702 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0514 23:14:14.405276 3702 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0514 23:14:14.405416 3702 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0514 23:14:14.413603 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0514 23:14:14.413628 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0514 23:14:14.413634 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0514 23:14:14.413641 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0514 23:14:14.413647 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0514 23:14:14.413653 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0514 23:14:14.413659 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0514 23:14:14.453225 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0514 23:14:14.453260 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0514 23:14:14.453271 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0514 23:14:14.453280 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0514 23:14:14.453289 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0514 23:14:14.453297 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0514 23:14:14.453306 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0514 23:14:14.453314 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0514 23:14:14.453321 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0514 23:14:14.453328 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0514 23:14:14.492960 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0514 23:14:14.492972 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0514 23:14:14.492962 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0514 23:14:14.492985 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0514 23:14:14.492992 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0514 23:14:15.170081 3702 egressfirewall.go:723] Syncing node "ip-10-0-0-78.ec2.internal" for egress firewall I0514 23:14:15.170105 3702 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-78.ec2.internal in network "default" I0514 23:14:15.170116 3702 node_tracker.go:220] Node ip-10-0-0-78.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0514 23:14:15.170184 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal E0514 23:14:15.170200 3702 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-78.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0514 23:14:15.170201 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 34.496µs I0514 23:14:15.170215 3702 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-78.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-78.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-78.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-78.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-78.ec2.internal, error parsing annotation for node ip-10-0-0-78.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0514 23:14:15.182459 3702 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-78.ec2.internal annotation: unexpected end of JSON input I0514 23:14:15.182487 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0514 23:14:15.182535 3702 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-78.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0514 23:14:15.193215 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal I0514 23:14:15.193225 3702 egressfirewall.go:723] Syncing node "ip-10-0-0-78.ec2.internal" for egress firewall I0514 23:14:15.193229 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 20.63µs E0514 23:14:15.193271 3702 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-78.ec2.internal annotation: unexpected end of JSON input I0514 23:14:15.193282 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0514 23:14:15.193338 3702 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-78.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0514 23:14:15.211445 3702 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-78.ec2.internal annotation: unexpected end of JSON input E0514 23:14:15.211502 3702 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-78.ec2.internal annotation: unexpected end of JSON input I0514 23:14:15.211520 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0514 23:14:15.211581 3702 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-78.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-78.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-78.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-78.ec2.internal" I0514 23:14:15.230068 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0514 23:14:15.368495 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal I0514 23:14:15.368518 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 30.191µs I0514 23:14:15.368495 3702 egressfirewall.go:723] Syncing node "ip-10-0-0-78.ec2.internal" for egress firewall I0514 23:14:15.368552 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:15.381858 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:15.623779 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:15.706036 3702 default_node_network_controller.go:1380] Starting node controller node sync W0514 23:14:15.706074 3702 egressip_healthcheck.go:76] Health checking using insecure connection I0514 23:14:15.706141 3702 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0514 23:14:15.717541 3702 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.485511ms I0514 23:14:15.717757 3702 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0514 23:14:15.718039 3702 egressservice_node.go:93] Setting up event handlers for Egress Services I0514 23:14:15.718325 3702 egressservice_node.go:183] Starting Egress Services Controller I0514 23:14:15.718348 3702 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0514 23:14:15.718363 3702 shared_informer.go:356] "Caches are synced" controller="egressservices" I0514 23:14:15.718373 3702 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0514 23:14:15.718381 3702 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0514 23:14:15.718391 3702 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0514 23:14:15.718399 3702 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0514 23:14:15.718411 3702 egressservice_node.go:197] Repairing Egress Services I0514 23:14:15.728634 3702 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0514 23:14:15.728645 3702 external_controller.go:278] Starting Admin Policy Based Route Controller I0514 23:14:15.728831 3702 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0514 23:14:15.728850 3702 link_network_manager.go:119] Link manager is running I0514 23:14:15.728867 3702 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0514 23:14:15.729048 3702 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0514 23:14:15.729527 3702 vrf_manager.go:128] VRF manager is running I0514 23:14:15.729629 3702 node_controller_manager.go:437] Removing flows to drop GARP I0514 23:14:15.729639 3702 gateway.go:532] Reconciling gateway with updates I0514 23:14:15.750072 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0514 23:14:15.750084 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0514 23:14:15.750085 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0514 23:14:15.750097 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0514 23:14:15.750102 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0514 23:14:15.750105 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0514 23:14:15.750108 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0514 23:14:15.750104 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0514 23:14:15.750118 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0514 23:14:15.750119 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0514 23:14:15.750121 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0514 23:14:15.750124 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0514 23:14:15.750125 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0514 23:14:15.750130 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0514 23:14:15.750137 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0514 23:14:15.750147 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0514 23:14:15.750150 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0514 23:14:15.750155 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0514 23:14:15.750186 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0514 23:14:15.750192 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0514 23:14:15.750194 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0514 23:14:15.750111 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0514 23:14:15.750205 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0514 23:14:15.750207 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0514 23:14:15.750205 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0514 23:14:15.750225 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0514 23:14:15.750228 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0514 23:14:15.750234 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:15.750252 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:15.750258 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0514 23:14:15.750274 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0514 23:14:15.750072 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0514 23:14:15.750285 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0514 23:14:15.750300 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0514 23:14:15.750308 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0514 23:14:15.750314 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0514 23:14:15.750238 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0514 23:14:15.750323 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0514 23:14:15.750099 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0514 23:14:15.750397 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0514 23:14:15.750226 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0514 23:14:15.750409 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0514 23:14:15.750459 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0514 23:14:15.750211 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0514 23:14:15.750470 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0514 23:14:15.750495 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0514 23:14:15.750509 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0514 23:14:15.750517 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0514 23:14:15.750500 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0514 23:14:15.750212 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0514 23:14:15.750523 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0514 23:14:15.750536 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0514 23:14:15.750542 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0514 23:14:15.750547 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0514 23:14:15.750549 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0514 23:14:15.750555 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0514 23:14:15.750563 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0514 23:14:15.750562 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0514 23:14:15.750564 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0514 23:14:15.750571 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0514 23:14:15.750577 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0514 23:14:15.750578 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0514 23:14:15.750579 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0514 23:14:15.750590 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0514 23:14:15.750595 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0514 23:14:15.750597 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0514 23:14:15.750596 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0514 23:14:15.750231 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0514 23:14:15.750602 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0514 23:14:15.750610 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0514 23:14:15.750613 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0514 23:14:15.750615 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0514 23:14:15.750619 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0514 23:14:15.750630 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0514 23:14:15.750638 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0514 23:14:15.750634 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0514 23:14:15.750640 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0514 23:14:15.750598 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:14:15.750654 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0514 23:14:15.750660 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:14:15.750663 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0514 23:14:15.750664 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0514 23:14:15.750614 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0514 23:14:15.750683 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0514 23:14:15.750583 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0514 23:14:15.750704 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0514 23:14:15.750715 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0514 23:14:15.750601 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0514 23:14:15.750536 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0514 23:14:15.750730 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0514 23:14:15.750735 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0514 23:14:15.750741 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0514 23:14:15.750752 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0514 23:14:15.750588 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0514 23:14:15.750762 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0514 23:14:15.750661 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0514 23:14:15.750762 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0514 23:14:15.750209 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0514 23:14:15.750798 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0514 23:14:15.750810 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0514 23:14:15.750526 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0514 23:14:15.750828 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0514 23:14:15.750833 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0514 23:14:15.750843 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0514 23:14:15.750569 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0514 23:14:15.750861 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0514 23:14:15.750574 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0514 23:14:15.750314 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0514 23:14:15.750586 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0514 23:14:15.750929 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0514 23:14:15.750936 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0514 23:14:15.750603 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0514 23:14:15.750603 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0514 23:14:15.750951 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0514 23:14:15.750622 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0514 23:14:15.750643 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0514 23:14:15.750642 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0514 23:14:15.750721 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0514 23:14:15.750754 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0514 23:14:15.750225 3702 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0514 23:14:15.751028 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0514 23:14:15.751040 3702 port_claim.go:71] Svc openshift-ingress/router-default: port 31443 is already open, no action needed I0514 23:14:15.751052 3702 port_claim.go:71] Svc openshift-ingress/router-default: port 30875 is already open, no action needed I0514 23:14:15.750893 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0514 23:14:15.750581 3702 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0514 23:14:15.789828 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0514 23:14:15.789849 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0514 23:14:15.789857 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0514 23:14:15.789863 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0514 23:14:15.828334 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0514 23:14:15.828360 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0514 23:14:15.868336 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0514 23:14:15.868347 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0514 23:14:15.868364 3702 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0514 23:14:16.465868 3702 default_network_controller.go:1025] Node "ip-10-0-2-152.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:17.829605 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-98.ec2.internal I0514 23:14:17.829699 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-98.ec2.internal: 101.974µs I0514 23:14:17.861613 3702 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-55bc57d65c-h99kj] creating logical port openshift-image-registry_image-registry-55bc57d65c-h99kj for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.861856 3702 kube.go:256] Updating pod openshift-image-registry/image-registry-55bc57d65c-h99kj I0514 23:14:17.878014 3702 pod.go:62] [openshift-image-registry/image-registry-55bc57d65c-h99kj] pod update took 16.210781ms I0514 23:14:17.878041 3702 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-55bc57d65c-h99kj] addLogicalPort annotation time took 16.244679ms I0514 23:14:17.879144 3702 pods.go:271] [openshift-image-registry/image-registry-55bc57d65c-h99kj] addLogicalPort took 17.554464ms, libovsdb time 830.57µs I0514 23:14:17.905732 3702 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-gcr6r] creating logical port openshift-dns_dns-default-gcr6r for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.905931 3702 kube.go:256] Updating pod openshift-dns/dns-default-gcr6r I0514 23:14:17.912610 3702 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-jw47b] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-jw47b for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.912730 3702 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9bbch] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-6944588dcc-9bbch for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.912802 3702 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-jw47b I0514 23:14:17.912886 3702 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9bbch I0514 23:14:17.912917 3702 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-tbtdr] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-tbtdr for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.913161 3702 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-tr4b9] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-tr4b9 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.913326 3702 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-tr4b9 I0514 23:14:17.913220 3702 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-tbtdr I0514 23:14:17.914769 3702 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-778445446b-fvv77] creating logical port openshift-ingress_router-default-778445446b-fvv77 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.914875 3702 kube.go:256] Updating pod openshift-ingress/router-default-778445446b-fvv77 I0514 23:14:17.915049 3702 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-477wn] creating logical port openshift-insights_insights-operator-6dbb575d6c-477wn for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.915156 3702 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-477wn I0514 23:14:17.955603 3702 pod.go:62] [openshift-dns/dns-default-gcr6r] pod update took 49.721986ms I0514 23:14:17.955625 3702 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-gcr6r] addLogicalPort annotation time took 49.754371ms I0514 23:14:17.955758 3702 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-jw47b] pod update took 43.01929ms I0514 23:14:17.955779 3702 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-jw47b] addLogicalPort annotation time took 43.046266ms I0514 23:14:17.956836 3702 pods.go:271] [openshift-dns/dns-default-gcr6r] addLogicalPort took 51.127575ms, libovsdb time 908.583µs I0514 23:14:17.957260 3702 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-jw47b] addLogicalPort took 44.675079ms, libovsdb time 1.1916ms I0514 23:14:17.957339 3702 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-7f4c9f6df5-xjsx9] creating logical port openshift-ingress_router-default-7f4c9f6df5-xjsx9 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.957423 3702 kube.go:256] Updating pod openshift-ingress/router-default-7f4c9f6df5-xjsx9 I0514 23:14:17.958489 3702 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-zcvvx] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-zcvvx for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.958595 3702 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-zcvvx I0514 23:14:17.958646 3702 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mw8mt] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67f4d568c7-mw8mt for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.958747 3702 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mw8mt I0514 23:14:17.958963 3702 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29646670-mt7xp for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.959054 3702 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-9v2w8] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-9v2w8 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.959060 3702 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp I0514 23:14:17.959191 3702 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-9v2w8 I0514 23:14:17.960034 3702 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-tr4b9] pod update took 46.739611ms I0514 23:14:17.960065 3702 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-tr4b9] addLogicalPort annotation time took 46.760935ms I0514 23:14:17.961119 3702 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-tr4b9] addLogicalPort took 47.968397ms, libovsdb time 807.542µs I0514 23:14:17.961165 3702 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-9zk7z] creating logical port openshift-ingress-canary_ingress-canary-9zk7z for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.961246 3702 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-9zk7z I0514 23:14:17.980522 3702 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-tbtdr] pod update took 67.346366ms I0514 23:14:17.980543 3702 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-tbtdr] addLogicalPort annotation time took 67.369784ms I0514 23:14:17.980593 3702 pod.go:62] [openshift-ingress/router-default-778445446b-fvv77] pod update took 65.746246ms I0514 23:14:17.980526 3702 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-477wn] pod update took 65.391052ms I0514 23:14:17.980612 3702 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-778445446b-fvv77] addLogicalPort annotation time took 65.764213ms I0514 23:14:17.980614 3702 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-477wn] addLogicalPort annotation time took 65.483503ms I0514 23:14:17.981221 3702 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9bbch] pod update took 68.367824ms I0514 23:14:17.981236 3702 base_network_controller_pods.go:951] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9bbch] addLogicalPort annotation time took 68.403963ms I0514 23:14:17.981961 3702 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-tbtdr] addLogicalPort took 69.049942ms, libovsdb time 1.120293ms I0514 23:14:17.982057 3702 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-bbwvr] creating logical port openshift-service-ca-operator_service-ca-operator-bc9564db4-bbwvr for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.982195 3702 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-bc9564db4-bbwvr I0514 23:14:17.982255 3702 pods.go:271] [openshift-ingress/router-default-778445446b-fvv77] addLogicalPort took 67.495022ms, libovsdb time 1.305683ms I0514 23:14:17.982671 3702 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-477wn] addLogicalPort took 67.631927ms, libovsdb time 1.673462ms I0514 23:14:17.982735 3702 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-c2fbf] creating logical port openshift-console-operator_console-operator-66dbbb6468-c2fbf for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.984149 3702 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9bbch] addLogicalPort took 71.440251ms, libovsdb time 2.691104ms I0514 23:14:17.984617 3702 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-c2fbf I0514 23:14:17.988101 3702 pod.go:62] [openshift-ingress/router-default-7f4c9f6df5-xjsx9] pod update took 30.709037ms I0514 23:14:17.988191 3702 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-7f4c9f6df5-xjsx9] addLogicalPort annotation time took 30.785863ms I0514 23:14:17.988927 3702 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mw8mt] pod update took 30.210938ms I0514 23:14:17.988978 3702 base_network_controller_pods.go:951] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mw8mt] addLogicalPort annotation time took 30.26649ms I0514 23:14:17.989982 3702 pods.go:271] [openshift-ingress/router-default-7f4c9f6df5-xjsx9] addLogicalPort took 32.656088ms, libovsdb time 1.187254ms I0514 23:14:17.993012 3702 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mw8mt] addLogicalPort took 34.375987ms, libovsdb time 3.446674ms I0514 23:14:17.993129 3702 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-xvq6n] creating logical port openshift-network-diagnostics_network-check-source-84fd6d9d54-xvq6n for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:17.993314 3702 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-84fd6d9d54-xvq6n I0514 23:14:17.998243 3702 pod.go:62] [openshift-ingress-canary/ingress-canary-9zk7z] pod update took 37.018643ms I0514 23:14:17.998268 3702 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-9zk7z] addLogicalPort annotation time took 37.04663ms I0514 23:14:17.999225 3702 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp] pod update took 40.187391ms I0514 23:14:17.999292 3702 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp] addLogicalPort annotation time took 40.253543ms I0514 23:14:17.999477 3702 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-zcvvx] pod update took 40.902702ms I0514 23:14:17.999541 3702 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-zcvvx] addLogicalPort annotation time took 40.968961ms I0514 23:14:18.001120 3702 pods.go:271] [openshift-ingress-canary/ingress-canary-9zk7z] addLogicalPort took 39.958122ms, libovsdb time 1.908962ms I0514 23:14:18.001245 3702 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp] addLogicalPort took 42.291698ms, libovsdb time 1.351282ms I0514 23:14:18.001957 3702 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-zcvvx] addLogicalPort took 43.501596ms, libovsdb time 1.871593ms I0514 23:14:18.004114 3702 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-9v2w8] pod update took 44.958518ms I0514 23:14:18.004130 3702 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-9v2w8] addLogicalPort annotation time took 44.976613ms I0514 23:14:18.005217 3702 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-9v2w8] addLogicalPort took 46.172353ms, libovsdb time 760.304µs I0514 23:14:18.008479 3702 pod.go:62] [openshift-service-ca-operator/service-ca-operator-bc9564db4-bbwvr] pod update took 26.367531ms I0514 23:14:18.008503 3702 base_network_controller_pods.go:951] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-bbwvr] addLogicalPort annotation time took 26.393507ms I0514 23:14:18.009485 3702 pods.go:271] [openshift-service-ca-operator/service-ca-operator-bc9564db4-bbwvr] addLogicalPort took 27.454989ms, libovsdb time 709.603µs I0514 23:14:18.018015 3702 pod.go:62] [openshift-network-diagnostics/network-check-source-84fd6d9d54-xvq6n] pod update took 24.740107ms I0514 23:14:18.018033 3702 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-xvq6n] addLogicalPort annotation time took 24.794704ms I0514 23:14:18.019022 3702 pods.go:271] [openshift-network-diagnostics/network-check-source-84fd6d9d54-xvq6n] addLogicalPort took 25.902503ms, libovsdb time 599.555µs I0514 23:14:18.021832 3702 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-c2fbf] pod update took 39.049389ms I0514 23:14:18.021854 3702 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-c2fbf] addLogicalPort annotation time took 39.07256ms I0514 23:14:18.022800 3702 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-c2fbf] addLogicalPort took 40.074001ms, libovsdb time 577.39µs I0514 23:14:19.681752 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:19.796406 3702 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp 01ba6dd4cf8a75e7eb71292aa8a2d840f7931da0c7c3de35c5405a34a65d2b9d network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp 01ba6dd4cf8a75e7eb71292aa8a2d840f7931da0c7c3de35c5405a34a65d2b9d network default NAD default NAD key ] I0514 23:14:19.796849 3702 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-zcvvx c5602b3268b294a33c3f78b69fcbcfbda411bef8401c51de10af07b46e104b65 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-zcvvx c5602b3268b294a33c3f78b69fcbcfbda411bef8401c51de10af07b46e104b65 network default NAD default NAD key ] I0514 23:14:19.796848 3702 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-9v2w8 b088f5fd70dcf38560e473c19e94ebc1d2251980400f073b96598af38f3ebec0 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-9v2w8 b088f5fd70dcf38560e473c19e94ebc1d2251980400f073b96598af38f3ebec0 network default NAD default NAD key ] I0514 23:14:19.798074 3702 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-477wn 41f05466a0cf9640552757bfff8587a7a4bf7d7bb97f98651f5d3be58c54b0e7 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-477wn 41f05466a0cf9640552757bfff8587a7a4bf7d7bb97f98651f5d3be58c54b0e7 network default NAD default NAD key ] I0514 23:14:19.798192 3702 cni.go:452] [openshift-service-ca-operator/service-ca-operator-bc9564db4-bbwvr bed0e72e452d4a9a1e949ca6a25015a61820351ece434984f5459fef6bcc81fb network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-bbwvr bed0e72e452d4a9a1e949ca6a25015a61820351ece434984f5459fef6bcc81fb network default NAD default NAD key ] I0514 23:14:19.799792 3702 cni.go:452] [openshift-network-diagnostics/network-check-source-84fd6d9d54-xvq6n f374e39d1abf7dbf576d0584b3c6590af26c22d640c996f05479d041274c07c8 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-xvq6n f374e39d1abf7dbf576d0584b3c6590af26c22d640c996f05479d041274c07c8 network default NAD default NAD key ] I0514 23:14:19.801247 3702 cni.go:452] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9bbch 48c11f8f56e59853357b46a9f6e6d869c4290d67e9447d4c84141a6f3c00341f network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9bbch 48c11f8f56e59853357b46a9f6e6d869c4290d67e9447d4c84141a6f3c00341f network default NAD default NAD key ] I0514 23:14:19.807776 3702 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-c2fbf 0fda4945d7c689aa969a2279c9eaf4ce2b1823d6251a4336c02b8d1b219d3903 network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-c2fbf 0fda4945d7c689aa969a2279c9eaf4ce2b1823d6251a4336c02b8d1b219d3903 network default NAD default NAD key ] I0514 23:14:19.821503 3702 cni.go:452] [openshift-ingress-canary/ingress-canary-9zk7z 3e08a5439a7352e056f1de294334b22604e0370a92d217b541360cba39e8dd3c network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-9zk7z 3e08a5439a7352e056f1de294334b22604e0370a92d217b541360cba39e8dd3c network default NAD default NAD key ] I0514 23:14:19.841881 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-84fd6d9d54-xvq6n, hostIfaceName: f374e39d1abf7db, network: default, NAD default, SandboxID: "f374e39d1abf7dbf576d0584b3c6590af26c22d640c996f05479d041274c07c8", PCI device ID: , UID: "dc0433ff-730b-4b6c-9dc8-4527f7298c72", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0514 23:14:19.847393 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-9zk7z, hostIfaceName: 3e08a5439a7352e, network: default, NAD default, SandboxID: "3e08a5439a7352e056f1de294334b22604e0370a92d217b541360cba39e8dd3c", PCI device ID: , UID: "a4946dc0-7f39-4fb3-bacc-a239a355de2b", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0514 23:14:19.848575 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-477wn, hostIfaceName: 41f05466a0cf964, network: default, NAD default, SandboxID: "41f05466a0cf9640552757bfff8587a7a4bf7d7bb97f98651f5d3be58c54b0e7", PCI device ID: , UID: "717614e5-c3f0-45d9-a4fd-785003520b62", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0514 23:14:19.850372 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-9v2w8, hostIfaceName: b088f5fd70dcf38, network: default, NAD default, SandboxID: "b088f5fd70dcf38560e473c19e94ebc1d2251980400f073b96598af38f3ebec0", PCI device ID: , UID: "98d6a31d-0429-4be3-a8fe-1263e3e55fae", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0514 23:14:19.852393 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-bc9564db4-bbwvr, hostIfaceName: bed0e72e452d4a9, network: default, NAD default, SandboxID: "bed0e72e452d4a9a1e949ca6a25015a61820351ece434984f5459fef6bcc81fb", PCI device ID: , UID: "2745132a-6640-4c0e-ba72-eb9cb9e8cd98", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0514 23:14:19.853225 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-c2fbf, hostIfaceName: 0fda4945d7c689a, network: default, NAD default, SandboxID: "0fda4945d7c689aa969a2279c9eaf4ce2b1823d6251a4336c02b8d1b219d3903", PCI device ID: , UID: "ce8b4f37-fead-437a-ab15-01fbd958d46d", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0514 23:14:19.854283 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29646670-mt7xp, hostIfaceName: 01ba6dd4cf8a75e, network: default, NAD default, SandboxID: "01ba6dd4cf8a75e7eb71292aa8a2d840f7931da0c7c3de35c5405a34a65d2b9d", PCI device ID: , UID: "bde7f47d-80bd-4661-8455-8cbe0313417a", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0514 23:14:19.867577 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-6944588dcc-9bbch, hostIfaceName: 48c11f8f56e5985, network: default, NAD default, SandboxID: "48c11f8f56e59853357b46a9f6e6d869c4290d67e9447d4c84141a6f3c00341f", PCI device ID: , UID: "30f1ddc5-7aa6-40dc-ae04-db3e1b819245", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0514 23:14:19.872712 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-zcvvx, hostIfaceName: c5602b3268b294a, network: default, NAD default, SandboxID: "c5602b3268b294a33c3f78b69fcbcfbda411bef8401c51de10af07b46e104b65", PCI device ID: , UID: "1c5629e3-87ec-420f-b5e1-539bf8ae95e9", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0514 23:14:19.917520 3702 cni.go:473] [openshift-network-diagnostics/network-check-source-84fd6d9d54-xvq6n f374e39d1abf7dbf576d0584b3c6590af26c22d640c996f05479d041274c07c8 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-xvq6n f374e39d1abf7dbf576d0584b3c6590af26c22d640c996f05479d041274c07c8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:d3:df:bc:6f:ff\",\"name\":\"f374e39d1abf7db\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/55e10dd6-593e-425d-a94c-5d58bb2bca26\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:19.928470 3702 cni.go:473] [openshift-ingress-canary/ingress-canary-9zk7z 3e08a5439a7352e056f1de294334b22604e0370a92d217b541360cba39e8dd3c network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-9zk7z 3e08a5439a7352e056f1de294334b22604e0370a92d217b541360cba39e8dd3c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:5a:94:f9:fb:55\",\"name\":\"3e08a5439a7352e\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0ba97eff-ac52-49f5-91b3-34cd1f57459c\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:19.928528 3702 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-477wn 41f05466a0cf9640552757bfff8587a7a4bf7d7bb97f98651f5d3be58c54b0e7 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-477wn 41f05466a0cf9640552757bfff8587a7a4bf7d7bb97f98651f5d3be58c54b0e7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:89:03:61:69:40\",\"name\":\"41f05466a0cf964\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/54d257c8-a60c-47fd-a1f6-bdb35f4a95b7\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:19.928632 3702 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-9v2w8 b088f5fd70dcf38560e473c19e94ebc1d2251980400f073b96598af38f3ebec0 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-9v2w8 b088f5fd70dcf38560e473c19e94ebc1d2251980400f073b96598af38f3ebec0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:2f:46:64:ba:ac\",\"name\":\"b088f5fd70dcf38\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eb442902-0cbd-4204-98f2-375f53af50c1\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:19.944891 3702 cni.go:473] [openshift-service-ca-operator/service-ca-operator-bc9564db4-bbwvr bed0e72e452d4a9a1e949ca6a25015a61820351ece434984f5459fef6bcc81fb network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-bbwvr bed0e72e452d4a9a1e949ca6a25015a61820351ece434984f5459fef6bcc81fb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:8d:d8:03:77:2e\",\"name\":\"bed0e72e452d4a9\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2e15c189-10e3-4a96-957c-6a1d319d553a\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:19.945666 3702 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-zcvvx c5602b3268b294a33c3f78b69fcbcfbda411bef8401c51de10af07b46e104b65 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-zcvvx c5602b3268b294a33c3f78b69fcbcfbda411bef8401c51de10af07b46e104b65 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:51:0a:ce:4d:13\",\"name\":\"c5602b3268b294a\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c422ec7f-2f28-4d28-a6ec-8c7e6006cbda\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:19.945930 3702 cni.go:473] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9bbch 48c11f8f56e59853357b46a9f6e6d869c4290d67e9447d4c84141a6f3c00341f network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9bbch 48c11f8f56e59853357b46a9f6e6d869c4290d67e9447d4c84141a6f3c00341f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:4f:59:b7:0b:46\",\"name\":\"48c11f8f56e5985\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/12a34399-255b-4916-bd14-cb375ec3ac69\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:19.946019 3702 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp 01ba6dd4cf8a75e7eb71292aa8a2d840f7931da0c7c3de35c5405a34a65d2b9d network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp 01ba6dd4cf8a75e7eb71292aa8a2d840f7931da0c7c3de35c5405a34a65d2b9d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:3b:06:3d:56:72\",\"name\":\"01ba6dd4cf8a75e\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/087d1f0c-3251-4d20-a840-5fb093981124\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:19.946816 3702 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-c2fbf 0fda4945d7c689aa969a2279c9eaf4ce2b1823d6251a4336c02b8d1b219d3903 network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-c2fbf 0fda4945d7c689aa969a2279c9eaf4ce2b1823d6251a4336c02b8d1b219d3903 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:f7:33:8f:70:7b\",\"name\":\"0fda4945d7c689a\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/663bf91c-5bba-4bdc-a5f8-61484fb313f5\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:20.830899 3702 cni.go:452] [kube-system/global-pull-secret-syncer-49qv2 ba0ed6129398fadb2a65dabfc28ac734f4f3e2101b6468cf6b30c7197826fdcb network default NAD default NAD key ] ADD starting CNI request [kube-system/global-pull-secret-syncer-49qv2 ba0ed6129398fadb2a65dabfc28ac734f4f3e2101b6468cf6b30c7197826fdcb network default NAD default NAD key ] I0514 23:14:20.843371 3702 helper_linux.go:556] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-49qv2, hostIfaceName: ba0ed6129398fad, network: default, NAD default, SandboxID: "ba0ed6129398fadb2a65dabfc28ac734f4f3e2101b6468cf6b30c7197826fdcb", PCI device ID: , UID: "ee393f76-20a0-443c-a4c8-54ec44f9df4e", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0514 23:14:20.896762 3702 cni.go:473] [kube-system/global-pull-secret-syncer-49qv2 ba0ed6129398fadb2a65dabfc28ac734f4f3e2101b6468cf6b30c7197826fdcb network default NAD default NAD key default] ADD finished CNI request [kube-system/global-pull-secret-syncer-49qv2 ba0ed6129398fadb2a65dabfc28ac734f4f3e2101b6468cf6b30c7197826fdcb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:59:a7:ca:00:4c\",\"name\":\"ba0ed6129398fad\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eee42db0-d67d-4dc0-9d45-70e2e815d446\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:21.147143 3702 cni.go:452] [openshift-network-diagnostics/network-check-target-x87mq 3bd9d9794572bdb3e4324218d34da178bc501f39b23c21357c15daf33d81fab5 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-x87mq 3bd9d9794572bdb3e4324218d34da178bc501f39b23c21357c15daf33d81fab5 network default NAD default NAD key ] I0514 23:14:21.158874 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-x87mq, hostIfaceName: 3bd9d9794572bdb, network: default, NAD default, SandboxID: "3bd9d9794572bdb3e4324218d34da178bc501f39b23c21357c15daf33d81fab5", PCI device ID: , UID: "3e7e0a86-e068-4cd0-86c4-02d821a7ad61", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0514 23:14:21.220399 3702 cni.go:473] [openshift-network-diagnostics/network-check-target-x87mq 3bd9d9794572bdb3e4324218d34da178bc501f39b23c21357c15daf33d81fab5 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-x87mq 3bd9d9794572bdb3e4324218d34da178bc501f39b23c21357c15daf33d81fab5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:18:88:64:48:28\",\"name\":\"3bd9d9794572bdb\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/de8e61aa-ab85-47f0-a8b6-6b427e1d9245\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:24.273971 3702 egressfirewall.go:723] Syncing node "ip-10-0-2-152.ec2.internal" for egress firewall I0514 23:14:24.274020 3702 default_network_controller.go:1025] Node "ip-10-0-2-152.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:24.274035 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-152.ec2.internal I0514 23:14:24.274050 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-152.ec2.internal: 19.61µs I0514 23:14:25.255076 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:26.834346 3702 default_network_controller.go:1025] Node "ip-10-0-2-152.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:29.681310 3702 default_network_controller.go:1025] Node "ip-10-0-2-152.ec2.internal" in remote zone "ip-10-0-2-152.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:29.681308 3702 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-152.ec2.internal in network "default" I0514 23:14:29.681606 3702 node_tracker.go:235] Node ip-10-0-2-152.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-152.ec2.internal" I0514 23:14:29.681633 3702 node_tracker.go:169] Node ip-10-0-2-152.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:29.681649 3702 services_controller.go:574] Full service sync requested for network=default I0514 23:14:29.744041 3702 default_network_controller.go:1025] Node "ip-10-0-2-152.ec2.internal" in remote zone "ip-10-0-2-152.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:29.848049 3702 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-152.ec2.internal in network "default" I0514 23:14:29.848080 3702 default_network_controller.go:1025] Node "ip-10-0-2-152.ec2.internal" in remote zone "ip-10-0-2-152.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:29.848147 3702 node_tracker.go:169] Node ip-10-0-2-152.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:29.848167 3702 services_controller.go:574] Full service sync requested for network=default I0514 23:14:29.849237 3702 zone_ic_handler.go:195] Time taken to create transit switch: 309.624µs I0514 23:14:29.849256 3702 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-152.ec2.internal for the network default I0514 23:14:29.850492 3702 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-152.ec2.internal" on network "default" took: 1.577585ms I0514 23:14:31.906869 3702 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-152.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0514 23:14:31.906957 3702 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-152.ec2.internal", IP: 10.0.2.152 to PMTUD blocking rules I0514 23:14:31.906885 3702 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-152.ec2.internal in network "default" I0514 23:14:31.907109 3702 node_tracker.go:169] Node ip-10-0-2-152.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:31.907133 3702 services_controller.go:574] Full service sync requested for network=default I0514 23:14:31.906926 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-152.ec2.internal I0514 23:14:31.907224 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-152.ec2.internal: 303.782µs I0514 23:14:31.906942 3702 egressfirewall.go:723] Syncing node "ip-10-0-2-152.ec2.internal" for egress firewall I0514 23:14:31.913696 3702 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-152.ec2.internal. Updating no re-route policies I0514 23:14:31.923932 3702 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-152.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0514 23:14:31.923998 3702 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-152.ec2.internal", IP: 10.0.2.152 to PMTUD blocking rules W0514 23:14:32.266141 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0514 23:14:32.437298 3702 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0514 23:14:32.484017 3702 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:32.762390 3702 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0514 23:14:32.763278 3702 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 869.612µs I0514 23:14:32.831757 3702 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0514 23:14:32.930127 3702 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zhsfz] creating logical port openshift-kube-storage-version-migrator_migrator-846d9b8bdc-zhsfz for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:32.930296 3702 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zhsfz I0514 23:14:32.944168 3702 pod.go:62] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zhsfz] pod update took 13.902654ms I0514 23:14:32.944275 3702 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zhsfz] addLogicalPort annotation time took 14.012857ms I0514 23:14:32.945646 3702 pods.go:271] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zhsfz] addLogicalPort took 15.544663ms, libovsdb time 1.006534ms I0514 23:14:33.097617 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal I0514 23:14:33.097643 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 32.975µs I0514 23:14:33.097641 3702 egressfirewall.go:723] Syncing node "ip-10-0-0-78.ec2.internal" for egress firewall I0514 23:14:33.097665 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:33.355118 3702 cni.go:452] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zhsfz fb5f2f0d39c14e4e0516893cf73855553d8c6b914cc7dbafba86f39564845bb1 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zhsfz fb5f2f0d39c14e4e0516893cf73855553d8c6b914cc7dbafba86f39564845bb1 network default NAD default NAD key ] I0514 23:14:33.364361 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-846d9b8bdc-zhsfz, hostIfaceName: fb5f2f0d39c14e4, network: default, NAD default, SandboxID: "fb5f2f0d39c14e4e0516893cf73855553d8c6b914cc7dbafba86f39564845bb1", PCI device ID: , UID: "887f028b-3bd2-4ee6-be6f-4dbedc8e6d5b", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0514 23:14:33.413065 3702 cni.go:473] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zhsfz fb5f2f0d39c14e4e0516893cf73855553d8c6b914cc7dbafba86f39564845bb1 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zhsfz fb5f2f0d39c14e4e0516893cf73855553d8c6b914cc7dbafba86f39564845bb1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:30:d2:50:45:25\",\"name\":\"fb5f2f0d39c14e4\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c2baaf19-ade7-45b1-8ee5-98ef81f81b53\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:33.686378 3702 namespace.go:98] [openshift-service-ca] adding namespace I0514 23:14:33.687157 3702 namespace.go:102] [openshift-service-ca] adding namespace took 758.636µs I0514 23:14:33.700777 3702 namespace.go:142] [openshift-service-ca] updating namespace I0514 23:14:35.365817 3702 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-bxx84] creating logical port openshift-service-ca_service-ca-d9cb48497-bxx84 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:35.365940 3702 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-bxx84 I0514 23:14:35.380970 3702 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-bxx84] pod update took 15.051877ms I0514 23:14:35.380996 3702 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-bxx84] addLogicalPort annotation time took 15.082915ms I0514 23:14:35.382337 3702 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-bxx84] addLogicalPort took 16.536973ms, libovsdb time 977.373µs I0514 23:14:35.627222 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-152.ec2.internal I0514 23:14:35.627253 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-152.ec2.internal: 36.139µs I0514 23:14:35.748759 3702 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-bxx84 cd62fb3bcc4f25e2d89739fb43b2906c8f0bad58205304895fcafc0728196a7b network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-bxx84 cd62fb3bcc4f25e2d89739fb43b2906c8f0bad58205304895fcafc0728196a7b network default NAD default NAD key ] I0514 23:14:35.757855 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-bxx84, hostIfaceName: cd62fb3bcc4f25e, network: default, NAD default, SandboxID: "cd62fb3bcc4f25e2d89739fb43b2906c8f0bad58205304895fcafc0728196a7b", PCI device ID: , UID: "136cba83-3906-4f5f-940b-97323d8554af", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0514 23:14:35.807856 3702 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-bxx84 cd62fb3bcc4f25e2d89739fb43b2906c8f0bad58205304895fcafc0728196a7b network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-bxx84 cd62fb3bcc4f25e2d89739fb43b2906c8f0bad58205304895fcafc0728196a7b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:c7:ae:aa:76:ac\",\"name\":\"cd62fb3bcc4f25e\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a31edb2f-f6a1-400b-8a8b-e2f43432b465\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:39.044300 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "ip-10-0-0-78.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:39.044307 3702 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-78.ec2.internal in network "default" I0514 23:14:39.044411 3702 node_tracker.go:235] Node ip-10-0-0-78.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-78.ec2.internal" I0514 23:14:39.044431 3702 node_tracker.go:169] Node ip-10-0-0-78.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:39.044445 3702 services_controller.go:574] Full service sync requested for network=default W0514 23:14:39.044708 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0514 23:14:39.044961 3702 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0514 23:14:39.044979 3702 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0514 23:14:39.131117 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "ip-10-0-0-78.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:39.168535 3702 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-78.ec2.internal in network "default" I0514 23:14:39.168557 3702 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "ip-10-0-0-78.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:39.168594 3702 node_tracker.go:169] Node ip-10-0-0-78.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:39.168607 3702 services_controller.go:574] Full service sync requested for network=default W0514 23:14:39.169063 3702 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0514 23:14:39.169086 3702 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0514 23:14:39.169226 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0514 23:14:39.169700 3702 zone_ic_handler.go:195] Time taken to create transit switch: 299.483µs I0514 23:14:39.169714 3702 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-78.ec2.internal for the network default I0514 23:14:39.170623 3702 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-78.ec2.internal" on network "default" took: 1.235013ms W0514 23:14:39.309662 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0514 23:14:40.464832 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0514 23:14:40.783576 3702 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-78.ec2.internal in network "default" I0514 23:14:40.783599 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal I0514 23:14:40.783617 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 24.522µs I0514 23:14:40.783622 3702 egressfirewall.go:723] Syncing node "ip-10-0-0-78.ec2.internal" for egress firewall I0514 23:14:40.783638 3702 node_tracker.go:169] Node ip-10-0-0-78.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:40.783651 3702 services_controller.go:574] Full service sync requested for network=default I0514 23:14:40.783675 3702 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-78.ec2.internal", IP: 10.0.0.78 to PMTUD blocking rules I0514 23:14:40.783687 3702 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-78.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0514 23:14:40.784058 3702 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0514 23:14:40.784210 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0514 23:14:40.789909 3702 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-78.ec2.internal. Updating no re-route policies I0514 23:14:40.823649 3702 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-78.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0514 23:14:40.823667 3702 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-78.ec2.internal", IP: 10.0.0.78 to PMTUD blocking rules W0514 23:14:41.087123 3702 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-854969bbd6-gtc27 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-854969bbd6-gtc27: no pod IPs found I0514 23:14:42.294640 3702 obj_retry.go:498] Detected object openshift-image-registry/image-registry-55bc57d65c-h99kj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:14:42.294679 3702 pods.go:176] Deleting pod: openshift-image-registry/image-registry-55bc57d65c-h99kj I0514 23:14:42.296008 3702 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-55bc57d65c-h99kj, ips: 10.128.0.6 I0514 23:14:42.296035 3702 obj_retry.go:498] Detected object openshift-image-registry/image-registry-55bc57d65c-h99kj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:14:42.303134 3702 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-d477cf66b-ddgmf] creating logical port openshift-image-registry_image-registry-d477cf66b-ddgmf for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:42.303244 3702 kube.go:256] Updating pod openshift-image-registry/image-registry-d477cf66b-ddgmf I0514 23:14:42.317115 3702 obj_retry.go:492] Detected object openshift-image-registry/image-registry-55bc57d65c-h99kj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:14:42.317134 3702 obj_retry.go:492] Detected object openshift-image-registry/image-registry-55bc57d65c-h99kj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:14:42.324093 3702 pod.go:62] [openshift-image-registry/image-registry-d477cf66b-ddgmf] pod update took 20.882656ms I0514 23:14:42.324122 3702 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-d477cf66b-ddgmf] addLogicalPort annotation time took 20.915739ms I0514 23:14:42.324166 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-55bc57d65c-h99kj I0514 23:14:42.324243 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-55bc57d65c-h99kj I0514 23:14:42.325467 3702 pods.go:271] [openshift-image-registry/image-registry-d477cf66b-ddgmf] addLogicalPort took 22.344519ms, libovsdb time 968.746µs I0514 23:14:42.662647 3702 cni.go:452] [openshift-image-registry/image-registry-d477cf66b-ddgmf 1d3925a3966680bd775d22b5137f9197f4a494ceef821270c1ea48250250f46e network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-d477cf66b-ddgmf 1d3925a3966680bd775d22b5137f9197f4a494ceef821270c1ea48250250f46e network default NAD default NAD key ] I0514 23:14:42.672472 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-d477cf66b-ddgmf, hostIfaceName: 1d3925a3966680b, network: default, NAD default, SandboxID: "1d3925a3966680bd775d22b5137f9197f4a494ceef821270c1ea48250250f46e", PCI device ID: , UID: "2c7e0e91-e522-44b0-b238-026ad98c4cb7", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0514 23:14:42.721889 3702 cni.go:473] [openshift-image-registry/image-registry-d477cf66b-ddgmf 1d3925a3966680bd775d22b5137f9197f4a494ceef821270c1ea48250250f46e network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-d477cf66b-ddgmf 1d3925a3966680bd775d22b5137f9197f4a494ceef821270c1ea48250250f46e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:a4:3b:ef:fa:12\",\"name\":\"1d3925a3966680b\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/10b0a517-b61b-47f6-abe9-fc2d09df401d\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:50.094675 3702 cni.go:452] [openshift-dns/dns-default-gcr6r c50b9eb70fbbabed0c4b04f4070b339bf80ca762a535df19ec9a1b2cc5199a2a network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-gcr6r c50b9eb70fbbabed0c4b04f4070b339bf80ca762a535df19ec9a1b2cc5199a2a network default NAD default NAD key ] I0514 23:14:50.104134 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-gcr6r, hostIfaceName: c50b9eb70fbbabe, network: default, NAD default, SandboxID: "c50b9eb70fbbabed0c4b04f4070b339bf80ca762a535df19ec9a1b2cc5199a2a", PCI device ID: , UID: "9acd5b33-795e-4c98-b249-ffe64d8e25ce", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0514 23:14:50.120633 3702 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-jw47b 67d3f691d04090c4f194dad51ff7dbd31301c55e33c3b41c06b86d1a20cf5dbc network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-jw47b 67d3f691d04090c4f194dad51ff7dbd31301c55e33c3b41c06b86d1a20cf5dbc network default NAD default NAD key ] I0514 23:14:50.131263 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-jw47b, hostIfaceName: 67d3f691d04090c, network: default, NAD default, SandboxID: "67d3f691d04090c4f194dad51ff7dbd31301c55e33c3b41c06b86d1a20cf5dbc", PCI device ID: , UID: "dbae4c43-c590-4ad0-b7e3-179a19cde75d", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0514 23:14:50.159363 3702 cni.go:473] [openshift-dns/dns-default-gcr6r c50b9eb70fbbabed0c4b04f4070b339bf80ca762a535df19ec9a1b2cc5199a2a network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-gcr6r c50b9eb70fbbabed0c4b04f4070b339bf80ca762a535df19ec9a1b2cc5199a2a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:6c:f1:60:aa:59\",\"name\":\"c50b9eb70fbbabe\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/878a70a6-318c-4ef8-bbab-e09e96657794\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:50.165134 3702 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-tbtdr ef795b0ed4ece31e4d506a6cf7fb9636992e62c4f3eaeaa6e541eb4d094a7cb3 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-tbtdr ef795b0ed4ece31e4d506a6cf7fb9636992e62c4f3eaeaa6e541eb4d094a7cb3 network default NAD default NAD key ] I0514 23:14:50.180345 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-tbtdr, hostIfaceName: ef795b0ed4ece31, network: default, NAD default, SandboxID: "ef795b0ed4ece31e4d506a6cf7fb9636992e62c4f3eaeaa6e541eb4d094a7cb3", PCI device ID: , UID: "d8072e0e-b970-4b9b-bc00-36025a7bcc9d", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0514 23:14:50.185377 3702 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-jw47b 67d3f691d04090c4f194dad51ff7dbd31301c55e33c3b41c06b86d1a20cf5dbc network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-jw47b 67d3f691d04090c4f194dad51ff7dbd31301c55e33c3b41c06b86d1a20cf5dbc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:90:79:da:b1:be\",\"name\":\"67d3f691d04090c\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/590d94d2-9fd5-4250-976c-dab3a4039bf6\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:50.213947 3702 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-tr4b9 15e8b13d42a7b9c609419fbe42642413f77fed1a29b15e34647f735cc8fcf3b2 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-tr4b9 15e8b13d42a7b9c609419fbe42642413f77fed1a29b15e34647f735cc8fcf3b2 network default NAD default NAD key ] I0514 23:14:50.228963 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-tr4b9, hostIfaceName: 15e8b13d42a7b9c, network: default, NAD default, SandboxID: "15e8b13d42a7b9c609419fbe42642413f77fed1a29b15e34647f735cc8fcf3b2", PCI device ID: , UID: "77898961-65e8-46a1-9550-0bcb64e56186", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0514 23:14:50.236394 3702 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-tbtdr ef795b0ed4ece31e4d506a6cf7fb9636992e62c4f3eaeaa6e541eb4d094a7cb3 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-tbtdr ef795b0ed4ece31e4d506a6cf7fb9636992e62c4f3eaeaa6e541eb4d094a7cb3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:1b:50:fe:16:57\",\"name\":\"ef795b0ed4ece31\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4bb6bd28-e77e-44bc-a47d-3674942dfce6\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:50.261607 3702 cni.go:452] [openshift-ingress/router-default-778445446b-fvv77 937a27f5a5d9b95ea606d55f576e7749efe113827ff53f69ec2331a4cd2a5c57 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-778445446b-fvv77 937a27f5a5d9b95ea606d55f576e7749efe113827ff53f69ec2331a4cd2a5c57 network default NAD default NAD key ] I0514 23:14:50.282136 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-778445446b-fvv77, hostIfaceName: 937a27f5a5d9b95, network: default, NAD default, SandboxID: "937a27f5a5d9b95ea606d55f576e7749efe113827ff53f69ec2331a4cd2a5c57", PCI device ID: , UID: "4e1656d1-05d0-4f2e-85dd-4066986d902b", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0514 23:14:50.283040 3702 cni.go:452] [openshift-ingress/router-default-7f4c9f6df5-xjsx9 3203d1489f6412e3e7cd5a594f71dd93b449a08ff805ccd929ff2684f52b1c66 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-7f4c9f6df5-xjsx9 3203d1489f6412e3e7cd5a594f71dd93b449a08ff805ccd929ff2684f52b1c66 network default NAD default NAD key ] I0514 23:14:50.286909 3702 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-tr4b9 15e8b13d42a7b9c609419fbe42642413f77fed1a29b15e34647f735cc8fcf3b2 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-tr4b9 15e8b13d42a7b9c609419fbe42642413f77fed1a29b15e34647f735cc8fcf3b2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:ca:b4:68:a3:dd\",\"name\":\"15e8b13d42a7b9c\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bee0031b-7415-40f1-9f88-80de5b064329\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:50.308071 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-7f4c9f6df5-xjsx9, hostIfaceName: 3203d1489f6412e, network: default, NAD default, SandboxID: "3203d1489f6412e3e7cd5a594f71dd93b449a08ff805ccd929ff2684f52b1c66", PCI device ID: , UID: "7e6f743a-aadf-4349-a916-c135134e0937", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0514 23:14:50.344822 3702 cni.go:473] [openshift-ingress/router-default-778445446b-fvv77 937a27f5a5d9b95ea606d55f576e7749efe113827ff53f69ec2331a4cd2a5c57 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-778445446b-fvv77 937a27f5a5d9b95ea606d55f576e7749efe113827ff53f69ec2331a4cd2a5c57 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a6:18:00:ff:1c:ca\",\"name\":\"937a27f5a5d9b95\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4ca0904a-7a63-44e6-bba4-38447f10f13b\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:50.360267 3702 cni.go:452] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mw8mt 6017d4ac8e204aafcd820ba5f4403b4ce34b12112ed791b4e51842711f5655c3 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mw8mt 6017d4ac8e204aafcd820ba5f4403b4ce34b12112ed791b4e51842711f5655c3 network default NAD default NAD key ] I0514 23:14:50.365128 3702 cni.go:473] [openshift-ingress/router-default-7f4c9f6df5-xjsx9 3203d1489f6412e3e7cd5a594f71dd93b449a08ff805ccd929ff2684f52b1c66 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-7f4c9f6df5-xjsx9 3203d1489f6412e3e7cd5a594f71dd93b449a08ff805ccd929ff2684f52b1c66 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:e4:79:d8:86:e7\",\"name\":\"3203d1489f6412e\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ae5a37e3-241e-4395-b5cb-86b1f2b0ef7e\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:50.369491 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67f4d568c7-mw8mt, hostIfaceName: 6017d4ac8e204aa, network: default, NAD default, SandboxID: "6017d4ac8e204aafcd820ba5f4403b4ce34b12112ed791b4e51842711f5655c3", PCI device ID: , UID: "898981a8-9d4d-4df2-a4e2-d7c718554b23", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0514 23:14:50.420373 3702 cni.go:473] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mw8mt 6017d4ac8e204aafcd820ba5f4403b4ce34b12112ed791b4e51842711f5655c3 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mw8mt 6017d4ac8e204aafcd820ba5f4403b4ce34b12112ed791b4e51842711f5655c3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:ac:0e:6f:46:c1\",\"name\":\"6017d4ac8e204aa\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bc8f70c4-0dc6-4318-b7bb-e1d74170bd2b\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:53.249604 3702 cni.go:452] [openshift-multus/network-metrics-daemon-mhnwf 471ccd0b1e0608a9c30b5187e2d57cd5e77ae6a70ef0122ee8cb7f5433d06981 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-mhnwf 471ccd0b1e0608a9c30b5187e2d57cd5e77ae6a70ef0122ee8cb7f5433d06981 network default NAD default NAD key ] I0514 23:14:53.259131 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-mhnwf, hostIfaceName: 471ccd0b1e0608a, network: default, NAD default, SandboxID: "471ccd0b1e0608a9c30b5187e2d57cd5e77ae6a70ef0122ee8cb7f5433d06981", PCI device ID: , UID: "eed80ecc-1f1e-4d2a-ae1d-6aab76cf14d7", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0514 23:14:53.310003 3702 cni.go:473] [openshift-multus/network-metrics-daemon-mhnwf 471ccd0b1e0608a9c30b5187e2d57cd5e77ae6a70ef0122ee8cb7f5433d06981 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-mhnwf 471ccd0b1e0608a9c30b5187e2d57cd5e77ae6a70ef0122ee8cb7f5433d06981 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:b3:3b:2d:e2:3e\",\"name\":\"471ccd0b1e0608a\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/772ef4ee-0628-4c13-b506-07c539807ec8\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0514 23:14:57.903660 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0514 23:14:57.908458 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0514 23:14:57.908479 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0514 23:14:57.908486 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0514 23:14:57.955480 3702 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0514 23:14:58.019626 3702 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0514 23:14:58.102110 3702 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0514 23:14:58.127443 3702 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0514 23:14:58.204124 3702 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-d477cf66b-rhxtq IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-d477cf66b-rhxtq: no pod IPs found I0514 23:14:58.264415 3702 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mn7g8] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-mn7g8 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:58.264541 3702 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mn7g8 I0514 23:14:58.355444 3702 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-dhjdf] creating logical port openshift-insights_insights-runtime-extractor-dhjdf for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:58.355577 3702 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-dhjdf I0514 23:14:58.397120 3702 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mn7g8] pod update took 132.601802ms I0514 23:14:58.397154 3702 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mn7g8] addLogicalPort annotation time took 132.641256ms I0514 23:14:58.398880 3702 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mn7g8] addLogicalPort took 134.479225ms, libovsdb time 1.18783ms I0514 23:14:58.441298 3702 pod.go:62] [openshift-insights/insights-runtime-extractor-dhjdf] pod update took 85.748703ms I0514 23:14:58.441324 3702 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-dhjdf] addLogicalPort annotation time took 85.782202ms I0514 23:14:58.442563 3702 pods.go:271] [openshift-insights/insights-runtime-extractor-dhjdf] addLogicalPort took 87.136824ms, libovsdb time 873.354µs I0514 23:14:58.494613 3702 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-glzxh] creating logical port openshift-console_downloads-5c54f4944f-glzxh for pod on switch ip-10-0-1-98.ec2.internal I0514 23:14:58.494727 3702 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-glzxh I0514 23:14:58.539631 3702 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal I0514 23:14:58.539665 3702 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 40.09µs I0514 23:14:58.545370 3702 pod.go:62] [openshift-console/downloads-5c54f4944f-glzxh] pod update took 50.677862ms I0514 23:14:58.545391 3702 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-glzxh] addLogicalPort annotation time took 50.706625ms I0514 23:14:58.546662 3702 pods.go:271] [openshift-console/downloads-5c54f4944f-glzxh] addLogicalPort took 52.066325ms, libovsdb time 870.071µs I0514 23:14:58.679385 3702 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mn7g8 ee7f271f0c8b5b4b22323328d1e9cb13cdfa1e47fadc7198f495b83873fecdea network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mn7g8 ee7f271f0c8b5b4b22323328d1e9cb13cdfa1e47fadc7198f495b83873fecdea network default NAD default NAD key ] I0514 23:14:58.688819 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-mn7g8, hostIfaceName: ee7f271f0c8b5b4, network: default, NAD default, SandboxID: "ee7f271f0c8b5b4b22323328d1e9cb13cdfa1e47fadc7198f495b83873fecdea", PCI device ID: , UID: "109b6a76-b86d-45c8-b6c4-e1f68b49019a", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0514 23:14:58.740978 3702 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mn7g8 ee7f271f0c8b5b4b22323328d1e9cb13cdfa1e47fadc7198f495b83873fecdea network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mn7g8 ee7f271f0c8b5b4b22323328d1e9cb13cdfa1e47fadc7198f495b83873fecdea network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:91:a7:36:68:3a\",\"name\":\"ee7f271f0c8b5b4\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c971e0e2-1f5f-48a6-a59f-a6c58e287627\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:59.083780 3702 cni.go:452] [openshift-insights/insights-runtime-extractor-dhjdf 11da79ea74498779a664e53f1d56c475024dc337081eb88c29b6d6cc461f69cc network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-dhjdf 11da79ea74498779a664e53f1d56c475024dc337081eb88c29b6d6cc461f69cc network default NAD default NAD key ] I0514 23:14:59.093508 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-dhjdf, hostIfaceName: 11da79ea7449877, network: default, NAD default, SandboxID: "11da79ea74498779a664e53f1d56c475024dc337081eb88c29b6d6cc461f69cc", PCI device ID: , UID: "a8847f38-c7b1-4b93-98c3-c28178822470", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0514 23:14:59.145283 3702 cni.go:473] [openshift-insights/insights-runtime-extractor-dhjdf 11da79ea74498779a664e53f1d56c475024dc337081eb88c29b6d6cc461f69cc network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-dhjdf 11da79ea74498779a664e53f1d56c475024dc337081eb88c29b6d6cc461f69cc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:d2:7c:69:2f:cd\",\"name\":\"11da79ea7449877\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2f735626-0c30-422c-b143-b7f77a651f12\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:14:59.221274 3702 cni.go:452] [openshift-console/downloads-5c54f4944f-glzxh 1dee76f039a0e7727c11047890274dd3bad4435b11ef4b33e5c562d681831971 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-glzxh 1dee76f039a0e7727c11047890274dd3bad4435b11ef4b33e5c562d681831971 network default NAD default NAD key ] I0514 23:14:59.231191 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-glzxh, hostIfaceName: 1dee76f039a0e77, network: default, NAD default, SandboxID: "1dee76f039a0e7727c11047890274dd3bad4435b11ef4b33e5c562d681831971", PCI device ID: , UID: "cb8d5e21-9fd9-47b6-a867-83b34d162643", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0514 23:14:59.285226 3702 cni.go:473] [openshift-console/downloads-5c54f4944f-glzxh 1dee76f039a0e7727c11047890274dd3bad4435b11ef4b33e5c562d681831971 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-glzxh 1dee76f039a0e7727c11047890274dd3bad4435b11ef4b33e5c562d681831971 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:03:ff:c0:0b:9a\",\"name\":\"1dee76f039a0e77\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7f775cc4-785d-487b-8beb-100045dd24eb\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0514 23:15:00.277474 3702 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0514 23:15:00.515695 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0514 23:15:01.417271 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0514 23:15:03.261845 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0514 23:15:03.367798 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0514 23:15:03.367825 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0514 23:15:03.367831 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0514 23:15:04.316474 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0514 23:15:06.295147 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0514 23:15:06.544582 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0514 23:15:11.031994 3702 base_network_controller_pods.go:487] [default/openshift-console/console-79bf45776b-h4hm6] creating logical port openshift-console_console-79bf45776b-h4hm6 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:15:11.032116 3702 kube.go:256] Updating pod openshift-console/console-79bf45776b-h4hm6 I0514 23:15:11.051420 3702 pod.go:62] [openshift-console/console-79bf45776b-h4hm6] pod update took 19.334079ms I0514 23:15:11.051449 3702 base_network_controller_pods.go:951] [default/openshift-console/console-79bf45776b-h4hm6] addLogicalPort annotation time took 19.368096ms I0514 23:15:11.052818 3702 pods.go:271] [openshift-console/console-79bf45776b-h4hm6] addLogicalPort took 20.846147ms, libovsdb time 902.233µs W0514 23:15:11.059738 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0514 23:15:13.390898 3702 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:13.390928 3702 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:13.390936 3702 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0514 23:15:14.203879 3702 cni.go:452] [openshift-console/console-79bf45776b-h4hm6 2739f1f6a7ab55c390cc276fdbb4cab8f1e0c0946b49c26b0c632eb4d709bcce network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-79bf45776b-h4hm6 2739f1f6a7ab55c390cc276fdbb4cab8f1e0c0946b49c26b0c632eb4d709bcce network default NAD default NAD key ] I0514 23:15:14.214128 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-79bf45776b-h4hm6, hostIfaceName: 2739f1f6a7ab55c, network: default, NAD default, SandboxID: "2739f1f6a7ab55c390cc276fdbb4cab8f1e0c0946b49c26b0c632eb4d709bcce", PCI device ID: , UID: "14e8f421-e47f-4b53-9b99-f93e41628195", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0514 23:15:14.269388 3702 cni.go:473] [openshift-console/console-79bf45776b-h4hm6 2739f1f6a7ab55c390cc276fdbb4cab8f1e0c0946b49c26b0c632eb4d709bcce network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-79bf45776b-h4hm6 2739f1f6a7ab55c390cc276fdbb4cab8f1e0c0946b49c26b0c632eb4d709bcce network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:44:29:5d:77:29\",\"name\":\"2739f1f6a7ab55c\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b03f110b-3134-4766-9dfb-dcee59e84edd\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0514 23:15:14.454477 3702 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0514 23:15:14.573635 3702 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0514 23:15:14.600663 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0514 23:15:14.600694 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0514 23:15:15.510036 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0514 23:15:16.269913 3702 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-79bf45776b-fn62n IP address from the namespace address-set, err: pod openshift-console/console-79bf45776b-fn62n: no pod IPs found W0514 23:15:16.377420 3702 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:15:16.377448 3702 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:15:18.414509 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0514 23:15:18.480013 3702 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:18.480043 3702 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:18.480051 3702 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:18.480055 3702 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:19.644619 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0514 23:15:19.720855 3702 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:15:19.720875 3702 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:15:19.864235 3702 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:19.864257 3702 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:19.864262 3702 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:21.651103 3702 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:21.651121 3702 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:21.651126 3702 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:21.651130 3702 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0514 23:15:22.169206 3702 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0514 23:15:23.412292 3702 obj_retry.go:498] Detected object openshift-image-registry/image-registry-55bc57d65c-mbggh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:23.412897 3702 obj_retry.go:498] Detected object openshift-image-registry/image-registry-55bc57d65c-mbggh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:23.692538 3702 obj_retry.go:492] Detected object openshift-image-registry/image-registry-55bc57d65c-mbggh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:23.692562 3702 obj_retry.go:492] Detected object openshift-image-registry/image-registry-55bc57d65c-mbggh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:23.695699 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-55bc57d65c-mbggh I0514 23:15:23.695723 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-55bc57d65c-mbggh I0514 23:15:24.075938 3702 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7] creating logical port openshift-deployment-validation-operator_99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:15:24.076044 3702 kube.go:256] Updating pod openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 I0514 23:15:24.088909 3702 pod.go:62] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7] pod update took 12.893508ms I0514 23:15:24.088935 3702 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7] addLogicalPort annotation time took 12.923991ms I0514 23:15:24.090279 3702 pods.go:271] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7] addLogicalPort took 14.362446ms, libovsdb time 919.06µs I0514 23:15:24.441037 3702 cni.go:452] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 55c5a4cb449b1c3c75f20e82f729a4016d1c478477483b63e42fd80111f233c9 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 55c5a4cb449b1c3c75f20e82f729a4016d1c478477483b63e42fd80111f233c9 network default NAD default NAD key ] I0514 23:15:24.450410 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: 99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7, hostIfaceName: 55c5a4cb449b1c3, network: default, NAD default, SandboxID: "55c5a4cb449b1c3c75f20e82f729a4016d1c478477483b63e42fd80111f233c9", PCI device ID: , UID: "8645f741-8520-43a0-a0d5-140be5620dc6", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0514 23:15:24.503088 3702 cni.go:473] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 55c5a4cb449b1c3c75f20e82f729a4016d1c478477483b63e42fd80111f233c9 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 55c5a4cb449b1c3c75f20e82f729a4016d1c478477483b63e42fd80111f233c9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d2:79:67:0a:67:69\",\"name\":\"55c5a4cb449b1c3\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/56eaa4cf-1a24-4065-a61e-aae1dafc8d20\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:15:36.384837 3702 obj_retry.go:498] Detected object openshift-console/console-b57f75797-f2pc2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:36.385394 3702 obj_retry.go:498] Detected object openshift-console/console-b57f75797-f2pc2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:36.779087 3702 obj_retry.go:492] Detected object openshift-console/console-b57f75797-f2pc2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:36.779110 3702 obj_retry.go:492] Detected object openshift-console/console-b57f75797-f2pc2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:36.786860 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-b57f75797-f2pc2 I0514 23:15:36.786882 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-b57f75797-f2pc2 W0514 23:15:37.574092 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0514 23:15:37.574098 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0514 23:15:37.574264 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0514 23:15:38.580075 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0514 23:15:42.761994 3702 cni.go:452] [openshift-console/console-79bf45776b-h4hm6 2739f1f6a7ab55c390cc276fdbb4cab8f1e0c0946b49c26b0c632eb4d709bcce network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-79bf45776b-h4hm6 2739f1f6a7ab55c390cc276fdbb4cab8f1e0c0946b49c26b0c632eb4d709bcce network default NAD default NAD key ] I0514 23:15:42.850038 3702 cni.go:473] [openshift-console/console-79bf45776b-h4hm6 2739f1f6a7ab55c390cc276fdbb4cab8f1e0c0946b49c26b0c632eb4d709bcce network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-79bf45776b-h4hm6 2739f1f6a7ab55c390cc276fdbb4cab8f1e0c0946b49c26b0c632eb4d709bcce network default NAD default NAD key default], result "{}", err I0514 23:15:42.909841 3702 obj_retry.go:498] Detected object openshift-console/console-79bf45776b-h4hm6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:42.909884 3702 pods.go:176] Deleting pod: openshift-console/console-79bf45776b-h4hm6 I0514 23:15:42.911209 3702 pods.go:236] Attempting to release IPs for pod: openshift-console/console-79bf45776b-h4hm6, ips: 10.128.0.29 I0514 23:15:42.911237 3702 obj_retry.go:498] Detected object openshift-console/console-79bf45776b-h4hm6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:42.921540 3702 base_network_controller_pods.go:487] [default/openshift-console/console-7bf857b667-s2qrk] creating logical port openshift-console_console-7bf857b667-s2qrk for pod on switch ip-10-0-1-98.ec2.internal I0514 23:15:42.921647 3702 kube.go:256] Updating pod openshift-console/console-7bf857b667-s2qrk I0514 23:15:42.935404 3702 pod.go:62] [openshift-console/console-7bf857b667-s2qrk] pod update took 13.776451ms I0514 23:15:42.935428 3702 base_network_controller_pods.go:951] [default/openshift-console/console-7bf857b667-s2qrk] addLogicalPort annotation time took 13.808184ms I0514 23:15:42.936683 3702 pods.go:271] [openshift-console/console-7bf857b667-s2qrk] addLogicalPort took 15.154757ms, libovsdb time 854.664µs I0514 23:15:43.315806 3702 cni.go:452] [openshift-console/console-7bf857b667-s2qrk 5d6ffbb1ec94601458e7287b9f7d060e5fb84325c9de5868dd63fd9c8366ddcb network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7bf857b667-s2qrk 5d6ffbb1ec94601458e7287b9f7d060e5fb84325c9de5868dd63fd9c8366ddcb network default NAD default NAD key ] I0514 23:15:43.325312 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7bf857b667-s2qrk, hostIfaceName: 5d6ffbb1ec94601, network: default, NAD default, SandboxID: "5d6ffbb1ec94601458e7287b9f7d060e5fb84325c9de5868dd63fd9c8366ddcb", PCI device ID: , UID: "dae91a11-7ce9-4fad-b4ff-23ec4e9dbcea", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0514 23:15:43.378838 3702 cni.go:473] [openshift-console/console-7bf857b667-s2qrk 5d6ffbb1ec94601458e7287b9f7d060e5fb84325c9de5868dd63fd9c8366ddcb network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7bf857b667-s2qrk 5d6ffbb1ec94601458e7287b9f7d060e5fb84325c9de5868dd63fd9c8366ddcb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:b0:e7:93:5a:df\",\"name\":\"5d6ffbb1ec94601\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/969a12b6-7097-42f5-b7fe-6db95e1bdf1e\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:15:43.641052 3702 obj_retry.go:492] Detected object openshift-console/console-79bf45776b-h4hm6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:43.641078 3702 obj_retry.go:492] Detected object openshift-console/console-79bf45776b-h4hm6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:43.648060 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-79bf45776b-h4hm6 I0514 23:15:43.648093 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-79bf45776b-h4hm6 W0514 23:15:43.678356 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0514 23:15:43.678419 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0514 23:15:43.678440 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0514 23:15:44.620433 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0514 23:15:44.623201 3702 cni.go:452] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 55c5a4cb449b1c3c75f20e82f729a4016d1c478477483b63e42fd80111f233c9 network default NAD default NAD key ] DEL starting CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 55c5a4cb449b1c3c75f20e82f729a4016d1c478477483b63e42fd80111f233c9 network default NAD default NAD key ] I0514 23:15:44.710693 3702 cni.go:473] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 55c5a4cb449b1c3c75f20e82f729a4016d1c478477483b63e42fd80111f233c9 network default NAD default NAD key default] DEL finished CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 55c5a4cb449b1c3c75f20e82f729a4016d1c478477483b63e42fd80111f233c9 network default NAD default NAD key default], result "{}", err I0514 23:15:44.777060 3702 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:44.777100 3702 pods.go:176] Deleting pod: openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 I0514 23:15:44.778526 3702 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7, ips: 10.128.0.30 I0514 23:15:44.778556 3702 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:44.778578 3702 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:15:44.778589 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0514 23:15:44.778798 3702 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:15:44.778814 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0514 23:15:45.815535 3702 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:45.815562 3702 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:45.815583 3702 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:45.815598 3702 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:46.535361 3702 cni.go:452] [openshift-image-registry/image-registry-d477cf66b-ddgmf 1d3925a3966680bd775d22b5137f9197f4a494ceef821270c1ea48250250f46e network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-d477cf66b-ddgmf 1d3925a3966680bd775d22b5137f9197f4a494ceef821270c1ea48250250f46e network default NAD default NAD key ] I0514 23:15:46.625689 3702 cni.go:473] [openshift-image-registry/image-registry-d477cf66b-ddgmf 1d3925a3966680bd775d22b5137f9197f4a494ceef821270c1ea48250250f46e network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-d477cf66b-ddgmf 1d3925a3966680bd775d22b5137f9197f4a494ceef821270c1ea48250250f46e network default NAD default NAD key default], result "{}", err I0514 23:15:46.690218 3702 obj_retry.go:498] Detected object openshift-image-registry/image-registry-d477cf66b-ddgmf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:46.690255 3702 pods.go:176] Deleting pod: openshift-image-registry/image-registry-d477cf66b-ddgmf I0514 23:15:46.691591 3702 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-d477cf66b-ddgmf, ips: 10.128.0.25 I0514 23:15:46.691617 3702 obj_retry.go:498] Detected object openshift-image-registry/image-registry-d477cf66b-ddgmf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0514 23:15:47.527540 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0514 23:15:47.527572 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0514 23:15:47.576753 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0514 23:15:47.576778 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0514 23:15:47.630230 3702 obj_retry.go:492] Detected object openshift-image-registry/image-registry-d477cf66b-ddgmf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:47.630253 3702 obj_retry.go:492] Detected object openshift-image-registry/image-registry-d477cf66b-ddgmf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:47.634988 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-d477cf66b-ddgmf I0514 23:15:47.635012 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-d477cf66b-ddgmf I0514 23:15:49.153716 3702 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0514 23:15:49.163490 3702 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0514 23:15:53.676079 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0514 23:15:54.312712 3702 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0514 23:16:02.710114 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0514 23:16:02.710118 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0514 23:16:02.710169 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0514 23:16:03.716802 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0514 23:16:08.249313 3702 cni.go:452] [openshift-ingress/router-default-7f4c9f6df5-xjsx9 3203d1489f6412e3e7cd5a594f71dd93b449a08ff805ccd929ff2684f52b1c66 network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-7f4c9f6df5-xjsx9 3203d1489f6412e3e7cd5a594f71dd93b449a08ff805ccd929ff2684f52b1c66 network default NAD default NAD key ] I0514 23:16:08.334024 3702 cni.go:473] [openshift-ingress/router-default-7f4c9f6df5-xjsx9 3203d1489f6412e3e7cd5a594f71dd93b449a08ff805ccd929ff2684f52b1c66 network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-7f4c9f6df5-xjsx9 3203d1489f6412e3e7cd5a594f71dd93b449a08ff805ccd929ff2684f52b1c66 network default NAD default NAD key default], result "{}", err I0514 23:16:08.401039 3702 obj_retry.go:498] Detected object openshift-ingress/router-default-7f4c9f6df5-xjsx9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:08.401070 3702 pods.go:176] Deleting pod: openshift-ingress/router-default-7f4c9f6df5-xjsx9 I0514 23:16:08.402461 3702 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-7f4c9f6df5-xjsx9, ips: 10.128.0.14 I0514 23:16:08.402491 3702 obj_retry.go:498] Detected object openshift-ingress/router-default-7f4c9f6df5-xjsx9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:08.731650 3702 obj_retry.go:492] Detected object openshift-ingress/router-default-7f4c9f6df5-xjsx9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:08.731680 3702 obj_retry.go:492] Detected object openshift-ingress/router-default-7f4c9f6df5-xjsx9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:08.735314 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-7f4c9f6df5-xjsx9 I0514 23:16:08.735336 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-7f4c9f6df5-xjsx9 I0514 23:16:12.879529 3702 obj_retry.go:498] Detected object openshift-console/console-b57f75797-44prb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:12.880090 3702 obj_retry.go:498] Detected object openshift-console/console-b57f75797-44prb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:13.865822 3702 obj_retry.go:492] Detected object openshift-console/console-b57f75797-44prb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:13.865850 3702 obj_retry.go:492] Detected object openshift-console/console-b57f75797-44prb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:13.873944 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-b57f75797-44prb I0514 23:16:13.873966 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-b57f75797-44prb I0514 23:16:30.971436 3702 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:30.972000 3702 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:31.957896 3702 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:31.957920 3702 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:31.968810 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0514 23:16:31.968835 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0514 23:16:35.396438 3702 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:35.396991 3702 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:35.988139 3702 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:35.988166 3702 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:35.996795 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0514 23:16:35.996819 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0514 23:16:53.837610 3702 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:53.838222 3702 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:54.786702 3702 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:54.786726 3702 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:54.793903 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0514 23:16:54.793923 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0514 23:17:03.422914 3702 obj_retry.go:498] Detected object openshift-console/console-7bf857b667-hvfs6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:03.423478 3702 obj_retry.go:498] Detected object openshift-console/console-7bf857b667-hvfs6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:03.830217 3702 obj_retry.go:492] Detected object openshift-console/console-7bf857b667-hvfs6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:03.830242 3702 obj_retry.go:492] Detected object openshift-console/console-7bf857b667-hvfs6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:03.837801 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7bf857b667-hvfs6 I0514 23:17:03.837826 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7bf857b667-hvfs6 I0514 23:17:14.275484 3702 cni.go:452] [openshift-console/console-7bf857b667-s2qrk 5d6ffbb1ec94601458e7287b9f7d060e5fb84325c9de5868dd63fd9c8366ddcb network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7bf857b667-s2qrk 5d6ffbb1ec94601458e7287b9f7d060e5fb84325c9de5868dd63fd9c8366ddcb network default NAD default NAD key ] I0514 23:17:14.367437 3702 cni.go:473] [openshift-console/console-7bf857b667-s2qrk 5d6ffbb1ec94601458e7287b9f7d060e5fb84325c9de5868dd63fd9c8366ddcb network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7bf857b667-s2qrk 5d6ffbb1ec94601458e7287b9f7d060e5fb84325c9de5868dd63fd9c8366ddcb network default NAD default NAD key default], result "{}", err I0514 23:17:14.414209 3702 obj_retry.go:498] Detected object openshift-console/console-7bf857b667-s2qrk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:14.414258 3702 pods.go:176] Deleting pod: openshift-console/console-7bf857b667-s2qrk I0514 23:17:14.415490 3702 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7bf857b667-s2qrk, ips: 10.128.0.31 I0514 23:17:14.415518 3702 obj_retry.go:498] Detected object openshift-console/console-7bf857b667-s2qrk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:15.020311 3702 obj_retry.go:492] Detected object openshift-console/console-7bf857b667-s2qrk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:15.020339 3702 obj_retry.go:492] Detected object openshift-console/console-7bf857b667-s2qrk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:15.023520 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7bf857b667-s2qrk I0514 23:17:15.023552 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7bf857b667-s2qrk I0514 23:17:29.089711 3702 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp 01ba6dd4cf8a75e7eb71292aa8a2d840f7931da0c7c3de35c5405a34a65d2b9d network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp 01ba6dd4cf8a75e7eb71292aa8a2d840f7931da0c7c3de35c5405a34a65d2b9d network default NAD default NAD key ] I0514 23:17:29.163108 3702 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp 01ba6dd4cf8a75e7eb71292aa8a2d840f7931da0c7c3de35c5405a34a65d2b9d network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp 01ba6dd4cf8a75e7eb71292aa8a2d840f7931da0c7c3de35c5405a34a65d2b9d network default NAD default NAD key default], result "{}", err I0514 23:17:29.205694 3702 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:29.205733 3702 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp I0514 23:17:29.207076 3702 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp, ips: 10.128.0.17 I0514 23:17:29.207112 3702 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:30.102219 3702 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:30.102243 3702 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:38.597367 3702 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:38.597884 3702 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:38.971121 3702 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:38.971143 3702 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:38.979639 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0514 23:17:38.979663 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0514 23:18:50.827826 3702 base_network_controller_pods.go:487] [default/openshift-console/console-58f495ddb-kcpwk] creating logical port openshift-console_console-58f495ddb-kcpwk for pod on switch ip-10-0-1-98.ec2.internal I0514 23:18:50.827974 3702 kube.go:256] Updating pod openshift-console/console-58f495ddb-kcpwk I0514 23:18:50.853549 3702 pod.go:62] [openshift-console/console-58f495ddb-kcpwk] pod update took 25.601008ms I0514 23:18:50.853582 3702 base_network_controller_pods.go:951] [default/openshift-console/console-58f495ddb-kcpwk] addLogicalPort annotation time took 25.640163ms I0514 23:18:50.855040 3702 pods.go:271] [openshift-console/console-58f495ddb-kcpwk] addLogicalPort took 27.233947ms, libovsdb time 1.023337ms I0514 23:18:51.188988 3702 cni.go:452] [openshift-console/console-58f495ddb-kcpwk 8f3c3a0daf2794861ca9f64ceb916f8e2380167e92a1a758d95d0abddc9ae2aa network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-58f495ddb-kcpwk 8f3c3a0daf2794861ca9f64ceb916f8e2380167e92a1a758d95d0abddc9ae2aa network default NAD default NAD key ] I0514 23:18:51.198790 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-58f495ddb-kcpwk, hostIfaceName: 8f3c3a0daf27948, network: default, NAD default, SandboxID: "8f3c3a0daf2794861ca9f64ceb916f8e2380167e92a1a758d95d0abddc9ae2aa", PCI device ID: , UID: "c334e71d-0d0b-4504-98be-f3c3b531293f", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0514 23:18:51.249087 3702 cni.go:473] [openshift-console/console-58f495ddb-kcpwk 8f3c3a0daf2794861ca9f64ceb916f8e2380167e92a1a758d95d0abddc9ae2aa network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-58f495ddb-kcpwk 8f3c3a0daf2794861ca9f64ceb916f8e2380167e92a1a758d95d0abddc9ae2aa network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:d4:9e:89:36:89\",\"name\":\"8f3c3a0daf27948\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/08eb5f21-a49d-4810-92f9-adcacbf8a353\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0514 23:19:01.464688 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0514 23:19:16.138262 3702 obj_retry.go:498] Detected object openshift-console/console-6dfb89476d-28rdd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:19:16.138802 3702 obj_retry.go:498] Detected object openshift-console/console-6dfb89476d-28rdd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:19:16.331808 3702 obj_retry.go:492] Detected object openshift-console/console-6dfb89476d-28rdd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:19:16.331839 3702 obj_retry.go:492] Detected object openshift-console/console-6dfb89476d-28rdd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:19:16.339599 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6dfb89476d-28rdd I0514 23:19:16.339624 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6dfb89476d-28rdd I0514 23:19:26.805989 3702 obj_retry.go:498] Detected object openshift-console/console-6dfb89476d-ktf8r of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:19:26.806616 3702 obj_retry.go:498] Detected object openshift-console/console-6dfb89476d-ktf8r of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:19:27.712474 3702 obj_retry.go:492] Detected object openshift-console/console-6dfb89476d-ktf8r of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:19:27.712498 3702 obj_retry.go:492] Detected object openshift-console/console-6dfb89476d-ktf8r of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:19:27.722951 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6dfb89476d-ktf8r I0514 23:19:27.722981 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6dfb89476d-ktf8r I0514 23:19:32.143910 3702 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0514 23:19:42.061527 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0514 23:19:51.066773 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0514 23:20:00.173627 3702 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29646680-5fcxh for pod on switch ip-10-0-1-98.ec2.internal I0514 23:20:00.173724 3702 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh I0514 23:20:00.187855 3702 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh] pod update took 14.152293ms I0514 23:20:00.187884 3702 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh] addLogicalPort annotation time took 14.185749ms I0514 23:20:00.189157 3702 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh] addLogicalPort took 15.545734ms, libovsdb time 826.38µs I0514 23:20:00.549672 3702 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh d2afb291b9eb97b608b273757b73051762d9e3e2253bad58df003e14d528f518 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh d2afb291b9eb97b608b273757b73051762d9e3e2253bad58df003e14d528f518 network default NAD default NAD key ] I0514 23:20:00.559251 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29646680-5fcxh, hostIfaceName: d2afb291b9eb97b, network: default, NAD default, SandboxID: "d2afb291b9eb97b608b273757b73051762d9e3e2253bad58df003e14d528f518", PCI device ID: , UID: "0f9b1c98-359c-48fd-9282-280f8362eaf1", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0514 23:20:00.609831 3702 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh d2afb291b9eb97b608b273757b73051762d9e3e2253bad58df003e14d528f518 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh d2afb291b9eb97b608b273757b73051762d9e3e2253bad58df003e14d528f518 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:36:4e:b5:77:34\",\"name\":\"d2afb291b9eb97b\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f8749bc8-19c8-4282-a4c3-c8b71a3f5dc6\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:20:02.178139 3702 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0514 23:20:03.732383 3702 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh d2afb291b9eb97b608b273757b73051762d9e3e2253bad58df003e14d528f518 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh d2afb291b9eb97b608b273757b73051762d9e3e2253bad58df003e14d528f518 network default NAD default NAD key ] I0514 23:20:03.804285 3702 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh d2afb291b9eb97b608b273757b73051762d9e3e2253bad58df003e14d528f518 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh d2afb291b9eb97b608b273757b73051762d9e3e2253bad58df003e14d528f518 network default NAD default NAD key default], result "{}", err I0514 23:20:03.846300 3702 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:20:03.846343 3702 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh I0514 23:20:03.847982 3702 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh, ips: 10.128.0.33 I0514 23:20:03.848020 3702 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:20:04.742550 3702 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:20:04.742575 3702 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:20:13.154303 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0514 23:20:19.171947 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0514 23:20:26.063523 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0514 23:20:50.439929 3702 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-x4v82] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-5d7684fb5d-x4v82 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:20:50.440061 3702 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-x4v82 I0514 23:20:50.455929 3702 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-x4v82] pod update took 15.898198ms I0514 23:20:50.455954 3702 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-x4v82] addLogicalPort annotation time took 15.929154ms I0514 23:20:50.457301 3702 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-x4v82] addLogicalPort took 17.386181ms, libovsdb time 938.248µs I0514 23:20:50.798338 3702 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-x4v82 4782bb0b89080b829f3bfbd549d7b20eab6477173cabca05ad671a6e9968d0e6 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-x4v82 4782bb0b89080b829f3bfbd549d7b20eab6477173cabca05ad671a6e9968d0e6 network default NAD default NAD key ] I0514 23:20:50.807652 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-5d7684fb5d-x4v82, hostIfaceName: 4782bb0b89080b8, network: default, NAD default, SandboxID: "4782bb0b89080b829f3bfbd549d7b20eab6477173cabca05ad671a6e9968d0e6", PCI device ID: , UID: "cc6db623-e597-444c-9c6c-9448b6b9baf0", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0514 23:20:50.860732 3702 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-x4v82 4782bb0b89080b829f3bfbd549d7b20eab6477173cabca05ad671a6e9968d0e6 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-x4v82 4782bb0b89080b829f3bfbd549d7b20eab6477173cabca05ad671a6e9968d0e6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:8e:3c:bf:fe:6a\",\"name\":\"4782bb0b89080b8\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8446319f-e44b-42ea-9a61-c9326a2e5551\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:20:52.160651 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0514 23:20:53.204795 3702 namespace.go:98] [tsf] adding namespace I0514 23:20:53.205678 3702 namespace.go:102] [tsf] adding namespace took 862.729µs I0514 23:20:53.229547 3702 namespace.go:142] [tsf] updating namespace I0514 23:20:53.237396 3702 namespace.go:142] [tsf] updating namespace W0514 23:20:53.952060 3702 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:20:57.613301 3702 namespace.go:98] [tsf-keycloak] adding namespace I0514 23:20:57.614152 3702 namespace.go:102] [tsf-keycloak] adding namespace took 828.576µs I0514 23:20:57.618265 3702 namespace.go:98] [konflux-operator] adding namespace I0514 23:20:57.618280 3702 namespace.go:98] [rhtpa-operator] adding namespace I0514 23:20:57.618492 3702 namespace.go:98] [tsf-tpa] adding namespace I0514 23:20:57.618507 3702 namespace.go:98] [konflux-ui] adding namespace I0514 23:20:57.618644 3702 namespace.go:98] [cert-manager-operator] adding namespace I0514 23:20:57.618797 3702 namespace.go:98] [rhbk-operator] adding namespace I0514 23:20:57.619051 3702 namespace.go:102] [konflux-operator] adding namespace took 768.074µs I0514 23:20:57.619700 3702 namespace.go:102] [rhtpa-operator] adding namespace took 1.407536ms I0514 23:20:57.620320 3702 namespace.go:98] [tsf-tas] adding namespace I0514 23:20:57.620346 3702 namespace.go:102] [tsf-tpa] adding namespace took 1.837105ms I0514 23:20:57.620946 3702 namespace.go:102] [konflux-ui] adding namespace took 2.422838ms I0514 23:20:57.621629 3702 namespace.go:102] [cert-manager-operator] adding namespace took 2.968243ms I0514 23:20:57.622194 3702 namespace.go:102] [rhbk-operator] adding namespace took 3.385647ms I0514 23:20:57.622721 3702 namespace.go:102] [tsf-tas] adding namespace took 2.388881ms I0514 23:20:57.632321 3702 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:20:57.645305 3702 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:20:57.667517 3702 namespace.go:142] [tsf-tpa] updating namespace I0514 23:20:57.667569 3702 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:20:57.667685 3702 namespace.go:142] [konflux-operator] updating namespace I0514 23:20:57.671740 3702 namespace.go:142] [konflux-ui] updating namespace I0514 23:20:57.680752 3702 namespace.go:142] [konflux-operator] updating namespace I0514 23:20:57.683716 3702 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:20:57.683766 3702 namespace.go:142] [rhbk-operator] updating namespace I0514 23:20:57.686861 3702 namespace.go:142] [tsf-tas] updating namespace I0514 23:20:57.688412 3702 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:20:57.713548 3702 namespace.go:142] [tsf-tpa] updating namespace I0514 23:20:57.729432 3702 namespace.go:142] [tsf-tpa] updating namespace I0514 23:20:57.732015 3702 namespace.go:142] [konflux-ui] updating namespace I0514 23:20:57.744335 3702 namespace.go:142] [konflux-ui] updating namespace I0514 23:20:57.750840 3702 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:20:57.766639 3702 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:20:57.770442 3702 namespace.go:142] [rhbk-operator] updating namespace I0514 23:20:57.781463 3702 namespace.go:142] [rhbk-operator] updating namespace I0514 23:20:57.787745 3702 namespace.go:142] [tsf-tas] updating namespace I0514 23:20:57.799394 3702 namespace.go:142] [tsf-tas] updating namespace I0514 23:20:57.805383 3702 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:20:57.818087 3702 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:20:57.834414 3702 namespace.go:142] [konflux-operator] updating namespace I0514 23:20:58.812589 3702 namespace.go:142] [tsf-tpa] updating namespace I0514 23:20:58.819423 3702 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:20:58.978563 3702 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-1-98.ec2.internal I0514 23:20:58.978689 3702 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0514 23:20:59.000233 3702 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 21.567215ms I0514 23:20:59.000264 3702 base_network_controller_pods.go:951] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 21.602624ms I0514 23:20:59.001611 3702 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 23.063377ms, libovsdb time 885.854µs I0514 23:20:59.354091 3702 cni.go:452] [tsf/test-tsf-subscriptions 212e47c3b1727ac04b0dded28a708171698888bb63476da035e21aca4ce16561 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-subscriptions 212e47c3b1727ac04b0dded28a708171698888bb63476da035e21aca4ce16561 network default NAD default NAD key ] I0514 23:20:59.363408 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: 212e47c3b1727ac, network: default, NAD default, SandboxID: "212e47c3b1727ac04b0dded28a708171698888bb63476da035e21aca4ce16561", PCI device ID: , UID: "dbd3fbb1-3205-4e6c-a4c1-fb4b408c757f", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0514 23:20:59.416599 3702 cni.go:473] [tsf/test-tsf-subscriptions 212e47c3b1727ac04b0dded28a708171698888bb63476da035e21aca4ce16561 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-subscriptions 212e47c3b1727ac04b0dded28a708171698888bb63476da035e21aca4ce16561 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:5c:85:20:96:36\",\"name\":\"212e47c3b1727ac\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3331514e-07ef-461f-9987-17d426f04936\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:21:00.182910 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0514 23:21:00.274060 3702 base_network_controller_pods.go:487] [default/openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db] creating logical port openshift-marketplace_2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db for pod on switch ip-10-0-1-98.ec2.internal I0514 23:21:00.274288 3702 kube.go:256] Updating pod openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db I0514 23:21:00.288055 3702 pod.go:62] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db] pod update took 13.799151ms I0514 23:21:00.288086 3702 base_network_controller_pods.go:951] [default/openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db] addLogicalPort annotation time took 13.833707ms I0514 23:21:00.289442 3702 pods.go:271] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db] addLogicalPort took 15.404111ms, libovsdb time 898.629µs I0514 23:21:00.376558 3702 base_network_controller_pods.go:487] [default/openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd] creating logical port openshift-marketplace_79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd for pod on switch ip-10-0-1-98.ec2.internal I0514 23:21:00.376722 3702 kube.go:256] Updating pod openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd I0514 23:21:00.389436 3702 pod.go:62] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd] pod update took 12.758433ms I0514 23:21:00.389468 3702 base_network_controller_pods.go:951] [default/openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd] addLogicalPort annotation time took 12.796019ms I0514 23:21:00.390927 3702 pods.go:271] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd] addLogicalPort took 14.39688ms, libovsdb time 931.113µs I0514 23:21:01.046945 3702 cni.go:452] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db e93728a54b3d410c6e0109367671a4b074ffd0c82e28857f74b449761f52d320 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db e93728a54b3d410c6e0109367671a4b074ffd0c82e28857f74b449761f52d320 network default NAD default NAD key ] I0514 23:21:01.047066 3702 cni.go:452] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd 48938a37fc962b76d7498f8785131760caec167c8a1113fd7d4ea512b795d8bd network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd 48938a37fc962b76d7498f8785131760caec167c8a1113fd7d4ea512b795d8bd network default NAD default NAD key ] I0514 23:21:01.058586 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd, hostIfaceName: 48938a37fc962b7, network: default, NAD default, SandboxID: "48938a37fc962b76d7498f8785131760caec167c8a1113fd7d4ea512b795d8bd", PCI device ID: , UID: "f4fe6252-97a7-4bf2-9b2c-59d6208a9f25", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0514 23:21:01.058632 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db, hostIfaceName: e93728a54b3d410, network: default, NAD default, SandboxID: "e93728a54b3d410c6e0109367671a4b074ffd0c82e28857f74b449761f52d320", PCI device ID: , UID: "8071a97f-7a74-4bde-8058-ffdf087753b1", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0514 23:21:01.112308 3702 cni.go:473] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd 48938a37fc962b76d7498f8785131760caec167c8a1113fd7d4ea512b795d8bd network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd 48938a37fc962b76d7498f8785131760caec167c8a1113fd7d4ea512b795d8bd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:53:9f:7b:3c:ee\",\"name\":\"48938a37fc962b7\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7a263ed8-f3b0-4375-8108-bc782df03402\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:21:01.132126 3702 cni.go:473] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db e93728a54b3d410c6e0109367671a4b074ffd0c82e28857f74b449761f52d320 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db e93728a54b3d410c6e0109367671a4b074ffd0c82e28857f74b449761f52d320 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:d4:41:f2:db:0b\",\"name\":\"e93728a54b3d410\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/22aca7b3-f646-469b-b9c0-809010a8526c\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:21:01.166894 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0514 23:21:04.930820 3702 base_network_controller_pods.go:487] [default/openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw] creating logical port openshift-marketplace_5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw for pod on switch ip-10-0-1-98.ec2.internal I0514 23:21:04.931013 3702 kube.go:256] Updating pod openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw I0514 23:21:04.944595 3702 pod.go:62] [openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw] pod update took 13.615116ms I0514 23:21:04.944625 3702 base_network_controller_pods.go:951] [default/openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw] addLogicalPort annotation time took 13.654088ms I0514 23:21:04.946210 3702 pods.go:271] [openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw] addLogicalPort took 15.409299ms, libovsdb time 1.103684ms W0514 23:21:04.961162 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:21:05.005511 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0514 23:21:05.005693 3702 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-l9rdx IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-l9rdx: no pod IPs found I0514 23:21:05.033743 3702 cni.go:452] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd 48938a37fc962b76d7498f8785131760caec167c8a1113fd7d4ea512b795d8bd network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd 48938a37fc962b76d7498f8785131760caec167c8a1113fd7d4ea512b795d8bd network default NAD default NAD key ] I0514 23:21:05.120469 3702 cni.go:473] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd 48938a37fc962b76d7498f8785131760caec167c8a1113fd7d4ea512b795d8bd network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd 48938a37fc962b76d7498f8785131760caec167c8a1113fd7d4ea512b795d8bd network default NAD default NAD key default], result "{}", err I0514 23:21:05.122088 3702 base_network_controller_pods.go:487] [default/openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv] creating logical port openshift-marketplace_5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv for pod on switch ip-10-0-1-98.ec2.internal I0514 23:21:05.122287 3702 kube.go:256] Updating pod openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv I0514 23:21:05.137366 3702 pod.go:62] [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv] pod update took 15.115464ms I0514 23:21:05.137461 3702 base_network_controller_pods.go:951] [default/openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv] addLogicalPort annotation time took 15.214721ms I0514 23:21:05.138881 3702 pods.go:271] [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv] addLogicalPort took 16.809228ms, libovsdb time 1.004404ms I0514 23:21:06.263502 3702 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:06.263542 3702 pods.go:176] Deleting pod: openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd I0514 23:21:06.265008 3702 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd, ips: 10.128.0.37 I0514 23:21:06.265038 3702 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:06.265057 3702 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:06.265065 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:06.265248 3702 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:06.265263 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:07.194964 3702 base_network_controller_pods.go:487] [default/openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg] creating logical port openshift-marketplace_03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg for pod on switch ip-10-0-1-98.ec2.internal I0514 23:21:07.195124 3702 kube.go:256] Updating pod openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg I0514 23:21:07.207733 3702 pod.go:62] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg] pod update took 12.647201ms I0514 23:21:07.207832 3702 base_network_controller_pods.go:951] [default/openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg] addLogicalPort annotation time took 12.750167ms I0514 23:21:07.209207 3702 pods.go:271] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg] addLogicalPort took 14.264662ms, libovsdb time 942.825µs I0514 23:21:07.292671 3702 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:07.292723 3702 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:07.292739 3702 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:07.292755 3702 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:07.295888 3702 base_network_controller_pods.go:487] [default/openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww] creating logical port openshift-marketplace_09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww for pod on switch ip-10-0-1-98.ec2.internal I0514 23:21:07.295985 3702 kube.go:256] Updating pod openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww I0514 23:21:07.307141 3702 pod.go:62] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww] pod update took 11.187223ms I0514 23:21:07.307162 3702 base_network_controller_pods.go:951] [default/openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww] addLogicalPort annotation time took 11.213758ms I0514 23:21:07.309025 3702 pods.go:271] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww] addLogicalPort took 13.149796ms, libovsdb time 1.343365ms I0514 23:21:09.799125 3702 cni.go:452] [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv 7e2dd7dbf331b8b9ce8755d6c81e6fbbb6af57255bac8682b26950500895da08 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv 7e2dd7dbf331b8b9ce8755d6c81e6fbbb6af57255bac8682b26950500895da08 network default NAD default NAD key ] I0514 23:21:09.799153 3702 cni.go:452] [openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw a01845b5a4a851ab4b104514bd16b3aed0a27f6e80dc8b41f75f4f0f519ef0c1 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw a01845b5a4a851ab4b104514bd16b3aed0a27f6e80dc8b41f75f4f0f519ef0c1 network default NAD default NAD key ] I0514 23:21:09.810769 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv, hostIfaceName: 7e2dd7dbf331b8b, network: default, NAD default, SandboxID: "7e2dd7dbf331b8b9ce8755d6c81e6fbbb6af57255bac8682b26950500895da08", PCI device ID: , UID: "0df22ca4-2927-4e10-b1be-160dd7da2927", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0514 23:21:09.811161 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw, hostIfaceName: a01845b5a4a851a, network: default, NAD default, SandboxID: "a01845b5a4a851ab4b104514bd16b3aed0a27f6e80dc8b41f75f4f0f519ef0c1", PCI device ID: , UID: "6d6cd3f9-14d5-499a-a493-d1a1c4bd179b", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0514 23:21:09.813319 3702 cni.go:452] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww cc51bc6d906e64e2a037672ffb3fdb3d1aae7e12ee38b7605d5bd1e993528168 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww cc51bc6d906e64e2a037672ffb3fdb3d1aae7e12ee38b7605d5bd1e993528168 network default NAD default NAD key ] I0514 23:21:09.815240 3702 cni.go:452] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg 707f822179fb9d2b45add00ad34f992da342436e040472e4dfa326556cf04c42 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg 707f822179fb9d2b45add00ad34f992da342436e040472e4dfa326556cf04c42 network default NAD default NAD key ] I0514 23:21:09.829347 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww, hostIfaceName: cc51bc6d906e64e, network: default, NAD default, SandboxID: "cc51bc6d906e64e2a037672ffb3fdb3d1aae7e12ee38b7605d5bd1e993528168", PCI device ID: , UID: "0068b46a-60c1-4b18-91f9-c0d8725ef394", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0514 23:21:09.829511 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg, hostIfaceName: 707f822179fb9d2, network: default, NAD default, SandboxID: "707f822179fb9d2b45add00ad34f992da342436e040472e4dfa326556cf04c42", PCI device ID: , UID: "159f837f-d1b1-47e1-9073-d38cdc5844e5", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0514 23:21:09.880256 3702 cni.go:473] [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv 7e2dd7dbf331b8b9ce8755d6c81e6fbbb6af57255bac8682b26950500895da08 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv 7e2dd7dbf331b8b9ce8755d6c81e6fbbb6af57255bac8682b26950500895da08 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:b2:ee:51:49:49\",\"name\":\"7e2dd7dbf331b8b\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a917bbda-2593-4115-8590-a73af918b774\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:21:09.903362 3702 cni.go:473] [openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw a01845b5a4a851ab4b104514bd16b3aed0a27f6e80dc8b41f75f4f0f519ef0c1 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw a01845b5a4a851ab4b104514bd16b3aed0a27f6e80dc8b41f75f4f0f519ef0c1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:d1:c6:fb:cb:61\",\"name\":\"a01845b5a4a851a\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e071b8eb-1b69-4aae-a697-7ea306610e31\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:21:10.129076 3702 cni.go:473] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg 707f822179fb9d2b45add00ad34f992da342436e040472e4dfa326556cf04c42 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg 707f822179fb9d2b45add00ad34f992da342436e040472e4dfa326556cf04c42 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:f6:ea:56:3f:0f\",\"name\":\"707f822179fb9d2\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/522ee146-15a2-4c56-acb2-a8f44f009946\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:21:10.129114 3702 cni.go:473] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww cc51bc6d906e64e2a037672ffb3fdb3d1aae7e12ee38b7605d5bd1e993528168 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww cc51bc6d906e64e2a037672ffb3fdb3d1aae7e12ee38b7605d5bd1e993528168 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:0c:29:f6:46:13\",\"name\":\"cc51bc6d906e64e\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5a53582f-fcd9-47b7-a7ba-67a66ff2452e\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:21:11.067321 3702 cni.go:452] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db e93728a54b3d410c6e0109367671a4b074ffd0c82e28857f74b449761f52d320 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db e93728a54b3d410c6e0109367671a4b074ffd0c82e28857f74b449761f52d320 network default NAD default NAD key ] I0514 23:21:11.136939 3702 cni.go:473] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db e93728a54b3d410c6e0109367671a4b074ffd0c82e28857f74b449761f52d320 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db e93728a54b3d410c6e0109367671a4b074ffd0c82e28857f74b449761f52d320 network default NAD default NAD key default], result "{}", err I0514 23:21:11.205059 3702 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:11.205102 3702 pods.go:176] Deleting pod: openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db I0514 23:21:11.206677 3702 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db, ips: 10.128.0.36 I0514 23:21:11.206708 3702 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:11.206726 3702 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:11.206733 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:11.206959 3702 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:11.206995 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:12.233955 3702 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:12.233996 3702 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:12.234015 3702 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:12.234029 3702 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:14.100684 3702 cni.go:452] [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv 7e2dd7dbf331b8b9ce8755d6c81e6fbbb6af57255bac8682b26950500895da08 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv 7e2dd7dbf331b8b9ce8755d6c81e6fbbb6af57255bac8682b26950500895da08 network default NAD default NAD key ] I0514 23:21:14.180855 3702 cni.go:473] [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv 7e2dd7dbf331b8b9ce8755d6c81e6fbbb6af57255bac8682b26950500895da08 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv 7e2dd7dbf331b8b9ce8755d6c81e6fbbb6af57255bac8682b26950500895da08 network default NAD default NAD key default], result "{}", err I0514 23:21:14.237377 3702 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:14.237421 3702 pods.go:176] Deleting pod: openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv I0514 23:21:14.238861 3702 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv, ips: 10.128.0.39 I0514 23:21:14.238887 3702 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:14.238908 3702 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:14.238915 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:14.239097 3702 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:14.239119 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:15.108800 3702 cni.go:452] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg 707f822179fb9d2b45add00ad34f992da342436e040472e4dfa326556cf04c42 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg 707f822179fb9d2b45add00ad34f992da342436e040472e4dfa326556cf04c42 network default NAD default NAD key ] I0514 23:21:15.109118 3702 cni.go:452] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww cc51bc6d906e64e2a037672ffb3fdb3d1aae7e12ee38b7605d5bd1e993528168 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww cc51bc6d906e64e2a037672ffb3fdb3d1aae7e12ee38b7605d5bd1e993528168 network default NAD default NAD key ] I0514 23:21:15.109333 3702 cni.go:452] [openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw a01845b5a4a851ab4b104514bd16b3aed0a27f6e80dc8b41f75f4f0f519ef0c1 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw a01845b5a4a851ab4b104514bd16b3aed0a27f6e80dc8b41f75f4f0f519ef0c1 network default NAD default NAD key ] I0514 23:21:15.198346 3702 cni.go:473] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww cc51bc6d906e64e2a037672ffb3fdb3d1aae7e12ee38b7605d5bd1e993528168 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww cc51bc6d906e64e2a037672ffb3fdb3d1aae7e12ee38b7605d5bd1e993528168 network default NAD default NAD key default], result "{}", err I0514 23:21:15.253774 3702 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.253903 3702 pods.go:176] Deleting pod: openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww I0514 23:21:15.255509 3702 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww, ips: 10.128.0.41 I0514 23:21:15.255548 3702 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.255577 3702 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.255590 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:15.255824 3702 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.255844 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:15.257246 3702 cni.go:473] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg 707f822179fb9d2b45add00ad34f992da342436e040472e4dfa326556cf04c42 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg 707f822179fb9d2b45add00ad34f992da342436e040472e4dfa326556cf04c42 network default NAD default NAD key default], result "{}", err I0514 23:21:15.257966 3702 cni.go:473] [openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw a01845b5a4a851ab4b104514bd16b3aed0a27f6e80dc8b41f75f4f0f519ef0c1 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw a01845b5a4a851ab4b104514bd16b3aed0a27f6e80dc8b41f75f4f0f519ef0c1 network default NAD default NAD key default], result "{}", err I0514 23:21:15.265522 3702 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:15.265573 3702 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:15.265587 3702 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:15.265607 3702 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:15.316622 3702 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.316687 3702 pods.go:176] Deleting pod: openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg I0514 23:21:15.318385 3702 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg, ips: 10.128.0.40 I0514 23:21:15.318415 3702 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.318440 3702 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.318452 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:15.318703 3702 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.318722 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:15.340628 3702 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.340658 3702 pods.go:176] Deleting pod: openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw I0514 23:21:15.342147 3702 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw, ips: 10.128.0.38 I0514 23:21:15.342198 3702 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.342222 3702 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.342230 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:15.342475 3702 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.342491 3702 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:16.284581 3702 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.284604 3702 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.284620 3702 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.284635 3702 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.343008 3702 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.343034 3702 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.343047 3702 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.343059 3702 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.370846 3702 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.370868 3702 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.370877 3702 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.370889 3702 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:21:16.801273 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0514 23:21:20.492597 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0514 23:21:20.515340 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0514 23:21:20.527597 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0514 23:21:20.556627 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0514 23:21:29.132013 3702 namespace.go:98] [cert-manager] adding namespace I0514 23:21:29.132890 3702 namespace.go:102] [cert-manager] adding namespace took 851.397µs I0514 23:21:29.150949 3702 namespace.go:142] [cert-manager] updating namespace I0514 23:21:29.207609 3702 namespace.go:142] [cert-manager] updating namespace W0514 23:21:33.242937 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0514 23:21:33.242957 3702 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0514 23:21:38.683227 3702 namespace.go:142] [openshift-operators] updating namespace I0514 23:21:38.703238 3702 namespace.go:142] [openshift-operators] updating namespace W0514 23:21:44.973750 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0514 23:21:47.318369 3702 namespace.go:98] [trusted-artifact-signer] adding namespace I0514 23:21:47.319294 3702 namespace.go:102] [trusted-artifact-signer] adding namespace took 903.917µs W0514 23:21:47.351487 3702 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0514 23:21:47.374498 3702 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:21:47.396457 3702 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:21:51.474448 3702 namespace.go:142] [openshift-operators] updating namespace I0514 23:22:03.379237 3702 cni.go:452] [tsf/test-tsf-subscriptions 212e47c3b1727ac04b0dded28a708171698888bb63476da035e21aca4ce16561 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-subscriptions 212e47c3b1727ac04b0dded28a708171698888bb63476da035e21aca4ce16561 network default NAD default NAD key ] I0514 23:22:03.471523 3702 cni.go:473] [tsf/test-tsf-subscriptions 212e47c3b1727ac04b0dded28a708171698888bb63476da035e21aca4ce16561 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-subscriptions 212e47c3b1727ac04b0dded28a708171698888bb63476da035e21aca4ce16561 network default NAD default NAD key default], result "{}", err I0514 23:22:03.840601 3702 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:22:03.840637 3702 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0514 23:22:03.841939 3702 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.128.0.35 I0514 23:22:03.841967 3702 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:22:03.877348 3702 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:22:03.877369 3702 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:22:03.895236 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0514 23:22:03.895269 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0514 23:22:05.443735 3702 base_network_controller_pods.go:487] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-1-98.ec2.internal I0514 23:22:05.443834 3702 kube.go:256] Updating pod tsf/test-tsf-infrastructure I0514 23:22:05.461530 3702 pod.go:62] [tsf/test-tsf-infrastructure] pod update took 17.726648ms I0514 23:22:05.461554 3702 base_network_controller_pods.go:951] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 17.756226ms I0514 23:22:05.462844 3702 pods.go:271] [tsf/test-tsf-infrastructure] addLogicalPort took 19.124997ms, libovsdb time 930.562µs I0514 23:22:05.824158 3702 cni.go:452] [tsf/test-tsf-infrastructure 364f53c185e7830e6a6868ba477065e4ea2b5e7aaec9b81c511d2af438ebfbe8 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-infrastructure 364f53c185e7830e6a6868ba477065e4ea2b5e7aaec9b81c511d2af438ebfbe8 network default NAD default NAD key ] I0514 23:22:05.833837 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: 364f53c185e7830, network: default, NAD default, SandboxID: "364f53c185e7830e6a6868ba477065e4ea2b5e7aaec9b81c511d2af438ebfbe8", PCI device ID: , UID: "64a733f5-88c2-430a-b47f-84e190e05b91", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0514 23:22:05.888810 3702 cni.go:473] [tsf/test-tsf-infrastructure 364f53c185e7830e6a6868ba477065e4ea2b5e7aaec9b81c511d2af438ebfbe8 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-infrastructure 364f53c185e7830e6a6868ba477065e4ea2b5e7aaec9b81c511d2af438ebfbe8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:cd:df:27:0e:6c\",\"name\":\"364f53c185e7830\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5e574cfb-2416-4b04-8301-6741dde5cc2f\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:22:06.183760 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0514 23:22:08.060575 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=88 I0514 23:22:08.334879 3702 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-l6wwb] creating logical port tsf-keycloak_keycloak-pgsql-bee-744f6fc4d9-l6wwb for pod on switch ip-10-0-1-98.ec2.internal I0514 23:22:08.334991 3702 kube.go:256] Updating pod tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-l6wwb I0514 23:22:08.346751 3702 pod.go:62] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-l6wwb] pod update took 11.797312ms I0514 23:22:08.346779 3702 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-l6wwb] addLogicalPort annotation time took 11.83088ms I0514 23:22:08.348159 3702 pods.go:271] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-l6wwb] addLogicalPort took 13.2953ms, libovsdb time 1.000246ms I0514 23:22:08.499690 3702 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-xsrfr] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-xsrfr for pod on switch ip-10-0-1-98.ec2.internal I0514 23:22:08.499800 3702 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-xsrfr I0514 23:22:08.512001 3702 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-xsrfr] pod update took 12.224237ms I0514 23:22:08.512027 3702 base_network_controller_pods.go:951] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-xsrfr] addLogicalPort annotation time took 12.259632ms I0514 23:22:08.513324 3702 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-xsrfr] addLogicalPort took 13.657171ms, libovsdb time 873.086µs I0514 23:22:08.863546 3702 namespace.go:98] [openshift-pipelines] adding namespace I0514 23:22:08.864644 3702 namespace.go:102] [openshift-pipelines] adding namespace took 1.050276ms I0514 23:22:08.878036 3702 namespace.go:142] [openshift-pipelines] updating namespace I0514 23:22:09.701823 3702 namespace.go:142] [cert-manager] updating namespace I0514 23:22:09.901357 3702 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:22:10.304376 3702 namespace.go:142] [konflux-operator] updating namespace I0514 23:22:10.503699 3702 namespace.go:142] [konflux-ui] updating namespace I0514 23:22:10.704740 3702 namespace.go:142] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] updating namespace I0514 23:22:11.107756 3702 namespace.go:142] [rhbk-operator] updating namespace I0514 23:22:11.307427 3702 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:22:11.509999 3702 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:22:11.710538 3702 namespace.go:142] [tsf] updating namespace I0514 23:22:11.911909 3702 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:22:12.112673 3702 namespace.go:142] [tsf-tas] updating namespace I0514 23:22:12.313041 3702 namespace.go:142] [tsf-tpa] updating namespace I0514 23:22:12.450933 3702 namespace.go:142] [cert-manager] updating namespace I0514 23:22:12.460137 3702 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:22:12.470492 3702 namespace.go:142] [default] updating namespace I0514 23:22:12.481097 3702 namespace.go:142] [konflux-operator] updating namespace I0514 23:22:12.491385 3702 namespace.go:142] [konflux-ui] updating namespace I0514 23:22:12.503454 3702 namespace.go:142] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] updating namespace I0514 23:22:12.512661 3702 namespace.go:142] [openshift] updating namespace I0514 23:22:12.522957 3702 namespace.go:142] [rhbk-operator] updating namespace I0514 23:22:12.534684 3702 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:22:12.543918 3702 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:22:12.553144 3702 namespace.go:142] [tsf] updating namespace I0514 23:22:12.565380 3702 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:22:12.576529 3702 namespace.go:142] [tsf-tas] updating namespace I0514 23:22:12.595781 3702 namespace.go:142] [tsf-tpa] updating namespace I0514 23:22:12.697755 3702 namespace.go:142] [cert-manager] updating namespace I0514 23:22:12.800453 3702 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:22:12.898819 3702 namespace.go:142] [default] updating namespace I0514 23:22:13.001894 3702 namespace.go:142] [konflux-operator] updating namespace I0514 23:22:13.095841 3702 namespace.go:142] [konflux-ui] updating namespace I0514 23:22:13.200302 3702 namespace.go:142] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] updating namespace I0514 23:22:13.295345 3702 namespace.go:142] [openshift] updating namespace I0514 23:22:13.397030 3702 namespace.go:142] [rhbk-operator] updating namespace I0514 23:22:13.496590 3702 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:22:13.595812 3702 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:22:13.695557 3702 namespace.go:142] [tsf] updating namespace I0514 23:22:13.796371 3702 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:22:13.895348 3702 namespace.go:142] [tsf-tas] updating namespace I0514 23:22:13.998441 3702 namespace.go:142] [tsf-tpa] updating namespace I0514 23:22:14.255426 3702 namespace.go:142] [openshift-pipelines] updating namespace I0514 23:22:18.012321 3702 cni.go:452] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-l6wwb 8d050ac6450de7eccca16d50f379c0ce96487092ff7a2465bec1340ef0e97078 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-l6wwb 8d050ac6450de7eccca16d50f379c0ce96487092ff7a2465bec1340ef0e97078 network default NAD default NAD key ] I0514 23:22:18.021802 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-pgsql-bee-744f6fc4d9-l6wwb, hostIfaceName: 8d050ac6450de7e, network: default, NAD default, SandboxID: "8d050ac6450de7eccca16d50f379c0ce96487092ff7a2465bec1340ef0e97078", PCI device ID: , UID: "84e3d2e4-966a-4a9c-9077-db3348c75663", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0514 23:22:18.074082 3702 cni.go:473] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-l6wwb 8d050ac6450de7eccca16d50f379c0ce96487092ff7a2465bec1340ef0e97078 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-l6wwb 8d050ac6450de7eccca16d50f379c0ce96487092ff7a2465bec1340ef0e97078 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:be:81:05:1c:da\",\"name\":\"8d050ac6450de7e\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/690e8a89-a723-4c39-b604-a14ce2e3ca50\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:22:18.166057 3702 cni.go:452] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-xsrfr 0734cb217a6765caf9d01ff88ac65b4048b7f27b418a1b5b5a6a5957650bff29 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-xsrfr 0734cb217a6765caf9d01ff88ac65b4048b7f27b418a1b5b5a6a5957650bff29 network default NAD default NAD key ] I0514 23:22:18.176092 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-xsrfr, hostIfaceName: 0734cb217a6765c, network: default, NAD default, SandboxID: "0734cb217a6765caf9d01ff88ac65b4048b7f27b418a1b5b5a6a5957650bff29", PCI device ID: , UID: "b5d6228c-2b6a-41c4-bb48-88c786d18bda", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0514 23:22:18.229446 3702 cni.go:473] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-xsrfr 0734cb217a6765caf9d01ff88ac65b4048b7f27b418a1b5b5a6a5957650bff29 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-xsrfr 0734cb217a6765caf9d01ff88ac65b4048b7f27b418a1b5b5a6a5957650bff29 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:98:66:ec:e7:d7\",\"name\":\"0734cb217a6765c\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4dd85948-591d-4d77-aee6-1cb853542374\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0514 23:22:18.460518 3702 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0514 23:22:18.488741 3702 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0514 23:22:22.070478 3702 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-v7tcg] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-57f84cff6f-v7tcg for pod on switch ip-10-0-1-98.ec2.internal I0514 23:22:22.070643 3702 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-v7tcg I0514 23:22:22.086212 3702 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-v7tcg] pod update took 15.605908ms I0514 23:22:22.086236 3702 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-v7tcg] addLogicalPort annotation time took 15.637135ms I0514 23:22:22.087521 3702 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-v7tcg] addLogicalPort took 17.06535ms, libovsdb time 930.592µs I0514 23:22:22.437562 3702 cni.go:452] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-v7tcg a99a76f4f523e52eeee67fbc898271ea261bef80870e1afb72d0e69925142935 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-v7tcg a99a76f4f523e52eeee67fbc898271ea261bef80870e1afb72d0e69925142935 network default NAD default NAD key ] I0514 23:22:22.453468 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-57f84cff6f-v7tcg, hostIfaceName: a99a76f4f523e52, network: default, NAD default, SandboxID: "a99a76f4f523e52eeee67fbc898271ea261bef80870e1afb72d0e69925142935", PCI device ID: , UID: "32e287db-5097-4c7c-97a5-3c34fdc6b050", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0514 23:22:22.513440 3702 cni.go:473] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-v7tcg a99a76f4f523e52eeee67fbc898271ea261bef80870e1afb72d0e69925142935 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-v7tcg a99a76f4f523e52eeee67fbc898271ea261bef80870e1afb72d0e69925142935 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:7f:7f:8d:e2:e8\",\"name\":\"a99a76f4f523e52\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c3e37482-50f1-49f8-a290-cf107e3d9036\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0514 23:22:26.150767 3702 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0514 23:22:26.150789 3702 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0514 23:22:26.150794 3702 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0514 23:22:26.191168 3702 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0514 23:22:26.191209 3702 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0514 23:22:26.191214 3702 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0514 23:22:26.533735 3702 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:26.533756 3702 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:26.533761 3702 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:26.533765 3702 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:39.472377 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0514 23:22:40.488850 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5432: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0514 23:22:42.612744 3702 cni.go:452] [tsf/test-tsf-infrastructure 364f53c185e7830e6a6868ba477065e4ea2b5e7aaec9b81c511d2af438ebfbe8 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-infrastructure 364f53c185e7830e6a6868ba477065e4ea2b5e7aaec9b81c511d2af438ebfbe8 network default NAD default NAD key ] I0514 23:22:42.684570 3702 cni.go:473] [tsf/test-tsf-infrastructure 364f53c185e7830e6a6868ba477065e4ea2b5e7aaec9b81c511d2af438ebfbe8 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-infrastructure 364f53c185e7830e6a6868ba477065e4ea2b5e7aaec9b81c511d2af438ebfbe8 network default NAD default NAD key default], result "{}", err I0514 23:22:42.732383 3702 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:22:42.732418 3702 pods.go:176] Deleting pod: tsf/test-tsf-infrastructure I0514 23:22:42.733710 3702 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.128.0.42 I0514 23:22:42.733742 3702 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:22:42.752084 3702 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:22:42.752106 3702 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:22:42.759555 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0514 23:22:42.759580 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0514 23:22:43.827505 3702 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-1-98.ec2.internal I0514 23:22:43.827652 3702 kube.go:256] Updating pod tsf/test-tsf-iam I0514 23:22:43.846071 3702 pod.go:62] [tsf/test-tsf-iam] pod update took 18.443228ms I0514 23:22:43.846103 3702 base_network_controller_pods.go:951] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 18.480125ms I0514 23:22:43.847796 3702 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 20.306088ms, libovsdb time 1.138493ms I0514 23:22:44.208976 3702 cni.go:452] [tsf/test-tsf-iam 26c012ce2bb0a187d928f77c5b08a8497a0d0f297f07262eea1b66ffad4cc336 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-iam 26c012ce2bb0a187d928f77c5b08a8497a0d0f297f07262eea1b66ffad4cc336 network default NAD default NAD key ] I0514 23:22:44.220024 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 26c012ce2bb0a18, network: default, NAD default, SandboxID: "26c012ce2bb0a187d928f77c5b08a8497a0d0f297f07262eea1b66ffad4cc336", PCI device ID: , UID: "d9a779d4-0551-4324-9d75-e941aac19e5e", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0514 23:22:44.271997 3702 cni.go:473] [tsf/test-tsf-iam 26c012ce2bb0a187d928f77c5b08a8497a0d0f297f07262eea1b66ffad4cc336 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-iam 26c012ce2bb0a187d928f77c5b08a8497a0d0f297f07262eea1b66ffad4cc336 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:04:d1:49:da:c7\",\"name\":\"26c012ce2bb0a18\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/08679a68-8675-4ca2-b850-c5adf8ae589d\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:22:44.758291 3702 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-q5r7s] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-q5r7s for pod on switch ip-10-0-1-98.ec2.internal I0514 23:22:44.758427 3702 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-q5r7s I0514 23:22:44.773822 3702 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-q5r7s] pod update took 15.416156ms I0514 23:22:44.773852 3702 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-q5r7s] addLogicalPort annotation time took 15.452822ms I0514 23:22:44.775190 3702 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-q5r7s] addLogicalPort took 16.901887ms, libovsdb time 866.525µs I0514 23:22:44.805725 3702 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0514 23:22:44.806557 3702 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0514 23:22:44.806580 3702 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0514 23:22:44.808537 3702 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0514 23:22:45.120189 3702 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-q5r7s c21f1811ada5fd6fea9bf76287b74e0b21fffce068345d3873aff933417f9327 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-q5r7s c21f1811ada5fd6fea9bf76287b74e0b21fffce068345d3873aff933417f9327 network default NAD default NAD key ] I0514 23:22:45.130079 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-q5r7s, hostIfaceName: c21f1811ada5fd6, network: default, NAD default, SandboxID: "c21f1811ada5fd6fea9bf76287b74e0b21fffce068345d3873aff933417f9327", PCI device ID: , UID: "a2c2e169-3b5a-448c-be14-7d21484c68c9", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0514 23:22:45.186872 3702 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-q5r7s c21f1811ada5fd6fea9bf76287b74e0b21fffce068345d3873aff933417f9327 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-q5r7s c21f1811ada5fd6fea9bf76287b74e0b21fffce068345d3873aff933417f9327 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:db:4d:b7:52:19\",\"name\":\"c21f1811ada5fd6\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3aada592-da52-43f1-b1bb-026bd3a8cba9\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:22:45.193339 3702 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:22:45.193498 3702 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0514 23:22:45.212807 3702 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 19.346122ms I0514 23:22:45.212835 3702 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 19.379192ms I0514 23:22:45.214115 3702 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 20.79492ms, libovsdb time 851.45µs I0514 23:22:45.556027 3702 cni.go:452] [tsf-keycloak/keycloak-0 88a624e7f6a43b86851c49bc1cc2d85498ed38511df1c00ea86a13e030652c64 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-0 88a624e7f6a43b86851c49bc1cc2d85498ed38511df1c00ea86a13e030652c64 network default NAD default NAD key ] I0514 23:22:45.565613 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 88a624e7f6a43b8, network: default, NAD default, SandboxID: "88a624e7f6a43b86851c49bc1cc2d85498ed38511df1c00ea86a13e030652c64", PCI device ID: , UID: "05fabf35-24c5-40f1-b97d-1855a94f4533", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0514 23:22:45.620130 3702 cni.go:473] [tsf-keycloak/keycloak-0 88a624e7f6a43b86851c49bc1cc2d85498ed38511df1c00ea86a13e030652c64 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-0 88a624e7f6a43b86851c49bc1cc2d85498ed38511df1c00ea86a13e030652c64 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:13:89:27:2f:94\",\"name\":\"88a624e7f6a43b8\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1bb4c417-0c2b-4a7b-bb91-38ef00e972d7\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0514 23:22:47.626689 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0514 23:22:53.068494 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=976 W0514 23:22:55.702315 3702 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0514 23:22:55.702335 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0514 23:22:55.702431 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0514 23:22:55.702440 3702 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service I0514 23:22:59.149972 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0514 23:23:05.320061 3702 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0514 23:23:19.799588 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0514 23:23:19.799608 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0514 23:23:19.802858 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0514 23:23:19.802880 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9000: empty IP address endpoints for service tsf-keycloak/keycloak-service I0514 23:23:24.863945 3702 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-6ccvn] creating logical port tsf-keycloak_tsf-iam-6ccvn for pod on switch ip-10-0-1-98.ec2.internal I0514 23:23:24.864043 3702 kube.go:256] Updating pod tsf-keycloak/tsf-iam-6ccvn I0514 23:23:24.878376 3702 pod.go:62] [tsf-keycloak/tsf-iam-6ccvn] pod update took 14.356527ms I0514 23:23:24.878405 3702 base_network_controller_pods.go:951] [default/tsf-keycloak/tsf-iam-6ccvn] addLogicalPort annotation time took 14.389694ms I0514 23:23:24.879911 3702 pods.go:271] [tsf-keycloak/tsf-iam-6ccvn] addLogicalPort took 15.983381ms, libovsdb time 918.401µs I0514 23:23:25.231638 3702 cni.go:452] [tsf-keycloak/tsf-iam-6ccvn cb149e22bf7556dd14ceb495b1341086c119ca525c522c6377c7f0b4eec80bf4 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/tsf-iam-6ccvn cb149e22bf7556dd14ceb495b1341086c119ca525c522c6377c7f0b4eec80bf4 network default NAD default NAD key ] I0514 23:23:25.241033 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-6ccvn, hostIfaceName: cb149e22bf7556d, network: default, NAD default, SandboxID: "cb149e22bf7556dd14ceb495b1341086c119ca525c522c6377c7f0b4eec80bf4", PCI device ID: , UID: "326af6c7-a355-4717-836e-4b9c236d45ba", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0514 23:23:25.295458 3702 cni.go:473] [tsf-keycloak/tsf-iam-6ccvn cb149e22bf7556dd14ceb495b1341086c119ca525c522c6377c7f0b4eec80bf4 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/tsf-iam-6ccvn cb149e22bf7556dd14ceb495b1341086c119ca525c522c6377c7f0b4eec80bf4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:45:49:82:ab:41\",\"name\":\"cb149e22bf7556d\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c09a5ba7-e1b5-4fa9-bf91-242d657e8a61\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0514 23:23:25.839324 3702 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0514 23:23:25.839348 3702 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0514 23:23:25.839355 3702 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0514 23:23:34.959027 3702 cni.go:452] [tsf/test-tsf-iam 26c012ce2bb0a187d928f77c5b08a8497a0d0f297f07262eea1b66ffad4cc336 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-iam 26c012ce2bb0a187d928f77c5b08a8497a0d0f297f07262eea1b66ffad4cc336 network default NAD default NAD key ] I0514 23:23:35.041568 3702 cni.go:473] [tsf/test-tsf-iam 26c012ce2bb0a187d928f77c5b08a8497a0d0f297f07262eea1b66ffad4cc336 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-iam 26c012ce2bb0a187d928f77c5b08a8497a0d0f297f07262eea1b66ffad4cc336 network default NAD default NAD key default], result "{}", err I0514 23:23:35.097280 3702 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:35.097396 3702 pods.go:176] Deleting pod: tsf/test-tsf-iam I0514 23:23:35.098928 3702 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.128.0.46 I0514 23:23:35.098961 3702 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:35.141700 3702 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:35.141724 3702 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:35.148336 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0514 23:23:35.148364 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0514 23:23:35.947549 3702 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-1-98.ec2.internal I0514 23:23:35.947759 3702 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0514 23:23:35.961681 3702 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 13.941864ms I0514 23:23:35.961710 3702 base_network_controller_pods.go:951] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 13.986754ms I0514 23:23:35.963210 3702 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 15.684661ms, libovsdb time 999.305µs I0514 23:23:36.318343 3702 cni.go:452] [tsf-tas/test-tsf-tas 3bc5ba3eb9d7f64f2cf5adcca6f71c5161c769c65e79a193176aa4432dc3243b network default NAD default NAD key ] ADD starting CNI request [tsf-tas/test-tsf-tas 3bc5ba3eb9d7f64f2cf5adcca6f71c5161c769c65e79a193176aa4432dc3243b network default NAD default NAD key ] I0514 23:23:36.332234 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: 3bc5ba3eb9d7f64, network: default, NAD default, SandboxID: "3bc5ba3eb9d7f64f2cf5adcca6f71c5161c769c65e79a193176aa4432dc3243b", PCI device ID: , UID: "bb3655d3-d0d0-409f-9b31-659bcdfadb80", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0514 23:23:36.397532 3702 cni.go:473] [tsf-tas/test-tsf-tas 3bc5ba3eb9d7f64f2cf5adcca6f71c5161c769c65e79a193176aa4432dc3243b network default NAD default NAD key default] ADD finished CNI request [tsf-tas/test-tsf-tas 3bc5ba3eb9d7f64f2cf5adcca6f71c5161c769c65e79a193176aa4432dc3243b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:f5:28:b5:3b:7e\",\"name\":\"3bc5ba3eb9d7f64\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d997459d-fe22-4e42-8b25-bec43a1dbcfa\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:23:37.930626 3702 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-7f9f599dd9-lb9q6] creating logical port tsf-tas_trillian-logserver-7f9f599dd9-lb9q6 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:23:37.930752 3702 kube.go:256] Updating pod tsf-tas/trillian-logserver-7f9f599dd9-lb9q6 I0514 23:23:37.955318 3702 pod.go:62] [tsf-tas/trillian-logserver-7f9f599dd9-lb9q6] pod update took 24.57602ms I0514 23:23:37.955350 3702 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logserver-7f9f599dd9-lb9q6] addLogicalPort annotation time took 24.626373ms I0514 23:23:37.956736 3702 pods.go:271] [tsf-tas/trillian-logserver-7f9f599dd9-lb9q6] addLogicalPort took 26.126303ms, libovsdb time 995.741µs I0514 23:23:38.143805 3702 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=11 I0514 23:23:38.296866 3702 cni.go:452] [tsf-tas/trillian-logserver-7f9f599dd9-lb9q6 bafd64f4c08043b7946176a895763b633bb2b8d022f1ff78dd09dfc47a571ee1 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logserver-7f9f599dd9-lb9q6 bafd64f4c08043b7946176a895763b633bb2b8d022f1ff78dd09dfc47a571ee1 network default NAD default NAD key ] I0514 23:23:38.306539 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-7f9f599dd9-lb9q6, hostIfaceName: bafd64f4c08043b, network: default, NAD default, SandboxID: "bafd64f4c08043b7946176a895763b633bb2b8d022f1ff78dd09dfc47a571ee1", PCI device ID: , UID: "a20720ff-e436-4d8d-bbb6-c9669c934215", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0514 23:23:38.363595 3702 cni.go:473] [tsf-tas/trillian-logserver-7f9f599dd9-lb9q6 bafd64f4c08043b7946176a895763b633bb2b8d022f1ff78dd09dfc47a571ee1 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logserver-7f9f599dd9-lb9q6 bafd64f4c08043b7946176a895763b633bb2b8d022f1ff78dd09dfc47a571ee1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:b3:4a:55:e5:7e\",\"name\":\"bafd64f4c08043b\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fd8ce196-f9fb-4033-b16b-a135a861f687\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0514 23:23:40.890578 3702 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0514 23:23:40.890608 3702 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0514 23:23:46.035889 3702 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0514 23:23:46.040032 3702 cni.go:452] [tsf-keycloak/tsf-iam-6ccvn cb149e22bf7556dd14ceb495b1341086c119ca525c522c6377c7f0b4eec80bf4 network default NAD default NAD key ] DEL starting CNI request [tsf-keycloak/tsf-iam-6ccvn cb149e22bf7556dd14ceb495b1341086c119ca525c522c6377c7f0b4eec80bf4 network default NAD default NAD key ] I0514 23:23:46.115393 3702 cni.go:473] [tsf-keycloak/tsf-iam-6ccvn cb149e22bf7556dd14ceb495b1341086c119ca525c522c6377c7f0b4eec80bf4 network default NAD default NAD key default] DEL finished CNI request [tsf-keycloak/tsf-iam-6ccvn cb149e22bf7556dd14ceb495b1341086c119ca525c522c6377c7f0b4eec80bf4 network default NAD default NAD key default], result "{}", err I0514 23:23:46.164002 3702 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-6ccvn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:46.164042 3702 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-6ccvn I0514 23:23:46.165415 3702 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-6ccvn, ips: 10.128.0.49 I0514 23:23:46.165448 3702 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-6ccvn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:47.068745 3702 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-6ccvn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:47.068768 3702 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-6ccvn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:23:53.959778 3702 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0514 23:23:54.064920 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=220 W0514 23:23:55.613289 3702 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0514 23:23:57.214539 3702 namespace.go:142] [openshift-pipelines] updating namespace I0514 23:23:58.737241 3702 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-4d45n-nzds2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:58.737789 3702 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-4d45n-nzds2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:59.760088 3702 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-4d45n-nzds2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:59.760114 3702 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-4d45n-nzds2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:00.249742 3702 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bn84x] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-bn84x for pod on switch ip-10-0-1-98.ec2.internal I0514 23:24:00.249833 3702 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bn84x I0514 23:24:00.276220 3702 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bn84x] pod update took 26.408619ms I0514 23:24:00.276248 3702 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bn84x] addLogicalPort annotation time took 26.441656ms I0514 23:24:00.277591 3702 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bn84x] addLogicalPort took 27.863453ms, libovsdb time 945.842µs I0514 23:24:00.347407 3702 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-xk7rk] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-xk7rk for pod on switch ip-10-0-1-98.ec2.internal I0514 23:24:00.347513 3702 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-xk7rk I0514 23:24:00.368044 3702 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-xk7rk] pod update took 20.5642ms I0514 23:24:00.368071 3702 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-xk7rk] addLogicalPort annotation time took 20.594553ms I0514 23:24:00.369427 3702 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-xk7rk] addLogicalPort took 22.040038ms, libovsdb time 931.857µs I0514 23:24:00.447208 3702 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-b9d6t] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-b9d6t for pod on switch ip-10-0-1-98.ec2.internal I0514 23:24:00.447300 3702 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-b9d6t I0514 23:24:00.463654 3702 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b9d6t] pod update took 16.377031ms I0514 23:24:00.463683 3702 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-b9d6t] addLogicalPort annotation time took 16.409956ms I0514 23:24:00.464921 3702 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b9d6t] addLogicalPort took 17.745369ms, libovsdb time 801.11µs I0514 23:24:00.612389 3702 cni.go:452] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bn84x 1a8e21f8cf69a141885dd939dda4eb7805cb46f9e8ac70c54270da46adc31f0a network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bn84x 1a8e21f8cf69a141885dd939dda4eb7805cb46f9e8ac70c54270da46adc31f0a network default NAD default NAD key ] I0514 23:24:00.621894 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-bn84x, hostIfaceName: 1a8e21f8cf69a14, network: default, NAD default, SandboxID: "1a8e21f8cf69a141885dd939dda4eb7805cb46f9e8ac70c54270da46adc31f0a", PCI device ID: , UID: "7a37964d-785b-4679-9b40-75977631531b", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0514 23:24:00.675065 3702 cni.go:473] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bn84x 1a8e21f8cf69a141885dd939dda4eb7805cb46f9e8ac70c54270da46adc31f0a network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bn84x 1a8e21f8cf69a141885dd939dda4eb7805cb46f9e8ac70c54270da46adc31f0a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:39:37:31:06:5e\",\"name\":\"1a8e21f8cf69a14\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/843fc460-49d8-4d42-87d1-a552d8d11ea2\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:24:00.711502 3702 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-xk7rk c35c57267775de25da88a75f7bb23c8a91a3e9e4827adf97ca4786846690dc6c network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-xk7rk c35c57267775de25da88a75f7bb23c8a91a3e9e4827adf97ca4786846690dc6c network default NAD default NAD key ] I0514 23:24:00.721210 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-xk7rk, hostIfaceName: c35c57267775de2, network: default, NAD default, SandboxID: "c35c57267775de25da88a75f7bb23c8a91a3e9e4827adf97ca4786846690dc6c", PCI device ID: , UID: "8efad09b-aefe-446b-b68a-5a957e0ed1d7", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0514 23:24:00.773748 3702 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-xk7rk c35c57267775de25da88a75f7bb23c8a91a3e9e4827adf97ca4786846690dc6c network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-xk7rk c35c57267775de25da88a75f7bb23c8a91a3e9e4827adf97ca4786846690dc6c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:68:53:72:44:7e\",\"name\":\"c35c57267775de2\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ea32ea91-7d21-49db-9e4b-1586ee4839c1\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:24:00.811928 3702 cni.go:452] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b9d6t 179d4985321a4aadb020a6f917f752b6c0b5e52666c763acda71763d0876c66e network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b9d6t 179d4985321a4aadb020a6f917f752b6c0b5e52666c763acda71763d0876c66e network default NAD default NAD key ] I0514 23:24:00.821540 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-b9d6t, hostIfaceName: 179d4985321a4aa, network: default, NAD default, SandboxID: "179d4985321a4aadb020a6f917f752b6c0b5e52666c763acda71763d0876c66e", PCI device ID: , UID: "23ceed69-43bd-4bb2-88c4-c28d836dd877", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0514 23:24:00.875525 3702 cni.go:473] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b9d6t 179d4985321a4aadb020a6f917f752b6c0b5e52666c763acda71763d0876c66e network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b9d6t 179d4985321a4aadb020a6f917f752b6c0b5e52666c763acda71763d0876c66e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:77:35:90:fc:b3\",\"name\":\"179d4985321a4aa\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/718ef30b-803e-4079-9009-14a24a7e0905\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0514 23:24:04.155498 3702 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0514 23:24:15.165549 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0514 23:24:19.193506 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0514 23:24:24.921331 3702 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-tsvkq-4kksv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:24.921945 3702 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-tsvkq-4kksv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:25.765970 3702 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-tsvkq-4kksv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:25.765997 3702 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-tsvkq-4kksv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:27.293112 3702 cni.go:452] [tsf-tas/test-tsf-tas 3bc5ba3eb9d7f64f2cf5adcca6f71c5161c769c65e79a193176aa4432dc3243b network default NAD default NAD key ] DEL starting CNI request [tsf-tas/test-tsf-tas 3bc5ba3eb9d7f64f2cf5adcca6f71c5161c769c65e79a193176aa4432dc3243b network default NAD default NAD key ] I0514 23:24:27.378481 3702 cni.go:473] [tsf-tas/test-tsf-tas 3bc5ba3eb9d7f64f2cf5adcca6f71c5161c769c65e79a193176aa4432dc3243b network default NAD default NAD key default] DEL finished CNI request [tsf-tas/test-tsf-tas 3bc5ba3eb9d7f64f2cf5adcca6f71c5161c769c65e79a193176aa4432dc3243b network default NAD default NAD key default], result "{}", err I0514 23:24:27.429788 3702 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:27.429821 3702 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0514 23:24:27.431119 3702 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.128.0.50 I0514 23:24:27.431147 3702 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:27.450217 3702 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:27.450245 3702 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:27.457145 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0514 23:24:27.457187 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas W0514 23:24:35.808046 3702 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0514 23:24:36.957572 3702 obj_retry.go:498] Detected object tsf/patch-tekton-config-z22jk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:36.958140 3702 obj_retry.go:498] Detected object tsf/patch-tekton-config-z22jk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:37.979836 3702 obj_retry.go:492] Detected object tsf/patch-tekton-config-z22jk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:37.979861 3702 obj_retry.go:492] Detected object tsf/patch-tekton-config-z22jk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:38.075615 3702 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-q77kq] creating logical port tsf_tsf-tekton-configuration-q77kq for pod on switch ip-10-0-1-98.ec2.internal I0514 23:24:38.075701 3702 kube.go:256] Updating pod tsf/tsf-tekton-configuration-q77kq I0514 23:24:38.087774 3702 pod.go:62] [tsf/tsf-tekton-configuration-q77kq] pod update took 12.096675ms I0514 23:24:38.087805 3702 base_network_controller_pods.go:951] [default/tsf/tsf-tekton-configuration-q77kq] addLogicalPort annotation time took 12.130906ms I0514 23:24:38.089093 3702 pods.go:271] [tsf/tsf-tekton-configuration-q77kq] addLogicalPort took 13.494321ms, libovsdb time 842.203µs I0514 23:24:38.439483 3702 cni.go:452] [tsf/tsf-tekton-configuration-q77kq d950b66e83d030804658df69ec2562f28c2dcc2eef9492f2fb04f72ee76eba14 network default NAD default NAD key ] ADD starting CNI request [tsf/tsf-tekton-configuration-q77kq d950b66e83d030804658df69ec2562f28c2dcc2eef9492f2fb04f72ee76eba14 network default NAD default NAD key ] I0514 23:24:38.448619 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-q77kq, hostIfaceName: d950b66e83d0308, network: default, NAD default, SandboxID: "d950b66e83d030804658df69ec2562f28c2dcc2eef9492f2fb04f72ee76eba14", PCI device ID: , UID: "9a225f94-bcc9-4880-bd27-fc359ca9cd8a", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0514 23:24:38.502422 3702 cni.go:473] [tsf/tsf-tekton-configuration-q77kq d950b66e83d030804658df69ec2562f28c2dcc2eef9492f2fb04f72ee76eba14 network default NAD default NAD key default] ADD finished CNI request [tsf/tsf-tekton-configuration-q77kq d950b66e83d030804658df69ec2562f28c2dcc2eef9492f2fb04f72ee76eba14 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:ff:93:01:2d:64\",\"name\":\"d950b66e83d0308\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/064955c6-5d4a-47a1-bf65-c9f274eb0a43\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:24:43.391808 3702 cni.go:452] [tsf/tsf-tekton-configuration-q77kq d950b66e83d030804658df69ec2562f28c2dcc2eef9492f2fb04f72ee76eba14 network default NAD default NAD key ] DEL starting CNI request [tsf/tsf-tekton-configuration-q77kq d950b66e83d030804658df69ec2562f28c2dcc2eef9492f2fb04f72ee76eba14 network default NAD default NAD key ] I0514 23:24:43.478973 3702 cni.go:473] [tsf/tsf-tekton-configuration-q77kq d950b66e83d030804658df69ec2562f28c2dcc2eef9492f2fb04f72ee76eba14 network default NAD default NAD key default] DEL finished CNI request [tsf/tsf-tekton-configuration-q77kq d950b66e83d030804658df69ec2562f28c2dcc2eef9492f2fb04f72ee76eba14 network default NAD default NAD key default], result "{}", err I0514 23:24:43.530109 3702 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-q77kq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:43.530144 3702 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-q77kq I0514 23:24:43.531588 3702 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-q77kq, ips: 10.128.0.55 I0514 23:24:43.531617 3702 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-q77kq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:44.360770 3702 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-7p2l9-j2vbg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:44.361372 3702 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-7p2l9-j2vbg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:44.548706 3702 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-q77kq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:44.548734 3702 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-q77kq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:44.582782 3702 obj_retry.go:492] Detected object tsf/patch-tekton-config-z22jk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:44.582804 3702 obj_retry.go:492] Detected object tsf/patch-tekton-config-z22jk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:44.588775 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-z22jk I0514 23:24:44.588806 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-z22jk I0514 23:24:45.236315 3702 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-7p2l9-j2vbg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:45.236343 3702 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-7p2l9-j2vbg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:45.238050 3702 base_network_controller_pods.go:487] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-1-98.ec2.internal I0514 23:24:45.238134 3702 kube.go:256] Updating pod tsf-tpa/pre-install-tsf-tpa I0514 23:24:45.254544 3702 pod.go:62] [tsf-tpa/pre-install-tsf-tpa] pod update took 16.427879ms I0514 23:24:45.254569 3702 base_network_controller_pods.go:951] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 16.459577ms I0514 23:24:45.255904 3702 pods.go:271] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 17.868794ms, libovsdb time 925.993µs I0514 23:24:45.598919 3702 cni.go:452] [tsf-tpa/pre-install-tsf-tpa b0f568277128b85b23a76b8eec9f95ca7eef4b88e67d0f5d3d3c8ab05019ce39 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa b0f568277128b85b23a76b8eec9f95ca7eef4b88e67d0f5d3d3c8ab05019ce39 network default NAD default NAD key ] I0514 23:24:45.608149 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: b0f568277128b85, network: default, NAD default, SandboxID: "b0f568277128b85b23a76b8eec9f95ca7eef4b88e67d0f5d3d3c8ab05019ce39", PCI device ID: , UID: "c37f0f27-eafc-4477-ae38-dce812164c49", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0514 23:24:45.661737 3702 cni.go:473] [tsf-tpa/pre-install-tsf-tpa b0f568277128b85b23a76b8eec9f95ca7eef4b88e67d0f5d3d3c8ab05019ce39 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa b0f568277128b85b23a76b8eec9f95ca7eef4b88e67d0f5d3d3c8ab05019ce39 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:0c:39:27:57:fe\",\"name\":\"b0f568277128b85\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6111ec7f-da1d-439c-9f74-be2025db041e\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:24:48.423199 3702 cni.go:452] [tsf-tpa/pre-install-tsf-tpa b0f568277128b85b23a76b8eec9f95ca7eef4b88e67d0f5d3d3c8ab05019ce39 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa b0f568277128b85b23a76b8eec9f95ca7eef4b88e67d0f5d3d3c8ab05019ce39 network default NAD default NAD key ] I0514 23:24:48.494770 3702 cni.go:473] [tsf-tpa/pre-install-tsf-tpa b0f568277128b85b23a76b8eec9f95ca7eef4b88e67d0f5d3d3c8ab05019ce39 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa b0f568277128b85b23a76b8eec9f95ca7eef4b88e67d0f5d3d3c8ab05019ce39 network default NAD default NAD key default], result "{}", err I0514 23:24:48.545967 3702 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:48.546007 3702 pods.go:176] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0514 23:24:48.547348 3702 pods.go:236] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.128.0.56 I0514 23:24:48.547376 3702 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:48.565730 3702 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:48.565753 3702 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:48.572495 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0514 23:24:48.572573 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0514 23:24:48.768134 3702 base_network_controller_pods.go:487] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-1-98.ec2.internal I0514 23:24:48.768249 3702 kube.go:256] Updating pod tsf-tpa/test-tsf-tpa I0514 23:24:48.782219 3702 pod.go:62] [tsf-tpa/test-tsf-tpa] pod update took 13.99575ms I0514 23:24:48.782241 3702 base_network_controller_pods.go:951] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 14.023169ms I0514 23:24:48.783583 3702 pods.go:271] [tsf-tpa/test-tsf-tpa] addLogicalPort took 15.464063ms, libovsdb time 916.451µs I0514 23:24:49.133294 3702 cni.go:452] [tsf-tpa/test-tsf-tpa 07db537e1eb687b289f6c8183d8a53d99241c9069f57f5e937f120bf1905b41f network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/test-tsf-tpa 07db537e1eb687b289f6c8183d8a53d99241c9069f57f5e937f120bf1905b41f network default NAD default NAD key ] I0514 23:24:49.163408 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: 07db537e1eb687b, network: default, NAD default, SandboxID: "07db537e1eb687b289f6c8183d8a53d99241c9069f57f5e937f120bf1905b41f", PCI device ID: , UID: "9aa68065-09e1-4238-a15a-39b88faa2e77", MAC: 0a:58:0a:80:00:39, IPs: [10.128.0.57/23] I0514 23:24:49.219226 3702 cni.go:473] [tsf-tpa/test-tsf-tpa 07db537e1eb687b289f6c8183d8a53d99241c9069f57f5e937f120bf1905b41f network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/test-tsf-tpa 07db537e1eb687b289f6c8183d8a53d99241c9069f57f5e937f120bf1905b41f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:21:f1:4a:67:ae\",\"name\":\"07db537e1eb687b\"},{\"mac\":\"0a:58:0a:80:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c4298298-bb69-4324-9237-ffc30e63d2d6\"}],\"ips\":[{\"address\":\"10.128.0.57/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0514 23:24:52.899034 3702 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0514 23:24:52.919951 3702 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0514 23:24:54.418726 3702 obj_retry.go:498] Detected object tsf-tpa/create-db-kjtlv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:54.419303 3702 obj_retry.go:498] Detected object tsf-tpa/create-db-kjtlv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:55.441549 3702 obj_retry.go:492] Detected object tsf-tpa/create-db-kjtlv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:55.441578 3702 obj_retry.go:492] Detected object tsf-tpa/create-db-kjtlv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:58.443714 3702 obj_retry.go:498] Detected object tsf-tpa/migrate-db-4fs8n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:58.444321 3702 obj_retry.go:498] Detected object tsf-tpa/migrate-db-4fs8n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:58.575389 3702 obj_retry.go:492] Detected object tsf-tpa/migrate-db-4fs8n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:58.575410 3702 obj_retry.go:492] Detected object tsf-tpa/migrate-db-4fs8n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:25:00.953680 3702 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0514 23:25:03.104845 3702 obj_retry.go:498] Detected object tsf-tpa/create-importers-x6kkn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:03.105421 3702 obj_retry.go:498] Detected object tsf-tpa/create-importers-x6kkn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:04.130236 3702 obj_retry.go:492] Detected object tsf-tpa/create-importers-x6kkn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:04.130260 3702 obj_retry.go:492] Detected object tsf-tpa/create-importers-x6kkn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:25:14.415811 3702 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0514 23:25:15.856074 3702 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-s6cwq-rwlz5] creating logical port tsf-tas_tuf-repository-init-s6cwq-rwlz5 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:25:15.856201 3702 kube.go:256] Updating pod tsf-tas/tuf-repository-init-s6cwq-rwlz5 I0514 23:25:15.869286 3702 pod.go:62] [tsf-tas/tuf-repository-init-s6cwq-rwlz5] pod update took 13.129315ms I0514 23:25:15.869310 3702 base_network_controller_pods.go:951] [default/tsf-tas/tuf-repository-init-s6cwq-rwlz5] addLogicalPort annotation time took 13.160316ms I0514 23:25:15.870725 3702 pods.go:271] [tsf-tas/tuf-repository-init-s6cwq-rwlz5] addLogicalPort took 14.671076ms, libovsdb time 957.726µs I0514 23:25:16.596727 3702 cni.go:452] [tsf-tpa/test-tsf-tpa 07db537e1eb687b289f6c8183d8a53d99241c9069f57f5e937f120bf1905b41f network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/test-tsf-tpa 07db537e1eb687b289f6c8183d8a53d99241c9069f57f5e937f120bf1905b41f network default NAD default NAD key ] I0514 23:25:16.670901 3702 cni.go:473] [tsf-tpa/test-tsf-tpa 07db537e1eb687b289f6c8183d8a53d99241c9069f57f5e937f120bf1905b41f network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/test-tsf-tpa 07db537e1eb687b289f6c8183d8a53d99241c9069f57f5e937f120bf1905b41f network default NAD default NAD key default], result "{}", err I0514 23:25:16.721984 3702 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:16.722020 3702 pods.go:176] Deleting pod: tsf-tpa/test-tsf-tpa I0514 23:25:16.723379 3702 pods.go:236] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.128.0.57 I0514 23:25:16.723410 3702 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:16.743087 3702 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:16.743107 3702 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:16.750210 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0514 23:25:16.750231 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0514 23:25:17.482908 3702 namespace.go:98] [image-controller] adding namespace I0514 23:25:17.483818 3702 namespace.go:102] [image-controller] adding namespace took 884.803µs I0514 23:25:17.488765 3702 namespace.go:98] [default-tenant] adding namespace I0514 23:25:17.488794 3702 namespace.go:98] [integration-service] adding namespace I0514 23:25:17.488935 3702 namespace.go:98] [build-service] adding namespace I0514 23:25:17.489555 3702 namespace.go:102] [default-tenant] adding namespace took 771.888µs I0514 23:25:17.490217 3702 namespace.go:102] [integration-service] adding namespace took 1.410734ms I0514 23:25:17.490744 3702 namespace.go:102] [build-service] adding namespace took 1.795659ms I0514 23:25:17.511106 3702 namespace.go:142] [image-controller] updating namespace I0514 23:25:17.537212 3702 namespace.go:142] [image-controller] updating namespace I0514 23:25:17.545447 3702 namespace.go:142] [default-tenant] updating namespace I0514 23:25:17.575916 3702 namespace.go:142] [integration-service] updating namespace I0514 23:25:17.579438 3702 namespace.go:142] [default-tenant] updating namespace I0514 23:25:17.597070 3702 namespace.go:142] [integration-service] updating namespace I0514 23:25:17.600776 3702 namespace.go:142] [build-service] updating namespace I0514 23:25:17.626740 3702 namespace.go:142] [build-service] updating namespace I0514 23:25:17.747669 3702 namespace.go:142] [build-service] updating namespace I0514 23:25:17.813363 3702 namespace.go:142] [integration-service] updating namespace I0514 23:25:17.865870 3702 namespace.go:98] [release-service] adding namespace I0514 23:25:17.866740 3702 namespace.go:102] [release-service] adding namespace took 846.139µs I0514 23:25:17.900445 3702 namespace.go:142] [release-service] updating namespace I0514 23:25:17.923242 3702 namespace.go:142] [release-service] updating namespace I0514 23:25:18.017717 3702 namespace.go:142] [konflux-ui] updating namespace I0514 23:25:18.156917 3702 namespace.go:98] [namespace-lister] adding namespace I0514 23:25:18.156988 3702 namespace.go:98] [konflux-info] adding namespace I0514 23:25:18.157871 3702 namespace.go:102] [namespace-lister] adding namespace took 934.312µs I0514 23:25:18.158530 3702 namespace.go:102] [konflux-info] adding namespace took 1.530046ms I0514 23:25:18.204264 3702 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:18.222390 3702 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:18.237270 3702 namespace.go:142] [konflux-info] updating namespace I0514 23:25:18.254356 3702 namespace.go:142] [konflux-info] updating namespace I0514 23:25:18.312244 3702 namespace.go:98] [enterprise-contract-service] adding namespace I0514 23:25:18.313073 3702 namespace.go:102] [enterprise-contract-service] adding namespace took 805.756µs I0514 23:25:18.356267 3702 namespace.go:142] [enterprise-contract-service] updating namespace I0514 23:25:18.379974 3702 namespace.go:142] [enterprise-contract-service] updating namespace I0514 23:25:18.389765 3702 namespace.go:142] [image-controller] updating namespace I0514 23:25:19.001229 3702 namespace.go:142] [default-tenant] updating namespace I0514 23:25:19.615658 3702 namespace.go:98] [konflux-cli] adding namespace I0514 23:25:19.616535 3702 namespace.go:102] [konflux-cli] adding namespace took 854.822µs I0514 23:25:19.648104 3702 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:19.670301 3702 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:19.882632 3702 namespace.go:98] [segment-bridge] adding namespace I0514 23:25:19.883437 3702 namespace.go:102] [segment-bridge] adding namespace took 781.668µs I0514 23:25:20.016207 3702 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:20.040000 3702 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:20.256518 3702 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0514 23:25:20.257154 3702 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0514 23:25:20.257199 3702 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0514 23:25:20.260162 3702 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0514 23:25:20.406306 3702 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0514 23:25:20.406333 3702 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0514 23:25:20.407338 3702 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0514 23:25:20.421332 3702 cni.go:452] [tsf-tas/tuf-repository-init-s6cwq-rwlz5 ab2a9e909a8b9d80e7481329bb32ffae54175afe4a3263f52e1c596c12054828 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/tuf-repository-init-s6cwq-rwlz5 ab2a9e909a8b9d80e7481329bb32ffae54175afe4a3263f52e1c596c12054828 network default NAD default NAD key ] I0514 23:25:20.430559 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-s6cwq-rwlz5, hostIfaceName: ab2a9e909a8b9d8, network: default, NAD default, SandboxID: "ab2a9e909a8b9d80e7481329bb32ffae54175afe4a3263f52e1c596c12054828", PCI device ID: , UID: "d1b19940-5535-4c0a-8644-a175ea33b54f", MAC: 0a:58:0a:80:00:3a, IPs: [10.128.0.58/23] I0514 23:25:20.484306 3702 cni.go:473] [tsf-tas/tuf-repository-init-s6cwq-rwlz5 ab2a9e909a8b9d80e7481329bb32ffae54175afe4a3263f52e1c596c12054828 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/tuf-repository-init-s6cwq-rwlz5 ab2a9e909a8b9d80e7481329bb32ffae54175afe4a3263f52e1c596c12054828 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:de:83:da:a2:bc\",\"name\":\"ab2a9e909a8b9d8\"},{\"mac\":\"0a:58:0a:80:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7bd80428-cf47-49e5-b5d6-ba0733346ab1\"}],\"ips\":[{\"address\":\"10.128.0.58/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:25:21.029188 3702 namespace.go:142] [build-service] updating namespace I0514 23:25:21.870500 3702 namespace.go:142] [default-tenant] updating namespace I0514 23:25:21.998712 3702 namespace.go:142] [enterprise-contract-service] updating namespace I0514 23:25:22.104527 3702 namespace.go:142] [image-controller] updating namespace I0514 23:25:22.201259 3702 base_network_controller_pods.go:487] [default/konflux-ui/dex-dfd7f8dcc-vnlvd] creating logical port konflux-ui_dex-dfd7f8dcc-vnlvd for pod on switch ip-10-0-1-98.ec2.internal I0514 23:25:22.201355 3702 kube.go:256] Updating pod konflux-ui/dex-dfd7f8dcc-vnlvd I0514 23:25:22.248481 3702 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7cbf488fcf-bqws2] creating logical port build-service_build-service-controller-manager-7cbf488fcf-bqws2 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:25:22.248587 3702 kube.go:256] Updating pod build-service/build-service-controller-manager-7cbf488fcf-bqws2 I0514 23:25:22.283395 3702 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x] creating logical port integration-service_integration-service-controller-manager-7b666bf6d8-vkt8x for pod on switch ip-10-0-1-98.ec2.internal I0514 23:25:22.283493 3702 kube.go:256] Updating pod integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x I0514 23:25:22.316925 3702 pod.go:62] [konflux-ui/dex-dfd7f8dcc-vnlvd] pod update took 115.59573ms I0514 23:25:22.316950 3702 base_network_controller_pods.go:951] [default/konflux-ui/dex-dfd7f8dcc-vnlvd] addLogicalPort annotation time took 115.626122ms I0514 23:25:22.318304 3702 pods.go:271] [konflux-ui/dex-dfd7f8dcc-vnlvd] addLogicalPort took 117.062347ms, libovsdb time 890.696µs I0514 23:25:22.351730 3702 pod.go:62] [build-service/build-service-controller-manager-7cbf488fcf-bqws2] pod update took 103.175046ms I0514 23:25:22.351758 3702 base_network_controller_pods.go:951] [default/build-service/build-service-controller-manager-7cbf488fcf-bqws2] addLogicalPort annotation time took 103.208585ms I0514 23:25:22.351728 3702 pod.go:62] [integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x] pod update took 68.2595ms I0514 23:25:22.351974 3702 base_network_controller_pods.go:951] [default/integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x] addLogicalPort annotation time took 68.509492ms I0514 23:25:22.353310 3702 pods.go:271] [build-service/build-service-controller-manager-7cbf488fcf-bqws2] addLogicalPort took 104.84743ms, libovsdb time 1.011032ms I0514 23:25:22.353629 3702 pods.go:271] [integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x] addLogicalPort took 70.255354ms, libovsdb time 1.208527ms I0514 23:25:22.884749 3702 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-6bfdb6bcb6-5p68n] creating logical port release-service_release-service-controller-manager-6bfdb6bcb6-5p68n for pod on switch ip-10-0-1-98.ec2.internal I0514 23:25:22.884844 3702 kube.go:256] Updating pod release-service/release-service-controller-manager-6bfdb6bcb6-5p68n I0514 23:25:22.900452 3702 pod.go:62] [release-service/release-service-controller-manager-6bfdb6bcb6-5p68n] pod update took 15.632322ms I0514 23:25:22.900478 3702 base_network_controller_pods.go:951] [default/release-service/release-service-controller-manager-6bfdb6bcb6-5p68n] addLogicalPort annotation time took 15.664102ms I0514 23:25:22.901919 3702 pods.go:271] [release-service/release-service-controller-manager-6bfdb6bcb6-5p68n] addLogicalPort took 17.189962ms, libovsdb time 967.321µs I0514 23:25:23.024481 3702 namespace.go:142] [integration-service] updating namespace I0514 23:25:23.035996 3702 base_network_controller_pods.go:487] [default/konflux-ui/proxy-66dccd68c9-n2fn8] creating logical port konflux-ui_proxy-66dccd68c9-n2fn8 for pod on switch ip-10-0-1-98.ec2.internal I0514 23:25:23.036103 3702 kube.go:256] Updating pod konflux-ui/proxy-66dccd68c9-n2fn8 I0514 23:25:23.060330 3702 pod.go:62] [konflux-ui/proxy-66dccd68c9-n2fn8] pod update took 24.266566ms I0514 23:25:23.060354 3702 base_network_controller_pods.go:951] [default/konflux-ui/proxy-66dccd68c9-n2fn8] addLogicalPort annotation time took 24.297923ms I0514 23:25:23.061745 3702 pods.go:271] [konflux-ui/proxy-66dccd68c9-n2fn8] addLogicalPort took 25.763059ms, libovsdb time 917.805µs I0514 23:25:23.240274 3702 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:23.480453 3702 namespace.go:142] [konflux-info] updating namespace I0514 23:25:23.732074 3702 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:23.923386 3702 namespace.go:142] [release-service] updating namespace I0514 23:25:24.050771 3702 namespace.go:142] [build-service] updating namespace I0514 23:25:24.068736 3702 namespace.go:142] [default-tenant] updating namespace I0514 23:25:24.089823 3702 namespace.go:142] [enterprise-contract-service] updating namespace W0514 23:25:24.101742 3702 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0514 23:25:24.102606 3702 namespace.go:142] [image-controller] updating namespace I0514 23:25:24.117336 3702 namespace.go:142] [integration-service] updating namespace I0514 23:25:24.161789 3702 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:24.180391 3702 namespace.go:142] [konflux-info] updating namespace I0514 23:25:24.207771 3702 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:24.210898 3702 cni.go:452] [build-service/build-service-controller-manager-7cbf488fcf-bqws2 0dd615a9fe320f983416f12fa8c28b07b9164d2720a2e49d70d8142b804ab548 network default NAD default NAD key ] ADD starting CNI request [build-service/build-service-controller-manager-7cbf488fcf-bqws2 0dd615a9fe320f983416f12fa8c28b07b9164d2720a2e49d70d8142b804ab548 network default NAD default NAD key ] I0514 23:25:24.220842 3702 namespace.go:142] [release-service] updating namespace I0514 23:25:24.220857 3702 helper_linux.go:556] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7cbf488fcf-bqws2, hostIfaceName: 0dd615a9fe320f9, network: default, NAD default, SandboxID: "0dd615a9fe320f983416f12fa8c28b07b9164d2720a2e49d70d8142b804ab548", PCI device ID: , UID: "41fa0203-b728-4cbb-b6da-cdd30a1fe48b", MAC: 0a:58:0a:80:00:3c, IPs: [10.128.0.60/23] I0514 23:25:24.275336 3702 namespace.go:142] [build-service] updating namespace I0514 23:25:24.277831 3702 cni.go:473] [build-service/build-service-controller-manager-7cbf488fcf-bqws2 0dd615a9fe320f983416f12fa8c28b07b9164d2720a2e49d70d8142b804ab548 network default NAD default NAD key default] ADD finished CNI request [build-service/build-service-controller-manager-7cbf488fcf-bqws2 0dd615a9fe320f983416f12fa8c28b07b9164d2720a2e49d70d8142b804ab548 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:b4:96:9d:d0:3c\",\"name\":\"0dd615a9fe320f9\"},{\"mac\":\"0a:58:0a:80:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/abbf06c2-6a57-4d4e-affa-7dbb969ab194\"}],\"ips\":[{\"address\":\"10.128.0.60/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:25:24.336132 3702 namespace.go:142] [default-tenant] updating namespace I0514 23:25:24.399969 3702 namespace.go:142] [enterprise-contract-service] updating namespace I0514 23:25:24.472039 3702 namespace.go:142] [image-controller] updating namespace I0514 23:25:24.570222 3702 namespace.go:142] [integration-service] updating namespace I0514 23:25:24.674921 3702 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:24.777621 3702 namespace.go:142] [konflux-info] updating namespace I0514 23:25:24.879597 3702 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:24.967707 3702 namespace.go:142] [release-service] updating namespace I0514 23:25:26.224405 3702 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:26.354278 3702 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:26.455700 3702 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:26.672293 3702 cni.go:452] [tsf-tas/tuf-repository-init-s6cwq-rwlz5 ab2a9e909a8b9d80e7481329bb32ffae54175afe4a3263f52e1c596c12054828 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/tuf-repository-init-s6cwq-rwlz5 ab2a9e909a8b9d80e7481329bb32ffae54175afe4a3263f52e1c596c12054828 network default NAD default NAD key ] I0514 23:25:26.757094 3702 cni.go:473] [tsf-tas/tuf-repository-init-s6cwq-rwlz5 ab2a9e909a8b9d80e7481329bb32ffae54175afe4a3263f52e1c596c12054828 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/tuf-repository-init-s6cwq-rwlz5 ab2a9e909a8b9d80e7481329bb32ffae54175afe4a3263f52e1c596c12054828 network default NAD default NAD key default], result "{}", err I0514 23:25:26.830693 3702 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-s6cwq-rwlz5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:26.830726 3702 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-s6cwq-rwlz5 I0514 23:25:26.832258 3702 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-s6cwq-rwlz5, ips: 10.128.0.58 I0514 23:25:26.832287 3702 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-s6cwq-rwlz5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0514 23:25:27.675042 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0514 23:25:27.851966 3702 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-s6cwq-rwlz5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:27.851998 3702 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-s6cwq-rwlz5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:27.982491 3702 base_network_controller_pods.go:487] [default/tsf-tas/tuf-97fc8ff76-kq98c] creating logical port tsf-tas_tuf-97fc8ff76-kq98c for pod on switch ip-10-0-1-98.ec2.internal I0514 23:25:27.982586 3702 kube.go:256] Updating pod tsf-tas/tuf-97fc8ff76-kq98c I0514 23:25:27.997306 3702 pod.go:62] [tsf-tas/tuf-97fc8ff76-kq98c] pod update took 14.740727ms I0514 23:25:27.997334 3702 base_network_controller_pods.go:951] [default/tsf-tas/tuf-97fc8ff76-kq98c] addLogicalPort annotation time took 14.775534ms I0514 23:25:27.998873 3702 pods.go:271] [tsf-tas/tuf-97fc8ff76-kq98c] addLogicalPort took 16.406384ms, libovsdb time 959.485µs W0514 23:25:28.117252 3702 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0514 23:25:30.071028 3702 cni.go:452] [konflux-ui/dex-dfd7f8dcc-vnlvd ff6085e217984886fc24258ccafe488898f2a6763c8b4cbfcf1960136243f362 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/dex-dfd7f8dcc-vnlvd ff6085e217984886fc24258ccafe488898f2a6763c8b4cbfcf1960136243f362 network default NAD default NAD key ] I0514 23:25:30.082532 3702 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: dex-dfd7f8dcc-vnlvd, hostIfaceName: ff6085e21798488, network: default, NAD default, SandboxID: "ff6085e217984886fc24258ccafe488898f2a6763c8b4cbfcf1960136243f362", PCI device ID: , UID: "07d7f68b-e219-41ce-8e28-e44a6caaf622", MAC: 0a:58:0a:80:00:3b, IPs: [10.128.0.59/23] I0514 23:25:30.137160 3702 cni.go:473] [konflux-ui/dex-dfd7f8dcc-vnlvd ff6085e217984886fc24258ccafe488898f2a6763c8b4cbfcf1960136243f362 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/dex-dfd7f8dcc-vnlvd ff6085e217984886fc24258ccafe488898f2a6763c8b4cbfcf1960136243f362 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:b1:95:cc:54:f3\",\"name\":\"ff6085e21798488\"},{\"mac\":\"0a:58:0a:80:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ed312b1c-da23-4c18-af1e-0409e8243677\"}],\"ips\":[{\"address\":\"10.128.0.59/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:25:30.448531 3702 cni.go:452] [integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x eb90705a4bc90ff11856f25dd08cf46376ec6405dbf1bca9ee60dae7e649e67c network default NAD default NAD key ] ADD starting CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x eb90705a4bc90ff11856f25dd08cf46376ec6405dbf1bca9ee60dae7e649e67c network default NAD default NAD key ] I0514 23:25:30.457977 3702 helper_linux.go:556] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-7b666bf6d8-vkt8x, hostIfaceName: eb90705a4bc90ff, network: default, NAD default, SandboxID: "eb90705a4bc90ff11856f25dd08cf46376ec6405dbf1bca9ee60dae7e649e67c", PCI device ID: , UID: "bd766983-d09e-420a-8c9b-a65e40777841", MAC: 0a:58:0a:80:00:3d, IPs: [10.128.0.61/23] I0514 23:25:30.509334 3702 cni.go:473] [integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x eb90705a4bc90ff11856f25dd08cf46376ec6405dbf1bca9ee60dae7e649e67c network default NAD default NAD key default] ADD finished CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x eb90705a4bc90ff11856f25dd08cf46376ec6405dbf1bca9ee60dae7e649e67c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:02:3d:da:3d:16\",\"name\":\"eb90705a4bc90ff\"},{\"mac\":\"0a:58:0a:80:00:3d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cfd35b85-753e-461d-b003-f2a46e116d0b\"}],\"ips\":[{\"address\":\"10.128.0.61/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:25:30.746548 3702 cni.go:452] [tsf-tas/tuf-97fc8ff76-kq98c 4dfd427522d598264ce47bb0aa88867050409fbf7e233b963e330837f969a060 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/tuf-97fc8ff76-kq98c 4dfd427522d598264ce47bb0aa88867050409fbf7e233b963e330837f969a060 network default NAD default NAD key ] I0514 23:25:30.756073 3702 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: tuf-97fc8ff76-kq98c, hostIfaceName: 4dfd427522d5982, network: default, NAD default, SandboxID: "4dfd427522d598264ce47bb0aa88867050409fbf7e233b963e330837f969a060", PCI device ID: , UID: "0cea49f6-35ba-4f89-9dd5-9eb96204edc9", MAC: 0a:58:0a:80:00:40, IPs: [10.128.0.64/23] I0514 23:25:30.764162 3702 cni.go:452] [release-service/release-service-controller-manager-6bfdb6bcb6-5p68n 7f4c2418f0711f5d2e6713a9e807f2f1ec6b3eb20a1681b4cc7c8867cebe8b65 network default NAD default NAD key ] ADD starting CNI request [release-service/release-service-controller-manager-6bfdb6bcb6-5p68n 7f4c2418f0711f5d2e6713a9e807f2f1ec6b3eb20a1681b4cc7c8867cebe8b65 network default NAD default NAD key ] I0514 23:25:30.773262 3702 helper_linux.go:556] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-6bfdb6bcb6-5p68n, hostIfaceName: 7f4c2418f0711f5, network: default, NAD default, SandboxID: "7f4c2418f0711f5d2e6713a9e807f2f1ec6b3eb20a1681b4cc7c8867cebe8b65", PCI device ID: , UID: "1890c8e5-2f19-4899-b484-4712314b6e4b", MAC: 0a:58:0a:80:00:3e, IPs: [10.128.0.62/23] I0514 23:25:30.809255 3702 cni.go:473] [tsf-tas/tuf-97fc8ff76-kq98c 4dfd427522d598264ce47bb0aa88867050409fbf7e233b963e330837f969a060 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/tuf-97fc8ff76-kq98c 4dfd427522d598264ce47bb0aa88867050409fbf7e233b963e330837f969a060 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:b4:a7:f3:96:e5\",\"name\":\"4dfd427522d5982\"},{\"mac\":\"0a:58:0a:80:00:40\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e2bf3f73-a70c-4270-b0d0-9e554c6d7ac2\"}],\"ips\":[{\"address\":\"10.128.0.64/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:25:30.834800 3702 cni.go:473] [release-service/release-service-controller-manager-6bfdb6bcb6-5p68n 7f4c2418f0711f5d2e6713a9e807f2f1ec6b3eb20a1681b4cc7c8867cebe8b65 network default NAD default NAD key default] ADD finished CNI request [release-service/release-service-controller-manager-6bfdb6bcb6-5p68n 7f4c2418f0711f5d2e6713a9e807f2f1ec6b3eb20a1681b4cc7c8867cebe8b65 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:17:a4:12:3f:93\",\"name\":\"7f4c2418f0711f5\"},{\"mac\":\"0a:58:0a:80:00:3e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c65a3b4d-55d8-4b4b-abd7-5c142974794a\"}],\"ips\":[{\"address\":\"10.128.0.62/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:25:30.903426 3702 cni.go:452] [konflux-ui/proxy-66dccd68c9-n2fn8 842db5970dfa718cfcdbbd5d43b76024f0787f7a0ff264dd63512e0d3820fcd6 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-66dccd68c9-n2fn8 842db5970dfa718cfcdbbd5d43b76024f0787f7a0ff264dd63512e0d3820fcd6 network default NAD default NAD key ] I0514 23:25:30.914396 3702 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-66dccd68c9-n2fn8, hostIfaceName: 842db5970dfa718, network: default, NAD default, SandboxID: "842db5970dfa718cfcdbbd5d43b76024f0787f7a0ff264dd63512e0d3820fcd6", PCI device ID: , UID: "b9e16fde-9b12-4121-88c4-bf219e1cfc35", MAC: 0a:58:0a:80:00:3f, IPs: [10.128.0.63/23] I0514 23:25:30.984956 3702 cni.go:473] [konflux-ui/proxy-66dccd68c9-n2fn8 842db5970dfa718cfcdbbd5d43b76024f0787f7a0ff264dd63512e0d3820fcd6 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-66dccd68c9-n2fn8 842db5970dfa718cfcdbbd5d43b76024f0787f7a0ff264dd63512e0d3820fcd6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:c2:d6:10:a2:59\",\"name\":\"842db5970dfa718\"},{\"mac\":\"0a:58:0a:80:00:3f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/844eadad-b611-4b29-a3d1-23d91ff439b5\"}],\"ips\":[{\"address\":\"10.128.0.63/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0514 23:25:31.135656 3702 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0514 23:25:31.138764 3702 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0514 23:25:32.714613 3702 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0514 23:25:33.063442 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 W0514 23:25:33.720927 3702 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0514 23:25:33.722876 3702 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0514 23:25:36.175933 3702 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0514 23:25:36.175956 3702 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0514 23:25:38.682151 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0514 23:25:38.780408 3702 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0514 23:25:38.781563 3702 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0514 23:25:38.799981 3702 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0514 23:25:38.839033 3702 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0514 23:25:38.839048 3702 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0514 23:25:42.146614 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0514 23:25:42.146640 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0514 23:25:42.149540 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0514 23:25:42.149562 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service I0514 23:25:42.328622 3702 cni.go:452] [integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x eb90705a4bc90ff11856f25dd08cf46376ec6405dbf1bca9ee60dae7e649e67c network default NAD default NAD key ] DEL starting CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x eb90705a4bc90ff11856f25dd08cf46376ec6405dbf1bca9ee60dae7e649e67c network default NAD default NAD key ] I0514 23:25:42.438303 3702 cni.go:473] [integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x eb90705a4bc90ff11856f25dd08cf46376ec6405dbf1bca9ee60dae7e649e67c network default NAD default NAD key default] DEL finished CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x eb90705a4bc90ff11856f25dd08cf46376ec6405dbf1bca9ee60dae7e649e67c network default NAD default NAD key default], result "{}", err I0514 23:25:42.499130 3702 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:42.499194 3702 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x I0514 23:25:42.500889 3702 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x, ips: 10.128.0.61 I0514 23:25:42.500922 3702 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:42.792817 3702 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:42.792841 3702 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:42.796608 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x I0514 23:25:42.796637 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x W0514 23:25:49.771666 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex W0514 23:25:49.807052 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0514 23:25:49.807150 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service W0514 23:25:49.846962 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf I0514 23:26:10.180536 3702 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 W0514 23:26:17.258124 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0514 23:26:17.258150 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0514 23:26:17.258164 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0514 23:26:17.258193 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0514 23:26:18.856427 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0514 23:26:18.856463 3702 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0514 23:26:29.174199 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0514 23:26:48.357753 3702 obj_retry.go:498] Detected object konflux-ui/proxy-78867c4458-c44l9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:26:48.358377 3702 obj_retry.go:498] Detected object konflux-ui/proxy-78867c4458-c44l9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:26:48.589974 3702 obj_retry.go:492] Detected object konflux-ui/proxy-78867c4458-c44l9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:26:48.590001 3702 obj_retry.go:492] Detected object konflux-ui/proxy-78867c4458-c44l9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:26:48.598079 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-78867c4458-c44l9 I0514 23:26:48.598112 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-78867c4458-c44l9 I0514 23:26:49.733781 3702 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:26:49.734408 3702 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:26:49.778467 3702 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:26:49.778496 3702 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:26:49.785466 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0514 23:26:49.785493 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0514 23:27:09.563893 3702 namespace.go:98] [managed-tenant-hjy2g] adding namespace I0514 23:27:09.564746 3702 namespace.go:102] [managed-tenant-hjy2g] adding namespace took 827.288µs I0514 23:27:09.590558 3702 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:09.609258 3702 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:10.617109 3702 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:10.752027 3702 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:10.850981 3702 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:51.169648 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0514 23:27:55.066342 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=81 I0514 23:27:59.186333 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=5 I0514 23:28:02.152149 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=5 I0514 23:28:16.146334 3702 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0514 23:28:39.070454 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=171 I0514 23:29:01.157091 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0514 23:29:06.146968 3702 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0514 23:29:23.070618 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=376 I0514 23:29:41.185609 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0514 23:30:00.215464 3702 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29646690-92lfd for pod on switch ip-10-0-1-98.ec2.internal I0514 23:30:00.215594 3702 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd I0514 23:30:00.244485 3702 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd] pod update took 28.918483ms I0514 23:30:00.244511 3702 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd] addLogicalPort annotation time took 28.950596ms I0514 23:30:00.245822 3702 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd] addLogicalPort took 30.379837ms, libovsdb time 885.318µs I0514 23:30:00.578605 3702 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd 17373b638b853e4f22a131e3ca79ebe0f75b95975bce4d780f76fdbe18b2f650 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd 17373b638b853e4f22a131e3ca79ebe0f75b95975bce4d780f76fdbe18b2f650 network default NAD default NAD key ] I0514 23:30:00.588523 3702 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29646690-92lfd, hostIfaceName: 17373b638b853e4, network: default, NAD default, SandboxID: "17373b638b853e4f22a131e3ca79ebe0f75b95975bce4d780f76fdbe18b2f650", PCI device ID: , UID: "1075effb-822e-410e-bf75-e0da820298e4", MAC: 0a:58:0a:80:00:41, IPs: [10.128.0.65/23] I0514 23:30:00.643446 3702 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd 17373b638b853e4f22a131e3ca79ebe0f75b95975bce4d780f76fdbe18b2f650 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd 17373b638b853e4f22a131e3ca79ebe0f75b95975bce4d780f76fdbe18b2f650 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:09:c0:30:97:4a\",\"name\":\"17373b638b853e4\"},{\"mac\":\"0a:58:0a:80:00:41\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bee91e61-5af7-40ef-ba86-b83a007897cf\"}],\"ips\":[{\"address\":\"10.128.0.65/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0514 23:30:04.317429 3702 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd 17373b638b853e4f22a131e3ca79ebe0f75b95975bce4d780f76fdbe18b2f650 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd 17373b638b853e4f22a131e3ca79ebe0f75b95975bce4d780f76fdbe18b2f650 network default NAD default NAD key ] I0514 23:30:04.411380 3702 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd 17373b638b853e4f22a131e3ca79ebe0f75b95975bce4d780f76fdbe18b2f650 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd 17373b638b853e4f22a131e3ca79ebe0f75b95975bce4d780f76fdbe18b2f650 network default NAD default NAD key default], result "{}", err I0514 23:30:04.464644 3702 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:30:04.464683 3702 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd I0514 23:30:04.466211 3702 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd, ips: 10.128.0.65 I0514 23:30:04.466246 3702 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:30:05.330845 3702 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:30:05.330892 3702 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:30:11.067770 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=79 I0514 23:30:14.163026 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0514 23:31:09.063269 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=74 I0514 23:31:57.066062 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0514 23:33:26.172322 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0514 23:33:58.183246 3702 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0514 23:34:17.188732 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0514 23:34:51.068583 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=8 I0514 23:34:55.051397 3702 obj_retry.go:492] Detected object tsf-tpa/create-db-kjtlv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:34:55.051437 3702 obj_retry.go:492] Detected object tsf-tpa/create-db-kjtlv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:34:55.057521 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-kjtlv I0514 23:34:55.057550 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-kjtlv I0514 23:34:58.041030 3702 obj_retry.go:492] Detected object tsf-tpa/migrate-db-4fs8n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:34:58.041085 3702 obj_retry.go:492] Detected object tsf-tpa/migrate-db-4fs8n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:34:58.046462 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-4fs8n I0514 23:34:58.046491 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-4fs8n I0514 23:35:04.033432 3702 obj_retry.go:492] Detected object tsf-tpa/create-importers-x6kkn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:35:04.033481 3702 obj_retry.go:492] Detected object tsf-tpa/create-importers-x6kkn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:35:04.036332 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-x6kkn I0514 23:35:04.036356 3702 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-x6kkn I0514 23:35:13.072570 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=8 I0514 23:35:23.176973 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0514 23:35:42.155085 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0514 23:35:54.165386 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0514 23:36:33.149741 3702 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0514 23:37:04.188388 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0514 23:37:08.159285 3702 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0514 23:37:10.068770 3702 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0514 23:37:25.148380 3702 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=11