+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-192.ec2.internal ++ [[ -n ip-10-0-0-192.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-192.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0327 15:09:24.026629371 - disable conntrack on geneve port' I0327 15:09:24.026629371 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0327 15:09:24.045018562 - starting ovnkube-node + echo 'I0327 15:09:24.045018562 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-192.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-192.ec2.internal --init-node ip-10-0-0-192.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-192.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0327 15:09:24.084746 3815 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0327 15:09:24.084808 3815 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-bccd260e05.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0327 15:09:24.086686 3815 kube.go:419] Waiting for certificate I0327 15:09:24.086732 3815 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0327 15:09:24.086779 3815 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0327 15:09:24.087642 3815 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 15:09:24.118971 3815 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:09:24.118988 3815 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:09:24.121377 3815 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:09:24.121469 3815 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-4qd95" I0327 15:09:24.128110 3815 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-4qd95" I0327 15:09:24.128177 3815 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:09:25.087654 3815 kube.go:426] Certificate found I0327 15:09:25.088264 3815 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 15:09:25.089236 3815 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0327 15:09:25.134102 3815 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-28 15:04:24 +0000 UTC" deadline="2026-03-28 10:57:16.442549043 +0000 UTC" I0327 15:09:25.134265 3815 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h47m51.308301163s" I0327 15:09:25.156853 3815 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 15:09:25.175169 3815 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0327 15:09:25.175250 3815 factory.go:531] Starting watch factory I0327 15:09:25.175363 3815 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.175383 3815 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.175413 3815 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.175422 3815 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.175432 3815 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.175444 3815 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.175416 3815 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.175518 3815 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.175530 3815 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.175543 3815 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.175420 3815 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.175607 3815 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.177082 3815 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0327 15:09:25.179762 3815 controller_manager.go:368] Starting the ovnkube controller I0327 15:09:25.179809 3815 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-192.ec2.internal I0327 15:09:25.179867 3815 controller_manager.go:393] NBDB zone sync took: 48.627µs I0327 15:09:25.179877 3815 factory.go:531] Starting watch factory I0327 15:09:25.197154 3815 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.197181 3815 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.197216 3815 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.197411 3815 ovn_northd.go:121] Registering OVN northd metrics I0327 15:09:25.198060 3815 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.198069 3815 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0327 15:09:25.199568 3815 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.202645 3815 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0327 15:09:25.202669 3815 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0327 15:09:25.205740 3815 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:09:25.218027 3815 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0327 15:09:25.281196 3815 factory.go:1890] *v1.Service informer cache synced successfully I0327 15:09:25.281233 3815 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 15:09:25.281237 3815 factory.go:1890] *v1.Pod informer cache synced successfully I0327 15:09:25.281241 3815 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 15:09:25.281244 3815 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 15:09:25.281247 3815 factory.go:1890] *v1.Node informer cache synced successfully I0327 15:09:25.281325 3815 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:09:25.281346 3815 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:09:25.281325 3815 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:09:25.281364 3815 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:09:25.284452 3815 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:09:25.284457 3815 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:09:25.285284 3815 factory.go:1890] *v1.Pod informer cache synced successfully I0327 15:09:25.285294 3815 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 15:09:25.285298 3815 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 15:09:25.285301 3815 factory.go:1890] *v1.Node informer cache synced successfully I0327 15:09:25.285304 3815 factory.go:1890] *v1.Service informer cache synced successfully I0327 15:09:25.285307 3815 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 15:09:25.286436 3815 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 15:09:25.286447 3815 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 15:09:25.286521 3815 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.286541 3815 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.289176 3815 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.291194 3815 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 15:09:25.291205 3815 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 15:09:25.292400 3815 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 15:09:25.292455 3815 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.292468 3815 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.295186 3815 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.297177 3815 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 15:09:25.297869 3815 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 15:09:25.297925 3815 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.297936 3815 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.300346 3815 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.302354 3815 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 15:09:25.303562 3815 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 15:09:25.303642 3815 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.303653 3815 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.306217 3815 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.308219 3815 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 15:09:25.309459 3815 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 15:09:25.309522 3815 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.309537 3815 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.312096 3815 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.314044 3815 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 15:09:25.315397 3815 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 15:09:25.315933 3815 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:09:25.315948 3815 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:09:25.318563 3815 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:09:25.319475 3815 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 15:09:25.320618 3815 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 15:09:25.320680 3815 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.320693 3815 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.320751 3815 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.320770 3815 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.323250 3815 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.323432 3815 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:09:25.325483 3815 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 15:09:25.325773 3815 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 15:09:25.325784 3815 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 15:09:25.325794 3815 factory.go:636] Watch Factory start up complete, took: 150.545868ms I0327 15:09:25.325873 3815 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0327 15:09:25.330769 3815 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 15:09:25.330780 3815 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 15:09:25.330787 3815 factory.go:636] Watch Factory start up complete, took: 150.908993ms I0327 15:09:25.330794 3815 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-0-192.ec2.internal" zone I0327 15:09:25.367393 3815 default_node_network_controller.go:709] Initializing the default node network controller I0327 15:09:25.419355 3815 udn_isolation.go:101] Starting UDN host isolation manager I0327 15:09:25.425405 3815 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0327 15:09:25.469921 3815 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0327 15:09:25.470008 3815 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0327 15:09:25.470025 3815 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0327 15:09:25.501811 3815 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0327 15:09:25.501874 3815 default_node_network_controller.go:808] Node ip-10-0-0-192.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0327 15:09:25.507227 3815 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.192"] k8s.ovn.org/zone-name:ip-10-0-0-192.ec2.internal] on node ip-10-0-0-192.ec2.internal I0327 15:09:25.526484 3815 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0327 15:09:25.531427 3815 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0327 15:09:25.531546 3815 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0327 15:09:25.531686 3815 gateway_init.go:263] Preparing Gateway I0327 15:09:25.531699 3815 gateway_shared_intf.go:1658] Creating new gateway I0327 15:09:25.581269 3815 controller_manager.go:427] Waiting for node in zone sync took: 250.463337ms I0327 15:09:25.586823 3815 controller_manager.go:608] Switching to transit router for layer2 networks I0327 15:09:25.586842 3815 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-192.ec2.internal I0327 15:09:25.601630 3815 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 15:09:25.602927 3815 iptables.go:108] Creating table: filter chain: FORWARD I0327 15:09:25.607155 3815 controller_manager.go:288] SCTP support detected in OVN I0327 15:09:25.607842 3815 services_controller.go:69] Creating services controller for network=default I0327 15:09:25.607931 3815 udn_enabled_svc.go:80] Waiting for service informer to sync I0327 15:09:25.607965 3815 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0327 15:09:25.607998 3815 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0327 15:09:25.608015 3815 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0327 15:09:25.608025 3815 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 15:09:25.608172 3815 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0327 15:09:25.608221 3815 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0327 15:09:25.608246 3815 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 18.091µs I0327 15:09:25.608258 3815 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 35.311µs I0327 15:09:25.608258 3815 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 50.793µs I0327 15:09:25.608265 3815 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0327 15:09:25.608276 3815 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0327 15:09:25.608285 3815 default_network_controller.go:321] Starting the default network controller I0327 15:09:25.608348 3815 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 11.177µs I0327 15:09:25.608457 3815 udn_enabled_svc.go:95] Performing full resync I0327 15:09:25.608824 3815 udn_enabled_svc.go:99] Waiting for handler to sync I0327 15:09:25.608837 3815 udn_enabled_svc.go:104] Starting worker I0327 15:09:25.609233 3815 iptables.go:108] Creating table: filter chain: OUTPUT I0327 15:09:25.615309 3815 gateway_shared_intf.go:1787] Gateway Creation Complete I0327 15:09:25.615321 3815 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-192.ec2.internal","mac-address":"12:ce:c3:19:bd:e7","ip-addresses":["10.0.0.192/24"],"ip-address":"10.0.0.192/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:fd1cfb49-27bf-44b2-85c2-c5fcfec7fb37 k8s.ovn.org/node-encap-ips:["10.0.0.192"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.192/24"} k8s.ovn.org/zone-name:ip-10-0-0-192.ec2.internal] on node ip-10-0-0-192.ec2.internal I0327 15:09:25.616197 3815 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0327 15:09:25.616255 3815 repair.go:33] Syncing exgw routes took 45.975µs I0327 15:09:25.616274 3815 default_network_controller.go:407] Starting all the Watchers... I0327 15:09:25.616397 3815 namespace.go:98] [openshift-backplane-cse] adding namespace I0327 15:09:25.616405 3815 namespace.go:98] [openshift-marketplace] adding namespace I0327 15:09:25.616414 3815 namespace.go:98] [openshift-dns] adding namespace I0327 15:09:25.616422 3815 namespace.go:98] [openshift-authentication] adding namespace I0327 15:09:25.616415 3815 namespace.go:98] [openshift-backplane-srep] adding namespace I0327 15:09:25.616424 3815 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0327 15:09:25.616400 3815 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0327 15:09:25.616437 3815 namespace.go:98] [openshift-network-operator] adding namespace I0327 15:09:25.616439 3815 namespace.go:98] [openshift-ingress] adding namespace I0327 15:09:25.616443 3815 namespace.go:98] [openshift-image-registry] adding namespace I0327 15:09:25.616445 3815 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0327 15:09:25.616442 3815 namespace.go:98] [openshift-controller-manager] adding namespace I0327 15:09:25.616449 3815 namespace.go:98] [openshift-host-network] adding namespace I0327 15:09:25.616469 3815 namespace.go:98] [openshift-operators] adding namespace I0327 15:09:25.616400 3815 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0327 15:09:25.617024 3815 namespace.go:102] [openshift-backplane-cse] adding namespace took 615.856µs I0327 15:09:25.617041 3815 namespace.go:98] [openshift-machine-config-operator] adding namespace I0327 15:09:25.617560 3815 namespace.go:102] [openshift-machine-config-operator] adding namespace took 514.304µs I0327 15:09:25.617572 3815 namespace.go:98] [openshift-network-node-identity] adding namespace I0327 15:09:25.618120 3815 namespace.go:102] [openshift-network-node-identity] adding namespace took 540.967µs I0327 15:09:25.618137 3815 namespace.go:98] [openshift-backplane-cee] adding namespace I0327 15:09:25.618609 3815 namespace.go:102] [openshift-marketplace] adding namespace took 2.194275ms I0327 15:09:25.618623 3815 namespace.go:98] [kube-system] adding namespace I0327 15:09:25.619234 3815 namespace.go:102] [openshift-dns] adding namespace took 2.807681ms I0327 15:09:25.619249 3815 namespace.go:98] [openshift-must-gather-operator] adding namespace I0327 15:09:25.619766 3815 namespace.go:102] [openshift-authentication] adding namespace took 3.33674ms I0327 15:09:25.619784 3815 namespace.go:98] [openshift-machine-api] adding namespace I0327 15:09:25.620610 3815 namespace.go:102] [openshift-backplane-srep] adding namespace took 4.179382ms I0327 15:09:25.620626 3815 namespace.go:98] [default] adding namespace I0327 15:09:25.621136 3815 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 4.702003ms I0327 15:09:25.621150 3815 namespace.go:98] [openshift-console] adding namespace I0327 15:09:25.621717 3815 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 5.280355ms I0327 15:09:25.621727 3815 namespace.go:98] [openshift-config-operator] adding namespace I0327 15:09:25.622259 3815 namespace.go:102] [openshift-network-operator] adding namespace took 5.81372ms I0327 15:09:25.622276 3815 namespace.go:98] [open-cluster-management-2pa4nkn99kaqqvukhcr451snphcdn29u] adding namespace I0327 15:09:25.622740 3815 namespace.go:102] [openshift-ingress] adding namespace took 6.291516ms I0327 15:09:25.622756 3815 namespace.go:98] [openshift-operators-redhat] adding namespace I0327 15:09:25.623298 3815 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.8468ms I0327 15:09:25.623313 3815 namespace.go:98] [openshift-config-managed] adding namespace I0327 15:09:25.623887 3815 namespace.go:102] [openshift-image-registry] adding namespace took 7.436602ms I0327 15:09:25.623903 3815 namespace.go:98] [openshift-backplane] adding namespace I0327 15:09:25.624454 3815 namespace.go:102] [openshift-controller-manager] adding namespace took 7.999106ms I0327 15:09:25.624469 3815 namespace.go:98] [dedicated-admin] adding namespace I0327 15:09:25.625126 3815 namespace.go:102] [openshift-host-network] adding namespace took 8.667248ms I0327 15:09:25.625138 3815 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0327 15:09:25.625662 3815 namespace.go:102] [openshift-operators] adding namespace took 9.178828ms I0327 15:09:25.625679 3815 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0327 15:09:25.626290 3815 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 9.786437ms I0327 15:09:25.626302 3815 namespace.go:98] [openshift-insights] adding namespace W0327 15:09:25.626762 3815 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-8qmwh: pod kube-system/global-pull-secret-syncer-8qmwh: no pod IPs found I0327 15:09:25.626789 3815 namespace.go:102] [openshift-backplane-cee] adding namespace took 8.646402ms I0327 15:09:25.626803 3815 namespace.go:98] [openshift-dns-operator] adding namespace I0327 15:09:25.627317 3815 namespace.go:102] [kube-system] adding namespace took 8.686783ms I0327 15:09:25.627333 3815 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0327 15:09:25.627836 3815 namespace.go:102] [openshift-must-gather-operator] adding namespace took 8.581557ms I0327 15:09:25.627850 3815 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0327 15:09:25.628260 3815 namespace.go:102] [openshift-machine-api] adding namespace took 8.470358ms I0327 15:09:25.628273 3815 namespace.go:98] [openshift-node] adding namespace I0327 15:09:25.628842 3815 namespace.go:102] [default] adding namespace took 8.211703ms I0327 15:09:25.628855 3815 namespace.go:98] [openshift-logging] adding namespace I0327 15:09:25.629305 3815 namespace.go:102] [openshift-console] adding namespace took 8.147852ms I0327 15:09:25.629322 3815 namespace.go:98] [openshift-console-operator] adding namespace I0327 15:09:25.629866 3815 namespace.go:102] [openshift-config-operator] adding namespace took 8.131639ms I0327 15:09:25.629881 3815 namespace.go:98] [openshift-network-diagnostics] adding namespace I0327 15:09:25.630394 3815 namespace.go:102] [open-cluster-management-2pa4nkn99kaqqvukhcr451snphcdn29u] adding namespace took 8.110188ms I0327 15:09:25.630407 3815 namespace.go:98] [openshift-ingress-canary] adding namespace I0327 15:09:25.630801 3815 namespace.go:102] [openshift-operators-redhat] adding namespace took 8.038809ms I0327 15:09:25.630816 3815 namespace.go:98] [openshift-etcd] adding namespace I0327 15:09:25.631198 3815 namespace.go:102] [openshift-config-managed] adding namespace took 7.879673ms I0327 15:09:25.631210 3815 namespace.go:98] [openshift-apiserver] adding namespace I0327 15:09:25.632055 3815 namespace.go:102] [openshift-backplane] adding namespace took 8.146981ms I0327 15:09:25.632069 3815 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0327 15:09:25.632382 3815 namespace.go:102] [dedicated-admin] adding namespace took 7.908817ms I0327 15:09:25.632394 3815 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0327 15:09:25.632605 3815 default_node_network_controller.go:518] Node connection status = connected I0327 15:09:25.632873 3815 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.731319ms I0327 15:09:25.632884 3815 namespace.go:98] [kube-node-lease] adding namespace I0327 15:09:25.633322 3815 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 7.636077ms I0327 15:09:25.633339 3815 namespace.go:98] [openshift-customer-monitoring] adding namespace I0327 15:09:25.633783 3815 namespace.go:102] [openshift-insights] adding namespace took 7.477029ms I0327 15:09:25.633796 3815 namespace.go:98] [openshift-config] adding namespace I0327 15:09:25.634200 3815 namespace.go:102] [openshift-dns-operator] adding namespace took 7.390354ms I0327 15:09:25.634616 3815 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.277961ms I0327 15:09:25.634629 3815 namespace.go:98] [openshift-infra] adding namespace I0327 15:09:25.635083 3815 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 7.227855ms I0327 15:09:25.635096 3815 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0327 15:09:25.635510 3815 namespace.go:102] [openshift-node] adding namespace took 7.232822ms I0327 15:09:25.635520 3815 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0327 15:09:25.635935 3815 namespace.go:102] [openshift-logging] adding namespace took 7.076306ms I0327 15:09:25.635946 3815 namespace.go:98] [openshift-monitoring] adding namespace W0327 15:09:25.636380 3815 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-dcfk8: pod openshift-network-diagnostics/network-check-target-dcfk8: no pod IPs found W0327 15:09:25.636393 3815 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-rwdfv: pod openshift-network-diagnostics/network-check-target-rwdfv: no pod IPs found I0327 15:09:25.636401 3815 namespace.go:102] [openshift-console-operator] adding namespace took 7.074114ms I0327 15:09:25.636410 3815 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0327 15:09:25.636954 3815 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.069531ms I0327 15:09:25.636964 3815 namespace.go:98] [openshift-apiserver-operator] adding namespace I0327 15:09:25.637403 3815 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.990375ms I0327 15:09:25.637415 3815 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0327 15:09:25.637888 3815 namespace.go:102] [openshift-etcd] adding namespace took 7.063479ms I0327 15:09:25.637905 3815 namespace.go:98] [kube-public] adding namespace I0327 15:09:25.638310 3815 namespace.go:102] [openshift-apiserver] adding namespace took 7.095374ms I0327 15:09:25.638321 3815 namespace.go:98] [openshift-route-controller-manager] adding namespace I0327 15:09:25.638809 3815 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.734821ms I0327 15:09:25.638819 3815 namespace.go:98] [openshift-authentication-operator] adding namespace I0327 15:09:25.639209 3815 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.808888ms I0327 15:09:25.639223 3815 namespace.go:98] [openshift-console-user-settings] adding namespace I0327 15:09:25.639633 3815 namespace.go:102] [kube-node-lease] adding namespace took 6.742034ms I0327 15:09:25.639649 3815 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0327 15:09:25.640059 3815 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.715172ms I0327 15:09:25.640071 3815 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0327 15:09:25.640462 3815 namespace.go:102] [openshift-config] adding namespace took 6.66102ms I0327 15:09:25.640476 3815 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0327 15:09:25.640879 3815 namespace.go:102] [openshift-infra] adding namespace took 6.245679ms I0327 15:09:25.640892 3815 namespace.go:98] [openshift-package-operator] adding namespace I0327 15:09:25.641247 3815 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.144547ms I0327 15:09:25.641261 3815 namespace.go:98] [openshift-backplane-csm] adding namespace I0327 15:09:25.641657 3815 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.130922ms I0327 15:09:25.641672 3815 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0327 15:09:25.642126 3815 namespace.go:102] [openshift-monitoring] adding namespace took 6.175006ms I0327 15:09:25.642140 3815 namespace.go:98] [openshift-kube-apiserver] adding namespace I0327 15:09:25.642545 3815 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.128943ms I0327 15:09:25.642561 3815 namespace.go:98] [openshift-service-ca-operator] adding namespace I0327 15:09:25.642941 3815 namespace.go:102] [openshift-apiserver-operator] adding namespace took 5.971769ms I0327 15:09:25.642952 3815 namespace.go:98] [openshift] adding namespace I0327 15:09:25.643361 3815 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 5.941732ms I0327 15:09:25.643371 3815 namespace.go:98] [openshift-ingress-operator] adding namespace I0327 15:09:25.643931 3815 namespace.go:102] [kube-public] adding namespace took 6.021958ms I0327 15:09:25.643942 3815 namespace.go:98] [openshift-cluster-version] adding namespace I0327 15:09:25.644332 3815 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.004364ms I0327 15:09:25.644346 3815 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0327 15:09:25.644754 3815 namespace.go:102] [openshift-authentication-operator] adding namespace took 5.930281ms I0327 15:09:25.645126 3815 namespace.go:102] [openshift-console-user-settings] adding namespace took 5.898346ms I0327 15:09:25.645137 3815 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0327 15:09:25.645514 3815 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 5.860698ms I0327 15:09:25.645525 3815 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0327 15:09:25.645984 3815 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 5.90711ms I0327 15:09:25.645997 3815 namespace.go:98] [openshift-kube-scheduler] adding namespace I0327 15:09:25.646366 3815 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 5.885746ms I0327 15:09:25.646377 3815 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0327 15:09:25.646801 3815 namespace.go:102] [openshift-package-operator] adding namespace took 5.904732ms I0327 15:09:25.647169 3815 namespace.go:102] [openshift-backplane-csm] adding namespace took 5.904113ms I0327 15:09:25.647180 3815 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0327 15:09:25.647664 3815 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 5.986634ms I0327 15:09:25.648049 3815 namespace.go:102] [openshift-kube-apiserver] adding namespace took 5.904547ms I0327 15:09:25.648061 3815 namespace.go:98] [openshift-multus] adding namespace I0327 15:09:25.648561 3815 namespace.go:102] [openshift-service-ca-operator] adding namespace took 5.995145ms I0327 15:09:25.648575 3815 namespace.go:98] [openshift-backplane-tam] adding namespace I0327 15:09:25.649036 3815 namespace.go:102] [openshift] adding namespace took 6.078439ms I0327 15:09:25.649047 3815 namespace.go:98] [openshift-network-console] adding namespace I0327 15:09:25.649416 3815 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.039586ms I0327 15:09:25.649878 3815 namespace.go:102] [openshift-cluster-version] adding namespace took 5.931741ms I0327 15:09:25.650224 3815 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 5.87229ms I0327 15:09:25.650685 3815 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 5.542625ms I0327 15:09:25.651068 3815 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 5.538294ms I0327 15:09:25.651478 3815 namespace.go:102] [openshift-kube-scheduler] adding namespace took 5.477551ms I0327 15:09:25.651967 3815 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 5.586325ms I0327 15:09:25.651979 3815 namespace.go:98] [openshift-backplane-mobb] adding namespace W0327 15:09:25.652371 3815 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-dr7nn: pod openshift-multus/network-metrics-daemon-dr7nn: no pod IPs found W0327 15:09:25.652384 3815 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-nfpdk: pod openshift-multus/network-metrics-daemon-nfpdk: no pod IPs found I0327 15:09:25.652410 3815 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 5.21942ms I0327 15:09:25.652905 3815 namespace.go:102] [openshift-multus] adding namespace took 4.838321ms I0327 15:09:25.653279 3815 namespace.go:102] [openshift-backplane-tam] adding namespace took 4.700291ms I0327 15:09:25.653704 3815 namespace.go:102] [openshift-network-console] adding namespace took 4.653639ms I0327 15:09:25.654121 3815 namespace.go:102] [openshift-backplane-mobb] adding namespace took 2.137142ms I0327 15:09:25.654485 3815 zone_ic_handler.go:186] Time taken to create transit switch: 256.612µs I0327 15:09:25.654566 3815 master.go:532] Adding or Updating local node "ip-10-0-0-192.ec2.internal" for network "default" I0327 15:09:25.654636 3815 obj_retry.go:544] Failed to create *v1.Node ip-10-0-2-34.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-34.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-34.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-34.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-34.ec2.internal I0327 15:09:25.655094 3815 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-144.ec2.internal for the network default I0327 15:09:25.655947 3815 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-144.ec2.internal" on network "default" took: 862.201µs W0327 15:09:25.662425 3815 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-8qmwh: pod kube-system/global-pull-secret-syncer-8qmwh: no pod IPs found W0327 15:09:25.662443 3815 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-rwdfv: pod openshift-network-diagnostics/network-check-target-rwdfv: no pod IPs found W0327 15:09:25.662456 3815 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-dr7nn: pod openshift-multus/network-metrics-daemon-dr7nn: no pod IPs found I0327 15:09:25.663428 3815 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-0-192.ec2.internal for the network default I0327 15:09:25.664149 3815 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0327 15:09:25.664180 3815 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0327 15:09:25.664194 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-192.ec2.internal in network "default" I0327 15:09:25.664198 3815 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0327 15:09:25.664244 3815 node_tracker.go:169] Node ip-10-0-0-192.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:25.664255 3815 services_controller.go:574] Full service sync requested for network=default I0327 15:09:25.664260 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-144.ec2.internal in network "default" I0327 15:09:25.664282 3815 node_tracker.go:169] Node ip-10-0-1-144.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:25.664288 3815 services_controller.go:574] Full service sync requested for network=default I0327 15:09:25.664295 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-34.ec2.internal in network "default" I0327 15:09:25.664304 3815 node_tracker.go:235] Node ip-10-0-2-34.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-34.ec2.internal" I0327 15:09:25.664313 3815 node_tracker.go:169] Node ip-10-0-2-34.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:25.664317 3815 services_controller.go:574] Full service sync requested for network=default I0327 15:09:25.764637 3815 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0327 15:09:25.764663 3815 services_controller.go:208] Setting up event handlers for services for network=default I0327 15:09:25.764749 3815 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0327 15:09:25.764814 3815 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0327 15:09:25.764831 3815 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0327 15:09:25.865694 3815 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0327 15:09:25.865719 3815 repair.go:60] Starting repairing loop for services I0327 15:09:25.865806 3815 repair.go:131] Deleted 0 stale service LBs I0327 15:09:25.865826 3815 repair.go:137] Deleted 0 stale Chassis Template Vars I0327 15:09:25.865842 3815 repair.go:62] Finished repairing loop for services: 125.641µs I0327 15:09:25.865866 3815 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0327 15:09:25.865877 3815 services_controller.go:254] Starting workers for network=default I0327 15:09:25.866297 3815 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-nfpdk, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-nfpdk: suppressed error logged: pod openshift-multus/network-metrics-daemon-nfpdk: no pod IPs found I0327 15:09:25.866530 3815 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-rwdfv] creating logical port openshift-network-diagnostics_network-check-target-rwdfv for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:25.866637 3815 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-rwdfv I0327 15:09:25.866718 3815 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-dcfk8, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-dcfk8: suppressed error logged: pod openshift-network-diagnostics/network-check-target-dcfk8: no pod IPs found I0327 15:09:25.866830 3815 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-8qmwh] creating logical port kube-system_global-pull-secret-syncer-8qmwh for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:25.866924 3815 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-8qmwh I0327 15:09:25.882750 3815 pod.go:62] [kube-system/global-pull-secret-syncer-8qmwh] pod update took 15.849331ms I0327 15:09:25.882776 3815 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-8qmwh] addLogicalPort annotation time took 15.879355ms I0327 15:09:25.882891 3815 pod.go:62] [openshift-network-diagnostics/network-check-target-rwdfv] pod update took 16.313747ms I0327 15:09:25.882910 3815 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-rwdfv] addLogicalPort annotation time took 16.33699ms I0327 15:09:25.883808 3815 pods.go:271] [kube-system/global-pull-secret-syncer-8qmwh] addLogicalPort took 16.998883ms, libovsdb time 728.532µs I0327 15:09:25.883850 3815 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-dr7nn] creating logical port openshift-multus_network-metrics-daemon-dr7nn for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:25.883909 3815 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-dr7nn I0327 15:09:25.884080 3815 pods.go:271] [openshift-network-diagnostics/network-check-target-rwdfv] addLogicalPort took 17.557426ms, libovsdb time 879.657µs I0327 15:09:25.899916 3815 pod.go:62] [openshift-multus/network-metrics-daemon-dr7nn] pod update took 16.026833ms I0327 15:09:25.899939 3815 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-dr7nn] addLogicalPort annotation time took 16.055301ms I0327 15:09:25.901004 3815 pods.go:271] [openshift-multus/network-metrics-daemon-dr7nn] addLogicalPort took 17.159163ms, libovsdb time 786.759µs I0327 15:09:25.901534 3815 admin_network_policy_controller.go:227] Starting controller default-network-controller I0327 15:09:25.901553 3815 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0327 15:09:25.901567 3815 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0327 15:09:25.901571 3815 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0327 15:09:25.901763 3815 repair.go:29] Repairing admin network policies took 186.192µs I0327 15:09:25.901860 3815 repair.go:92] Repairing baseline admin network policies took 89.046µs I0327 15:09:25.902342 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0327 15:09:25.902363 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0327 15:09:25.904056 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0327 15:09:25.904077 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0327 15:09:25.904085 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0327 15:09:25.904394 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0327 15:09:25.904414 3815 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0327 15:09:25.904430 3815 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0327 15:09:25.906077 3815 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0327 15:09:25.906098 3815 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0327 15:09:25.906108 3815 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0327 15:09:25.908166 3815 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0327 15:09:25.908187 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0327 15:09:25.908198 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0327 15:09:25.910122 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0327 15:09:25.910140 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0327 15:09:25.910149 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0327 15:09:25.910618 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0327 15:09:25.910634 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0327 15:09:25.910641 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0327 15:09:25.911106 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0327 15:09:25.911125 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0327 15:09:25.911133 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0327 15:09:25.912463 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0327 15:09:25.912482 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0327 15:09:25.912492 3815 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0327 15:09:25.914016 3815 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0327 15:09:25.914035 3815 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0327 15:09:25.914046 3815 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0327 15:09:25.915955 3815 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0327 15:09:25.915977 3815 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0327 15:09:25.915988 3815 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0327 15:09:25.917183 3815 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0327 15:09:25.917201 3815 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0327 15:09:25.917212 3815 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0327 15:09:25.918291 3815 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0327 15:09:25.918310 3815 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0327 15:09:25.918319 3815 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0327 15:09:25.920068 3815 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0327 15:09:25.920089 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0327 15:09:25.920098 3815 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0327 15:09:25.920568 3815 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0327 15:09:25.920599 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0327 15:09:25.921355 3815 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0327 15:09:25.921369 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 15:09:25.922136 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0327 15:09:25.922157 3815 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0327 15:09:25.922165 3815 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0327 15:09:25.922673 3815 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0327 15:09:25.922691 3815 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0327 15:09:25.922698 3815 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0327 15:09:25.923020 3815 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0327 15:09:25.923038 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0327 15:09:25.923046 3815 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0327 15:09:25.923755 3815 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0327 15:09:25.923774 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0327 15:09:25.923782 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0327 15:09:25.924261 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0327 15:09:25.924278 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0327 15:09:25.924286 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0327 15:09:25.924765 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0327 15:09:25.924783 3815 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0327 15:09:25.924790 3815 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0327 15:09:25.925051 3815 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0327 15:09:25.925068 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0327 15:09:25.925075 3815 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0327 15:09:25.925514 3815 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0327 15:09:25.925529 3815 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0327 15:09:25.925538 3815 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0327 15:09:25.926748 3815 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0327 15:09:25.926765 3815 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0327 15:09:25.926775 3815 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0327 15:09:25.927958 3815 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0327 15:09:25.927976 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0327 15:09:25.927984 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0327 15:09:25.928439 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0327 15:09:25.928456 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0327 15:09:25.928477 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 15:09:25.929179 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0327 15:09:25.929199 3815 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0327 15:09:25.929207 3815 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0327 15:09:25.930225 3815 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0327 15:09:25.930245 3815 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0327 15:09:25.930257 3815 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0327 15:09:25.931056 3815 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0327 15:09:25.931072 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0327 15:09:25.931081 3815 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0327 15:09:25.931319 3815 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0327 15:09:25.937348 3815 egressfirewall.go:319] Starting EgressFirewall controller I0327 15:09:25.937383 3815 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0327 15:09:25.937398 3815 controller.go:133] Adding controller egress-firewall-controller event handlers I0327 15:09:25.937406 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0327 15:09:25.937418 3815 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0327 15:09:25.937423 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0327 15:09:25.937432 3815 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0327 15:09:25.937707 3815 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0327 15:09:25.937721 3815 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0327 15:09:25.937732 3815 egressqos.go:193] Setting up event handlers for EgressQoS I0327 15:09:25.937803 3815 egressfirewall.go:628] Syncing node "ip-10-0-0-192.ec2.internal" for egress firewall I0327 15:09:25.937816 3815 egressfirewall.go:628] Syncing node "ip-10-0-1-144.ec2.internal" for egress firewall I0327 15:09:25.937820 3815 egressfirewall.go:628] Syncing node "ip-10-0-2-34.ec2.internal" for egress firewall I0327 15:09:25.937870 3815 egressqos.go:245] Starting EgressQoS Controller I0327 15:09:25.937881 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0327 15:09:25.937887 3815 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0327 15:09:25.937892 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0327 15:09:25.937897 3815 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0327 15:09:25.937902 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0327 15:09:25.937907 3815 shared_informer.go:356] "Caches are synced" controller="egressqos" I0327 15:09:25.937911 3815 egressqos.go:259] Repairing EgressQoSes I0327 15:09:25.937914 3815 egressqos.go:399] Starting repairing loop for egressqos I0327 15:09:25.937979 3815 egressqos.go:401] Finished repairing loop for egressqos: 64.044µs I0327 15:09:25.937993 3815 egressservice_zone.go:132] Setting up event handlers for Egress Services I0327 15:09:25.938027 3815 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-192.ec2.internal I0327 15:09:25.938075 3815 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-192.ec2.internal : 51.778µs I0327 15:09:25.938141 3815 egressservice_zone.go:210] Starting Egress Services Controller I0327 15:09:25.938151 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 15:09:25.938158 3815 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 15:09:25.938163 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 15:09:25.938168 3815 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 15:09:25.938173 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 15:09:25.938178 3815 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 15:09:25.938184 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0327 15:09:25.938188 3815 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0327 15:09:25.938192 3815 egressservice_zone.go:228] Repairing Egress Services I0327 15:09:25.938473 3815 master_controller.go:88] Starting Admin Policy Based Route Controller I0327 15:09:25.938484 3815 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 15:09:25.938539 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-192.ec2.internal I0327 15:09:25.938550 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-192.ec2.internal: 12.392µs I0327 15:09:25.938557 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-144.ec2.internal I0327 15:09:25.938566 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-144.ec2.internal: 9.532µs I0327 15:09:25.938571 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-34.ec2.internal I0327 15:09:25.938577 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-34.ec2.internal: 6.21µs I0327 15:09:25.938685 3815 default_network_controller.go:568] Completing all the Watchers took 322.402601ms I0327 15:09:25.938698 3815 default_network_controller.go:572] Starting unidling controllers I0327 15:09:25.938710 3815 unidle.go:46] Registering OVN SB ControllerEvent handler I0327 15:09:25.938715 3815 unidle.go:63] Populating Initial ContollerEvent events I0327 15:09:25.938730 3815 unidle.go:79] Setting up event handlers for services I0327 15:09:25.938775 3815 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0327 15:09:26.133400 3815 default_node_network_controller.go:518] Node connection status = connected I0327 15:09:26.633674 3815 default_node_network_controller.go:518] Node connection status = connected I0327 15:09:27.133641 3815 default_node_network_controller.go:518] Node connection status = connected I0327 15:09:27.633497 3815 default_node_network_controller.go:518] Node connection status = connected I0327 15:09:27.774739 3815 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0327 15:09:28.133901 3815 default_node_network_controller.go:518] Node connection status = connected I0327 15:09:28.148913 3815 bridgeconfig.go:537] Gateway is ready I0327 15:09:28.148934 3815 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0327 15:09:28.159535 3815 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.192. Updating OVN encap IP. I0327 15:09:28.164225 3815 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.192 - it is already configured I0327 15:09:28.164243 3815 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.192/24:{} 172.20.0.1/32:{}]. Updating annotations. I0327 15:09:28.164463 3815 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.192/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-192.ec2.internal","mac-address":"12:ce:c3:19:bd:e7","ip-addresses":["10.0.0.192/24"],"ip-address":"10.0.0.192/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:fd1cfb49-27bf-44b2-85c2-c5fcfec7fb37 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.192/24"}] on node ip-10-0-0-192.ec2.internal I0327 15:09:28.181798 3815 egressfirewall.go:628] Syncing node "ip-10-0-0-192.ec2.internal" for egress firewall I0327 15:09:28.181837 3815 master.go:532] Adding or Updating local node "ip-10-0-0-192.ec2.internal" for network "default" I0327 15:09:28.181858 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-192.ec2.internal I0327 15:09:28.181866 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-192.ec2.internal in network "default" I0327 15:09:28.181877 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-192.ec2.internal: 23.518µs I0327 15:09:28.181927 3815 node_tracker.go:169] Node ip-10-0-0-192.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:28.181998 3815 services_controller.go:574] Full service sync requested for network=default I0327 15:09:28.182046 3815 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.0.192 I0327 15:09:28.182145 3815 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-192.ec2.internal I0327 15:09:28.186205 3815 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-192.ec2.internal. Updating no re-route policies I0327 15:09:28.186275 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-t4dmw I0327 15:09:28.186283 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-hbzjk I0327 15:09:28.186290 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-t4dmw I0327 15:09:28.186291 3815 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-8qmwh I0327 15:09:28.186307 3815 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-8qmwh I0327 15:09:28.186308 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-jw7nm I0327 15:09:28.186312 3815 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-t4dmw after 0 failed attempt(s) I0327 15:09:28.186276 3815 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-cwg49 I0327 15:09:28.186322 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-jw7nm I0327 15:09:28.186325 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-hbzjk I0327 15:09:28.186328 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-fj76v I0327 15:09:28.186332 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-vkk6p I0327 15:09:28.186331 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-rwdfv I0327 15:09:28.186308 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-g5xj5 I0327 15:09:28.186342 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-fj76v I0327 15:09:28.186341 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-dcfk8 I0327 15:09:28.186347 3815 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-hbzjk after 0 failed attempt(s) I0327 15:09:28.186339 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-vkk6p I0327 15:09:28.186351 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-dr7nn I0327 15:09:28.186355 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-dcfk8 I0327 15:09:28.186349 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-rwdfv I0327 15:09:28.186356 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-nwn4l I0327 15:09:28.186358 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-f76cq I0327 15:09:28.186366 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-192.ec2.internal I0327 15:09:28.186367 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-nwn4l I0327 15:09:28.186367 3815 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-nfpdk I0327 15:09:28.186387 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-nfpdk I0327 15:09:28.186389 3815 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-rwdfv] creating logical port openshift-network-diagnostics_network-check-target-rwdfv for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:28.186415 3815 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-vkk6p after 0 failed attempt(s) I0327 15:09:28.186344 3815 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-8qmwh] creating logical port kube-system_global-pull-secret-syncer-8qmwh for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:28.186433 3815 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-dcfk8, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-dcfk8: suppressed error logged: pod openshift-network-diagnostics/network-check-target-dcfk8: no pod IPs found I0327 15:09:28.186435 3815 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-nfpdk, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-nfpdk: suppressed error logged: pod openshift-multus/network-metrics-daemon-nfpdk: no pod IPs found I0327 15:09:28.186358 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-dr7nn I0327 15:09:28.186476 3815 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-dr7nn] creating logical port openshift-multus_network-metrics-daemon-dr7nn for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:28.186345 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-g5xj5 I0327 15:09:28.186369 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-f76cq I0327 15:09:28.186344 3815 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-192.ec2.internal I0327 15:09:28.186531 3815 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-192.ec2.internal I0327 15:09:28.186535 3815 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-f76cq after 0 failed attempt(s) I0327 15:09:28.186368 3815 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-fj76v after 0 failed attempt(s) I0327 15:09:28.186389 3815 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-nwn4l after 0 failed attempt(s) I0327 15:09:28.186338 3815 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-jw7nm after 0 failed attempt(s) I0327 15:09:28.186548 3815 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-192.ec2.internal after 0 failed attempt(s) I0327 15:09:28.186520 3815 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-g5xj5 after 0 failed attempt(s) I0327 15:09:28.186326 3815 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-cwg49 I0327 15:09:28.186576 3815 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-cwg49 after 0 failed attempt(s) I0327 15:09:28.186373 3815 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-192.ec2.internal I0327 15:09:28.186625 3815 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-192.ec2.internal after 0 failed attempt(s) I0327 15:09:28.187171 3815 pods.go:271] [kube-system/global-pull-secret-syncer-8qmwh] addLogicalPort took 833.421µs, libovsdb time 323.487µs I0327 15:09:28.187187 3815 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-8qmwh after 0 failed attempt(s) I0327 15:09:28.187307 3815 pods.go:271] [openshift-multus/network-metrics-daemon-dr7nn] addLogicalPort took 839.06µs, libovsdb time 341.869µs I0327 15:09:28.187321 3815 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-dr7nn after 0 failed attempt(s) I0327 15:09:28.187557 3815 pods.go:271] [openshift-network-diagnostics/network-check-target-rwdfv] addLogicalPort took 1.181076ms, libovsdb time 198.828µs I0327 15:09:28.187570 3815 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-rwdfv after 0 failed attempt(s) I0327 15:09:28.196237 3815 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0327 15:09:28.209139 3815 iptables.go:108] Creating table: mangle chain: OUTPUT I0327 15:09:28.212515 3815 iptables.go:108] Creating table: nat chain: OUTPUT I0327 15:09:28.235289 3815 iptables.go:108] Creating table: nat chain: PREROUTING I0327 15:09:28.282503 3815 iptables.go:108] Creating table: filter chain: FORWARD I0327 15:09:28.299606 3815 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0327 15:09:28.299761 3815 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0327 15:09:28.299816 3815 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0327 15:09:28.299900 3815 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0327 15:09:28.299961 3815 gateway_init.go:325] Gateway and management port readiness took 2.670419921s I0327 15:09:28.299989 3815 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0327 15:09:28.300024 3815 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0327 15:09:28.300036 3815 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0327 15:09:28.300044 3815 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 15:09:28.300101 3815 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0327 15:09:28.300179 3815 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 51.087µs I0327 15:09:28.300458 3815 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0327 15:09:28.300484 3815 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 15.39µs I0327 15:09:28.300492 3815 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 35.333µs I0327 15:09:28.300498 3815 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0327 15:09:28.300507 3815 nad_controller.go:173] [node-nad-controller NAD controller]: started I0327 15:09:28.300514 3815 default_node_network_controller.go:898] Starting the default node network controller I0327 15:09:28.300533 3815 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0327 15:09:28.300539 3815 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 17.961µs I0327 15:09:28.300740 3815 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0327 15:09:28.300750 3815 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0327 15:09:28.300774 3815 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a93800 0xc0002d2000 0xc0002d2070 0xc0002d20e0 0xc0002d2150 0xc0002d2230 0xc0002d22a0 0xc000aaf950 0xc0002d2310 0xc0002d2380 0xc0004780a0 0xc0064678f0 0xc0002d23f0 map[0x2f67040:0xc002d8fce0 0x2f67580:0xc004d86ba0 0x2f68000:0xc00270aba0 0x2f682a0:0xc002d8fc80 0x2f687e0:0xc002d8fd40 0x2f68a80:0xc002d8fda0 0x2f68d20:0xc004d86b40 0x2f69500:0xc004d86c00 0x2f697a0:0xc004d86c60 0x2f93d40:0xc000d391a0 0x2f940e0:0xc000d39200 0x2f94bc0:0xc000b49bc0 0x2f96520:0xc001c2e000 0x2f97000:0xc00270ab40 0x2f97e80:0xc001c2e060] 0xc000ab08c0 0} I0327 15:09:28.300836 3815 gateway.go:151] Starting gateway service sync I0327 15:09:28.301277 3815 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0327 15:09:28.303462 3815 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0327 15:09:28.313844 3815 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0327 15:09:28.315040 3815 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0327 15:09:28.316229 3815 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0327 15:09:28.348737 3815 gateway.go:168] Gateway service sync done. Time taken: 47.886844ms I0327 15:09:28.349384 3815 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31433 I0327 15:09:28.349600 3815 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a93800 0xc0002d2000 0xc0002d2070 0xc0002d20e0 0xc0002d2150 0xc0002d2230 0xc0002d22a0 0xc000aaf950 0xc0002d2310 0xc0002d2380 0xc0004780a0 0xc0064678f0 0xc0002d23f0 map[0x2f67040:0xc002d8fce0 0x2f67580:0xc004d86ba0 0x2f68000:0xc00270aba0 0x2f682a0:0xc002d8fc80 0x2f687e0:0xc002d8fd40 0x2f68a80:0xc002d8fda0 0x2f68d20:0xc004d86b40 0x2f69500:0xc004d86c00 0x2f697a0:0xc004d86c60 0x2f93d40:0xc000d391a0 0x2f940e0:0xc000d39200 0x2f94bc0:0xc000b49bc0 0x2f96520:0xc001c2e000 0x2f97000:0xc00270ab40 0x2f97e80:0xc001c2e060] 0xc000ab08c0 0} I0327 15:09:28.350098 3815 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a93800 0xc0002d2000 0xc0002d2070 0xc0002d20e0 0xc0002d2150 0xc0002d2230 0xc0002d22a0 0xc000aaf950 0xc0002d2310 0xc0002d2380 0xc0004780a0 0xc0064678f0 0xc0002d23f0 map[0x2f67040:0xc002d8fce0 0x2f67580:0xc004d86ba0 0x2f68000:0xc00270aba0 0x2f682a0:0xc002d8fc80 0x2f687e0:0xc002d8fd40 0x2f68a80:0xc002d8fda0 0x2f68d20:0xc004d86b40 0x2f69500:0xc004d86c00 0x2f697a0:0xc004d86c60 0x2f93d40:0xc000d391a0 0x2f940e0:0xc000d39200 0x2f94bc0:0xc000b49bc0 0x2f96520:0xc001c2e000 0x2f97000:0xc00270ab40 0x2f97e80:0xc001c2e060] 0xc000ab08c0 0} I0327 15:09:28.350134 3815 gateway.go:519] Reconciling gateway with updates I0327 15:09:28.355656 3815 gateway.go:326] Spawning Conntrack Rule Check Thread I0327 15:09:28.355714 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 15:09:28.355725 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:09:28.355735 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 15:09:28.355742 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:09:28.355740 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:09:28.355759 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:09:28.355766 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:09:28.355777 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:09:28.355790 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:09:28.355797 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:09:28.355805 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:09:28.355801 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 15:09:28.355813 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:09:28.355816 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:09:28.355816 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:09:28.355823 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:09:28.355823 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:09:28.355827 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:09:28.355832 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:09:28.355833 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 15:09:28.355834 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:09:28.355845 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 15:09:28.355846 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:09:28.355847 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 15:09:28.355852 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 15:09:28.355792 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:09:28.355846 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:09:28.355863 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 15:09:28.355868 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 15:09:28.355871 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:09:28.355867 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:09:28.355881 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:09:28.355805 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:09:28.355884 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:09:28.355889 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:09:28.355891 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:09:28.355891 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 15:09:28.355889 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 15:09:28.355898 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:09:28.355904 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:09:28.355912 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:09:28.355913 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:09:28.355914 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:09:28.355922 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:09:28.355929 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:09:28.355937 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 15:09:28.355941 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 15:09:28.355897 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:09:28.355959 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:09:28.355971 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 15:09:28.355987 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 15:09:28.356000 3815 port_claim.go:71] Svc openshift-ingress/router-default: port 32501 is already open, no action needed I0327 15:09:28.355997 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:09:28.356009 3815 port_claim.go:71] Svc openshift-ingress/router-default: port 32216 is already open, no action needed I0327 15:09:28.356014 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:09:28.356018 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:09:28.356025 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:09:28.356048 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 15:09:28.356058 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:09:28.356058 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 15:09:28.356068 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:09:28.356066 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:09:28.355961 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 15:09:28.356072 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:09:28.356072 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:09:28.356083 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:09:28.355790 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:09:28.356088 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:09:28.356093 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 15:09:28.356084 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:09:28.356067 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:09:28.356108 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 15:09:28.355974 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:09:28.356124 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:09:28.355826 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 15:09:28.356168 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 15:09:28.355882 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:09:28.355914 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:09:28.356243 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:09:28.356251 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:09:28.356263 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:09:28.356269 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:09:28.356267 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 15:09:28.355886 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 15:09:28.356279 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:09:28.356285 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 15:09:28.355867 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:09:28.356329 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 15:09:28.355894 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 15:09:28.355856 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 15:09:28.355859 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:09:28.355869 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 15:09:28.356382 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 15:09:28.355807 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:09:28.356404 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:09:28.355872 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:09:28.356418 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:09:28.355875 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 15:09:28.355882 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 15:09:28.355883 3815 default_node_network_controller.go:943] Gateway and management port readiness took 55.122957ms I0327 15:09:28.355892 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:09:28.356494 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:09:28.356500 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 15:09:28.355912 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:09:28.356508 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:09:28.356513 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 15:09:28.355948 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 15:09:28.356054 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:09:28.356524 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:09:28.356072 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 15:09:28.356070 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:09:28.356623 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:09:28.356112 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:09:28.356650 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 15:09:28.357017 3815 node_ip_handler_linux.go:160] Node IP manager is running W0327 15:09:28.390913 3815 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 15:09:28.391138 3815 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 15:09:28.391336 3815 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0327 15:09:28.400466 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 15:09:28.400474 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 15:09:28.400490 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 15:09:28.400498 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 15:09:28.400507 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 15:09:28.400512 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 15:09:28.400518 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 15:09:28.400523 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 15:09:28.400530 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 15:09:28.400535 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 15:09:28.400540 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 15:09:28.400546 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 15:09:28.437804 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 15:09:28.456419 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 15:09:28.456427 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 15:09:28.456440 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 15:09:28.456455 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 15:09:28.456465 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 15:09:28.456472 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 15:09:28.456488 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 15:09:28.456493 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) W0327 15:09:29.573136 3815 egressip_healthcheck.go:76] Health checking using insecure connection I0327 15:09:29.573162 3815 default_node_network_controller.go:1364] Starting node controller node sync I0327 15:09:29.573179 3815 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0327 15:09:29.585007 3815 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.828015ms I0327 15:09:29.585127 3815 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-144.ec2.internal", IP: 10.0.1.144 to PMTUD blocking rules I0327 15:09:29.585141 3815 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-144.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 15:09:29.591525 3815 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0327 15:09:29.591807 3815 egressservice_node.go:93] Setting up event handlers for Egress Services I0327 15:09:29.591973 3815 egressservice_node.go:183] Starting Egress Services Controller I0327 15:09:29.591988 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 15:09:29.591999 3815 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 15:09:29.592005 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 15:09:29.592026 3815 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 15:09:29.592033 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 15:09:29.592038 3815 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 15:09:29.592042 3815 egressservice_node.go:197] Repairing Egress Services I0327 15:09:29.601751 3815 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0327 15:09:29.601762 3815 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 15:09:29.601923 3815 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0327 15:09:29.601940 3815 link_network_manager.go:117] Link manager is running I0327 15:09:29.601951 3815 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0327 15:09:29.602030 3815 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0327 15:09:29.602341 3815 vrf_manager.go:128] VRF manager is running I0327 15:09:29.602453 3815 node_controller_manager.go:437] Removing flows to drop GARP I0327 15:09:29.602466 3815 gateway.go:519] Reconciling gateway with updates I0327 15:09:29.625600 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:09:29.625614 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:09:29.625624 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:09:29.625635 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:09:29.625639 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:09:29.625637 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:09:29.625649 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:09:29.625648 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 15:09:29.625655 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:09:29.625657 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:09:29.625636 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:09:29.625663 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:09:29.625665 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 15:09:29.625627 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:09:29.625667 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:09:29.625676 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 15:09:29.625646 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:09:29.625682 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:09:29.625688 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:09:29.625696 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:09:29.625698 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 15:09:29.625697 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:09:29.625713 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:09:29.625599 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:09:29.625723 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:09:29.625726 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:09:29.625730 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:09:29.625735 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 15:09:29.625737 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:09:29.625749 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 15:09:29.625715 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 15:09:29.625759 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 15:09:29.625661 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:09:29.625767 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 15:09:29.625768 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 15:09:29.625771 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 15:09:29.625776 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:09:29.625782 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 15:09:29.625780 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:09:29.625782 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 15:09:29.625794 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:09:29.625757 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 15:09:29.625800 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 15:09:29.625795 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:09:29.625811 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:09:29.625769 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 15:09:29.625784 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:09:29.625825 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:09:29.625858 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 15:09:29.625859 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 15:09:29.625740 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:09:29.625882 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 15:09:29.625781 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:09:29.625887 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:09:29.625900 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:09:29.625904 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:09:29.625906 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:09:29.625903 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:09:29.625915 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:09:29.625919 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:09:29.625923 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:09:29.625932 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:09:29.625935 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:09:29.625804 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:09:29.625943 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:09:29.625886 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:09:29.625953 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 15:09:29.625961 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 15:09:29.625958 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:09:29.625972 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:09:29.625971 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 15:09:29.625978 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 15:09:29.625975 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 15:09:29.625982 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:09:29.625983 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:09:29.625755 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 15:09:29.625993 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 15:09:29.625998 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:09:29.625995 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:09:29.626004 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 15:09:29.625877 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:09:29.626039 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:09:29.625803 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:09:29.625915 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:09:29.626058 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:09:29.626060 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:09:29.626065 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 15:09:29.626067 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:09:29.626069 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:09:29.626063 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 15:09:29.626072 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:09:29.625891 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 15:09:29.626087 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 15:09:29.625908 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:09:29.625795 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 15:09:29.626133 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 15:09:29.626147 3815 port_claim.go:71] Svc openshift-ingress/router-default: port 32501 is already open, no action needed I0327 15:09:29.626154 3815 port_claim.go:71] Svc openshift-ingress/router-default: port 32216 is already open, no action needed I0327 15:09:29.625911 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:09:29.626187 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 15:09:29.625805 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:09:29.626198 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:09:29.625967 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 15:09:29.625926 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:09:29.626228 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:09:29.625982 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 15:09:29.625979 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:09:29.625817 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:09:29.626052 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 15:09:29.626073 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 15:09:29.625892 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:09:29.626305 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:09:29.625892 3815 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:09:29.626332 3815 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:09:29.626337 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 15:09:29.625896 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 15:09:29.664396 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 15:09:29.664422 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 15:09:29.664430 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 15:09:29.664437 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 15:09:29.664452 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 15:09:29.705942 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 15:09:29.705975 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 15:09:29.705983 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 15:09:29.705990 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 15:09:29.705997 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 15:09:29.706003 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 15:09:29.706010 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 15:09:29.746406 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 15:09:29.746419 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 15:09:29.746440 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 15:09:29.746452 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 15:09:29.746459 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 15:09:29.746491 3815 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 15:09:31.816992 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-192.ec2.internal I0327 15:09:31.817028 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-192.ec2.internal: 38.254µs I0327 15:09:31.847960 3815 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7c85498c95-7sbxq] creating logical port openshift-image-registry_image-registry-7c85498c95-7sbxq for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:31.848096 3815 kube.go:256] Updating pod openshift-image-registry/image-registry-7c85498c95-7sbxq W0327 15:09:31.852351 3815 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:09:31.871003 3815 pod.go:62] [openshift-image-registry/image-registry-7c85498c95-7sbxq] pod update took 22.943511ms I0327 15:09:31.871024 3815 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-7c85498c95-7sbxq] addLogicalPort annotation time took 22.969964ms I0327 15:09:31.872168 3815 pods.go:271] [openshift-image-registry/image-registry-7c85498c95-7sbxq] addLogicalPort took 24.222828ms, libovsdb time 876.754µs I0327 15:09:31.873168 3815 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-kkz7f] creating logical port openshift-dns_dns-default-kkz7f for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:31.873277 3815 kube.go:256] Updating pod openshift-dns/dns-default-kkz7f I0327 15:09:31.873331 3815 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-qg9vn] creating logical port openshift-ingress-canary_ingress-canary-qg9vn for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:31.873912 3815 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-qg9vn I0327 15:09:31.889657 3815 pod.go:62] [openshift-dns/dns-default-kkz7f] pod update took 16.407501ms I0327 15:09:31.889745 3815 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-kkz7f] addLogicalPort annotation time took 16.498281ms I0327 15:09:31.890872 3815 pod.go:62] [openshift-ingress-canary/ingress-canary-qg9vn] pod update took 16.985837ms I0327 15:09:31.890896 3815 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-qg9vn] addLogicalPort annotation time took 17.012753ms I0327 15:09:31.891168 3815 pods.go:271] [openshift-dns/dns-default-kkz7f] addLogicalPort took 18.01034ms, libovsdb time 1.066098ms I0327 15:09:31.892215 3815 pods.go:271] [openshift-ingress-canary/ingress-canary-qg9vn] addLogicalPort took 18.891417ms, libovsdb time 832.743µs I0327 15:09:32.926664 3815 cni.go:403] [openshift-ingress-canary/ingress-canary-qg9vn 7d601484c997df24c23f6804c00b134c309c3f986e8193484bb1a0d2ff83542e network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-qg9vn 7d601484c997df24c23f6804c00b134c309c3f986e8193484bb1a0d2ff83542e network default NAD default] I0327 15:09:32.943052 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-qg9vn, hostIfaceName: 7d601484c997df2, network: default, NAD default, SandboxID: "7d601484c997df24c23f6804c00b134c309c3f986e8193484bb1a0d2ff83542e", PCI device ID: , UID: "94870185-1aeb-4d7d-a281-1f654d1a8019", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0327 15:09:32.987570 3815 cni.go:424] [openshift-ingress-canary/ingress-canary-qg9vn 7d601484c997df24c23f6804c00b134c309c3f986e8193484bb1a0d2ff83542e network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-qg9vn 7d601484c997df24c23f6804c00b134c309c3f986e8193484bb1a0d2ff83542e network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:ca:b9:ba:84:03\",\"name\":\"7d601484c997df2\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9d30b874-21d9-4564-b303-2a505dd3727d\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:09:33.863346 3815 cni.go:403] [openshift-network-diagnostics/network-check-target-rwdfv 15cf9c24fe3a5e66e412c19518453e3722fa9259bb4dcd8b7f6557a07625787e network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-rwdfv 15cf9c24fe3a5e66e412c19518453e3722fa9259bb4dcd8b7f6557a07625787e network default NAD default] I0327 15:09:33.872868 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-rwdfv, hostIfaceName: 15cf9c24fe3a5e6, network: default, NAD default, SandboxID: "15cf9c24fe3a5e66e412c19518453e3722fa9259bb4dcd8b7f6557a07625787e", PCI device ID: , UID: "815a1a5d-a340-44ca-803d-2e2f367994ee", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0327 15:09:33.913062 3815 cni.go:424] [openshift-network-diagnostics/network-check-target-rwdfv 15cf9c24fe3a5e66e412c19518453e3722fa9259bb4dcd8b7f6557a07625787e network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-rwdfv 15cf9c24fe3a5e66e412c19518453e3722fa9259bb4dcd8b7f6557a07625787e network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:70:70:32:be:a6\",\"name\":\"15cf9c24fe3a5e6\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/96e54c46-3441-4c3d-ba3f-f38698e4cb20\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:09:35.088471 3815 cni.go:403] [kube-system/global-pull-secret-syncer-8qmwh 41a9038e0826b6623284e0f5b64bd892ecb93c92acc9159d4bbd7f3270a0b7ac network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-8qmwh 41a9038e0826b6623284e0f5b64bd892ecb93c92acc9159d4bbd7f3270a0b7ac network default NAD default] I0327 15:09:35.097812 3815 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-8qmwh, hostIfaceName: 41a9038e0826b66, network: default, NAD default, SandboxID: "41a9038e0826b6623284e0f5b64bd892ecb93c92acc9159d4bbd7f3270a0b7ac", PCI device ID: , UID: "08efbe39-f1c5-4230-88d2-6e68b0f2ec54", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0327 15:09:35.140002 3815 cni.go:424] [kube-system/global-pull-secret-syncer-8qmwh 41a9038e0826b6623284e0f5b64bd892ecb93c92acc9159d4bbd7f3270a0b7ac network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-8qmwh 41a9038e0826b6623284e0f5b64bd892ecb93c92acc9159d4bbd7f3270a0b7ac network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:5e:73:b2:8b:25\",\"name\":\"41a9038e0826b66\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a6b0fb77-c7d7-4baf-bea8-44214cc276e3\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:09:37.890980 3815 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-gvvvc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:37.891429 3815 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-gvvvc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:38.374198 3815 default_network_controller.go:1008] Node "ip-10-0-2-34.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:09:38.883925 3815 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-gvvvc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:38.883951 3815 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-gvvvc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:09:40.179927 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:09:42.096083 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-34.ec2.internal in network "default" I0327 15:09:42.096133 3815 node_tracker.go:235] Node ip-10-0-2-34.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-34.ec2.internal" I0327 15:09:42.096149 3815 node_tracker.go:169] Node ip-10-0-2-34.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:42.096155 3815 default_network_controller.go:1008] Node "ip-10-0-2-34.ec2.internal" in remote zone "ip-10-0-2-34.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:09:42.096163 3815 services_controller.go:574] Full service sync requested for network=default W0327 15:09:42.096514 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:09:42.145907 3815 default_network_controller.go:1008] Node "ip-10-0-2-34.ec2.internal" in remote zone "ip-10-0-2-34.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:09:42.226314 3815 default_network_controller.go:1008] Node "ip-10-0-2-34.ec2.internal" in remote zone "ip-10-0-2-34.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:09:42.226327 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-34.ec2.internal in network "default" I0327 15:09:42.226376 3815 node_tracker.go:169] Node ip-10-0-2-34.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:42.226390 3815 services_controller.go:574] Full service sync requested for network=default W0327 15:09:42.226892 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:09:42.227070 3815 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-34.ec2.internal for the network default I0327 15:09:42.228148 3815 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-34.ec2.internal" on network "default" took: 1.091627ms I0327 15:09:44.779485 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-34.ec2.internal in network "default" I0327 15:09:44.779504 3815 egressfirewall.go:628] Syncing node "ip-10-0-2-34.ec2.internal" for egress firewall I0327 15:09:44.779523 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-34.ec2.internal I0327 15:09:44.779536 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-34.ec2.internal: 16.663µs I0327 15:09:44.779541 3815 node_tracker.go:169] Node ip-10-0-2-34.ec2.internal switch + router changed, syncing services in network "default" I0327 15:09:44.779554 3815 services_controller.go:574] Full service sync requested for network=default I0327 15:09:44.779608 3815 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-34.ec2.internal", IP: 10.0.2.34 to PMTUD blocking rules I0327 15:09:44.779622 3815 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-34.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0327 15:09:44.779642 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:09:44.785872 3815 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-34.ec2.internal. Updating no re-route policies I0327 15:09:44.796002 3815 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-34.ec2.internal", IP: 10.0.2.34 to PMTUD blocking rules I0327 15:09:44.796017 3815 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-34.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 15:09:47.698180 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-34.ec2.internal I0327 15:09:47.698209 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-34.ec2.internal: 35.652µs I0327 15:09:57.485873 3815 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-88dqr] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-6ff64bb8b6-88dqr for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:57.485985 3815 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-88dqr I0327 15:09:57.491608 3815 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-cg89q] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-cg89q for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:57.491724 3815 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-cg89q I0327 15:09:57.504367 3815 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-88dqr] pod update took 18.402985ms I0327 15:09:57.504384 3815 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-88dqr] addLogicalPort annotation time took 18.425757ms I0327 15:09:57.505625 3815 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-88dqr] addLogicalPort took 19.769197ms, libovsdb time 944.732µs I0327 15:09:57.505669 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-pbnpf] creating logical port openshift-monitoring_cluster-monitoring-operator-6dd897695f-pbnpf for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:57.505722 3815 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-6dd897695f-pbnpf I0327 15:09:57.507349 3815 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-cg89q] pod update took 15.652603ms I0327 15:09:57.507371 3815 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-cg89q] addLogicalPort annotation time took 15.676855ms I0327 15:09:57.508482 3815 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-cg89q] addLogicalPort took 16.90722ms, libovsdb time 801.338µs I0327 15:09:57.508523 3815 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-48c66] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67b99c68bb-48c66 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:57.508579 3815 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-48c66 I0327 15:09:57.528092 3815 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-48c66] pod update took 19.527824ms I0327 15:09:57.528113 3815 base_network_controller_pods.go:937] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-48c66] addLogicalPort annotation time took 19.555491ms I0327 15:09:57.528959 3815 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-48c66] addLogicalPort took 20.440056ms, libovsdb time 585.954µs I0327 15:09:57.529190 3815 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-pbnpf] pod update took 23.48385ms I0327 15:09:57.529210 3815 base_network_controller_pods.go:937] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-pbnpf] addLogicalPort annotation time took 23.504334ms I0327 15:09:57.530030 3815 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-pbnpf] addLogicalPort took 24.365023ms, libovsdb time 569.239µs I0327 15:09:57.589180 3815 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-st4hf] creating logical port openshift-network-diagnostics_network-check-source-5c76bd6f9b-st4hf for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:57.589288 3815 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-5c76bd6f9b-st4hf I0327 15:09:57.589291 3815 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-5577457d66-m6bp9] creating logical port openshift-insights_insights-operator-5577457d66-m6bp9 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:57.589442 3815 kube.go:256] Updating pod openshift-insights/insights-operator-5577457d66-m6bp9 I0327 15:09:57.592189 3815 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-dgldk] creating logical port openshift-service-ca-operator_service-ca-operator-7b89bc695f-dgldk for pod on switch ip-10-0-0-192.ec2.internal I0327 15:09:57.592247 3815 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-7b89bc695f-dgldk I0327 15:09:57.609604 3815 pod.go:62] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-st4hf] pod update took 20.334667ms I0327 15:09:57.609624 3815 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-st4hf] addLogicalPort annotation time took 20.377864ms I0327 15:09:57.610751 3815 pods.go:271] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-st4hf] addLogicalPort took 21.587913ms, libovsdb time 858.377µs I0327 15:09:57.611266 3815 pod.go:62] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-dgldk] pod update took 19.031921ms I0327 15:09:57.611287 3815 base_network_controller_pods.go:937] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-dgldk] addLogicalPort annotation time took 19.054947ms I0327 15:09:57.612496 3815 pods.go:271] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-dgldk] addLogicalPort took 20.313951ms, libovsdb time 865.044µs I0327 15:09:57.617431 3815 pod.go:62] [openshift-insights/insights-operator-5577457d66-m6bp9] pod update took 28.007626ms I0327 15:09:57.617452 3815 base_network_controller_pods.go:937] [default/openshift-insights/insights-operator-5577457d66-m6bp9] addLogicalPort annotation time took 28.032129ms I0327 15:09:57.618265 3815 pods.go:271] [openshift-insights/insights-operator-5577457d66-m6bp9] addLogicalPort took 28.994354ms, libovsdb time 506.663µs I0327 15:09:57.932851 3815 cni.go:403] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-88dqr f7208dada1a456d6523d684e2f6c145de76487d7a427e07066077b40eb80b100 network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-88dqr f7208dada1a456d6523d684e2f6c145de76487d7a427e07066077b40eb80b100 network default NAD default] I0327 15:09:57.942717 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-6ff64bb8b6-88dqr, hostIfaceName: f7208dada1a456d, network: default, NAD default, SandboxID: "f7208dada1a456d6523d684e2f6c145de76487d7a427e07066077b40eb80b100", PCI device ID: , UID: "726b9d9d-5bcc-4428-948c-c516cb229dbb", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0327 15:09:57.985483 3815 cni.go:424] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-88dqr f7208dada1a456d6523d684e2f6c145de76487d7a427e07066077b40eb80b100 network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-88dqr f7208dada1a456d6523d684e2f6c145de76487d7a427e07066077b40eb80b100 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:af:2f:d5:22:05\",\"name\":\"f7208dada1a456d\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/009e472f-0d47-4b1c-93c2-9765dd359b3e\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:09:57.990196 3815 cni.go:403] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-st4hf cbe9f942dd4077d45cb7133f936af49d5218afb7129c17bde4b743d05c137e82 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-st4hf cbe9f942dd4077d45cb7133f936af49d5218afb7129c17bde4b743d05c137e82 network default NAD default] I0327 15:09:58.000339 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-5c76bd6f9b-st4hf, hostIfaceName: cbe9f942dd4077d, network: default, NAD default, SandboxID: "cbe9f942dd4077d45cb7133f936af49d5218afb7129c17bde4b743d05c137e82", PCI device ID: , UID: "7dad590f-2dc2-453e-9e43-44c9690be649", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0327 15:09:58.014036 3815 cni.go:403] [openshift-insights/insights-operator-5577457d66-m6bp9 461d4c39f643c5d68bdeb4eaf23d0c65e1306bbfb14ac70e52a752fce5f6ec8b network default NAD default] ADD starting CNI request [openshift-insights/insights-operator-5577457d66-m6bp9 461d4c39f643c5d68bdeb4eaf23d0c65e1306bbfb14ac70e52a752fce5f6ec8b network default NAD default] I0327 15:09:58.025332 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-5577457d66-m6bp9, hostIfaceName: 461d4c39f643c5d, network: default, NAD default, SandboxID: "461d4c39f643c5d68bdeb4eaf23d0c65e1306bbfb14ac70e52a752fce5f6ec8b", PCI device ID: , UID: "ef804ddd-5d20-4d9a-bd72-31bb6ebec36c", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0327 15:09:58.040296 3815 cni.go:403] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-dgldk 7d426d16a34912a78f633d17b5493030f69fa151875782d98665c27c45db8e5b network default NAD default] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-dgldk 7d426d16a34912a78f633d17b5493030f69fa151875782d98665c27c45db8e5b network default NAD default] I0327 15:09:58.045601 3815 cni.go:424] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-st4hf cbe9f942dd4077d45cb7133f936af49d5218afb7129c17bde4b743d05c137e82 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-st4hf cbe9f942dd4077d45cb7133f936af49d5218afb7129c17bde4b743d05c137e82 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:39:73:df:53:0c\",\"name\":\"cbe9f942dd4077d\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eece6f13-031a-489d-b56b-845bc40e7fed\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:09:58.049814 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-7b89bc695f-dgldk, hostIfaceName: 7d426d16a34912a, network: default, NAD default, SandboxID: "7d426d16a34912a78f633d17b5493030f69fa151875782d98665c27c45db8e5b", PCI device ID: , UID: "be6736f4-d540-4e46-8260-86a39582a28f", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0327 15:09:58.069144 3815 cni.go:424] [openshift-insights/insights-operator-5577457d66-m6bp9 461d4c39f643c5d68bdeb4eaf23d0c65e1306bbfb14ac70e52a752fce5f6ec8b network default NAD default] ADD finished CNI request [openshift-insights/insights-operator-5577457d66-m6bp9 461d4c39f643c5d68bdeb4eaf23d0c65e1306bbfb14ac70e52a752fce5f6ec8b network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:98:86:22:c4:a0\",\"name\":\"461d4c39f643c5d\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e40c3dad-0ca8-4c6b-bb67-9b9dd7de182d\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:09:58.089329 3815 cni.go:424] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-dgldk 7d426d16a34912a78f633d17b5493030f69fa151875782d98665c27c45db8e5b network default NAD default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-dgldk 7d426d16a34912a78f633d17b5493030f69fa151875782d98665c27c45db8e5b network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:2d:f2:34:02:c3\",\"name\":\"7d426d16a34912a\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/501e49c3-f5f0-4864-bb23-59cacb53993b\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:10:00.194389 3815 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29577070-zrrj6 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:10:00.194505 3815 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 I0327 15:10:00.210266 3815 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6] pod update took 15.779257ms I0327 15:10:00.210293 3815 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6] addLogicalPort annotation time took 15.813981ms I0327 15:10:00.211709 3815 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6] addLogicalPort took 17.336396ms, libovsdb time 1.088756ms I0327 15:10:01.829516 3815 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 1eb1dbfa6d534efa9a388a719b1b9d45097f534ac146505e2a5a8478b495bd2a network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 1eb1dbfa6d534efa9a388a719b1b9d45097f534ac146505e2a5a8478b495bd2a network default NAD default] I0327 15:10:01.839441 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29577070-zrrj6, hostIfaceName: 1eb1dbfa6d534ef, network: default, NAD default, SandboxID: "1eb1dbfa6d534efa9a388a719b1b9d45097f534ac146505e2a5a8478b495bd2a", PCI device ID: , UID: "16ca31c4-8788-4794-bc61-9b6e1275e09d", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0327 15:10:01.883884 3815 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 1eb1dbfa6d534efa9a388a719b1b9d45097f534ac146505e2a5a8478b495bd2a network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 1eb1dbfa6d534efa9a388a719b1b9d45097f534ac146505e2a5a8478b495bd2a network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:ed:d1:d2:7b:41\",\"name\":\"1eb1dbfa6d534ef\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ffa1e679-ed68-45f2-a242-d54679a51438\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:10:01.931820 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 15:10:01.955809 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 15:10:03.386729 3815 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0327 15:10:03.387718 3815 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 961.91µs I0327 15:10:03.412606 3815 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 15:10:04.307916 3815 namespace.go:98] [openshift-service-ca] adding namespace I0327 15:10:04.308867 3815 namespace.go:102] [openshift-service-ca] adding namespace took 924.96µs I0327 15:10:04.321375 3815 namespace.go:142] [openshift-service-ca] updating namespace W0327 15:10:11.288703 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:10:13.543835 3815 cni.go:403] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-48c66 7c4fa5c35b454830d61c6f3cdefd6f219ba5ee21c823b807ceeb13a6530d8f65 network default NAD default] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-48c66 7c4fa5c35b454830d61c6f3cdefd6f219ba5ee21c823b807ceeb13a6530d8f65 network default NAD default] I0327 15:10:13.552179 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67b99c68bb-48c66, hostIfaceName: 7c4fa5c35b45483, network: default, NAD default, SandboxID: "7c4fa5c35b454830d61c6f3cdefd6f219ba5ee21c823b807ceeb13a6530d8f65", PCI device ID: , UID: "f488c452-0404-478b-b6fa-8e4a67839665", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0327 15:10:13.593951 3815 cni.go:424] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-48c66 7c4fa5c35b454830d61c6f3cdefd6f219ba5ee21c823b807ceeb13a6530d8f65 network default NAD default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-48c66 7c4fa5c35b454830d61c6f3cdefd6f219ba5ee21c823b807ceeb13a6530d8f65 network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:8b:82:d9:49:d7\",\"name\":\"7c4fa5c35b45483\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a85abebb-4caa-46ef-b6e4-fc62b04a64e9\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:10:13.785516 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 15:10:14.988649 3815 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c85498c95-q75rf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:14.989342 3815 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c85498c95-q75rf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:15.003291 3815 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c85498c95-q75rf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:15.003312 3815 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c85498c95-q75rf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:15.014474 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7c85498c95-q75rf I0327 15:10:15.014503 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7c85498c95-q75rf I0327 15:10:15.044347 3815 obj_retry.go:464] Detected object openshift-image-registry/image-registry-59c85fb9b8-2rcd8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:15.044873 3815 obj_retry.go:464] Detected object openshift-image-registry/image-registry-59c85fb9b8-2rcd8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:15.057004 3815 obj_retry.go:458] Detected object openshift-image-registry/image-registry-59c85fb9b8-2rcd8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:15.057029 3815 obj_retry.go:458] Detected object openshift-image-registry/image-registry-59c85fb9b8-2rcd8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:15.064711 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-59c85fb9b8-2rcd8 I0327 15:10:15.064734 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-59c85fb9b8-2rcd8 W0327 15:10:29.705730 3815 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:10:29.705766 3815 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:29.705844 3815 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:29.705852 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default I0327 15:10:29.715838 3815 cni.go:403] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-pbnpf d52edc9a1cb13439ae24ca34d8944d2c3fd0fa74aceb517a1884ea1be4fbeae9 network default NAD default] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-pbnpf d52edc9a1cb13439ae24ca34d8944d2c3fd0fa74aceb517a1884ea1be4fbeae9 network default NAD default] I0327 15:10:29.724389 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-6dd897695f-pbnpf, hostIfaceName: d52edc9a1cb1343, network: default, NAD default, SandboxID: "d52edc9a1cb13439ae24ca34d8944d2c3fd0fa74aceb517a1884ea1be4fbeae9", PCI device ID: , UID: "99c4da71-d719-40c9-aee9-059a8031c7d3", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0327 15:10:29.732511 3815 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-cg89q edf0a78cacbae7a36f023a0e6b1d86f6e0d8ec58cee5a9d1e6785a8706e36a8a network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-cg89q edf0a78cacbae7a36f023a0e6b1d86f6e0d8ec58cee5a9d1e6785a8706e36a8a network default NAD default] I0327 15:10:29.741356 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-cg89q, hostIfaceName: edf0a78cacbae7a, network: default, NAD default, SandboxID: "edf0a78cacbae7a36f023a0e6b1d86f6e0d8ec58cee5a9d1e6785a8706e36a8a", PCI device ID: , UID: "648e5d4a-610f-4e8e-befd-430bf54e8de2", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0327 15:10:29.768263 3815 cni.go:424] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-pbnpf d52edc9a1cb13439ae24ca34d8944d2c3fd0fa74aceb517a1884ea1be4fbeae9 network default NAD default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-pbnpf d52edc9a1cb13439ae24ca34d8944d2c3fd0fa74aceb517a1884ea1be4fbeae9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:71:7f:89:06:ca\",\"name\":\"d52edc9a1cb1343\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/128c67a9-c0e6-4210-a0e1-2fb10a2e6e75\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:10:29.784032 3815 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-cg89q edf0a78cacbae7a36f023a0e6b1d86f6e0d8ec58cee5a9d1e6785a8706e36a8a network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-cg89q edf0a78cacbae7a36f023a0e6b1d86f6e0d8ec58cee5a9d1e6785a8706e36a8a network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:a5:5e:0d:c5:e1\",\"name\":\"edf0a78cacbae7a\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/09c165ea-ec33-41cb-afa6-8be68d32262d\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:10:29.793531 3815 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 15:10:29.910520 3815 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-hd2h2] creating logical port openshift-insights_insights-runtime-extractor-hd2h2 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:10:29.910667 3815 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-hd2h2 I0327 15:10:29.929430 3815 pod.go:62] [openshift-insights/insights-runtime-extractor-hd2h2] pod update took 18.795121ms I0327 15:10:29.929453 3815 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-hd2h2] addLogicalPort annotation time took 18.823534ms I0327 15:10:29.930694 3815 pods.go:271] [openshift-insights/insights-runtime-extractor-hd2h2] addLogicalPort took 20.190241ms, libovsdb time 894.885µs W0327 15:10:30.081258 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.081278 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.081286 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.084290 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 15:10:30.084312 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0327 15:10:30.115404 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 15:10:30.115419 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0327 15:10:30.122839 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.122857 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.122864 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.130275 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.130291 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.130298 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:10:30.139284 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 15:10:30.139293 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default I0327 15:10:30.268275 3815 cni.go:403] [openshift-insights/insights-runtime-extractor-hd2h2 62aea039b6c066273461870850556ac64b56f9da57da5f8e60008448776b15a8 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-hd2h2 62aea039b6c066273461870850556ac64b56f9da57da5f8e60008448776b15a8 network default NAD default] I0327 15:10:30.276452 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-hd2h2, hostIfaceName: 62aea039b6c0662, network: default, NAD default, SandboxID: "62aea039b6c066273461870850556ac64b56f9da57da5f8e60008448776b15a8", PCI device ID: , UID: "d1f0de06-a25b-4e83-85d8-71a0b9af68d7", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0327 15:10:30.317472 3815 cni.go:424] [openshift-insights/insights-runtime-extractor-hd2h2 62aea039b6c066273461870850556ac64b56f9da57da5f8e60008448776b15a8 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-hd2h2 62aea039b6c066273461870850556ac64b56f9da57da5f8e60008448776b15a8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:04:01:02:d9:82\",\"name\":\"62aea039b6c0662\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/70826061-496b-4098-8f42-fd7c27476b54\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:10:31.357048 3815 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c85498c95-7sbxq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:31.357092 3815 pods.go:176] Deleting pod: openshift-image-registry/image-registry-7c85498c95-7sbxq I0327 15:10:31.358453 3815 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-7c85498c95-7sbxq, ips: 10.129.0.6 I0327 15:10:31.358486 3815 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c85498c95-7sbxq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:31.367634 3815 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-65b68b76bc-c6rcx] creating logical port openshift-image-registry_image-registry-65b68b76bc-c6rcx for pod on switch ip-10-0-0-192.ec2.internal I0327 15:10:31.367735 3815 kube.go:256] Updating pod openshift-image-registry/image-registry-65b68b76bc-c6rcx I0327 15:10:31.369542 3815 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c85498c95-7sbxq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:31.369563 3815 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c85498c95-7sbxq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:31.380580 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7c85498c95-7sbxq I0327 15:10:31.380624 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7c85498c95-7sbxq I0327 15:10:31.385204 3815 pod.go:62] [openshift-image-registry/image-registry-65b68b76bc-c6rcx] pod update took 17.500337ms I0327 15:10:31.385225 3815 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-65b68b76bc-c6rcx] addLogicalPort annotation time took 17.524767ms I0327 15:10:31.386385 3815 pods.go:271] [openshift-image-registry/image-registry-65b68b76bc-c6rcx] addLogicalPort took 18.769371ms, libovsdb time 876.929µs I0327 15:10:31.731334 3815 cni.go:403] [openshift-image-registry/image-registry-65b68b76bc-c6rcx f7b1466d346649e43e6616141728a2372300818fe5987be7dd8106bc89e2b6d9 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-65b68b76bc-c6rcx f7b1466d346649e43e6616141728a2372300818fe5987be7dd8106bc89e2b6d9 network default NAD default] I0327 15:10:31.740098 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-65b68b76bc-c6rcx, hostIfaceName: f7b1466d346649e, network: default, NAD default, SandboxID: "f7b1466d346649e43e6616141728a2372300818fe5987be7dd8106bc89e2b6d9", PCI device ID: , UID: "410a3d16-a54b-4338-953b-8ea2e681f1a7", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0327 15:10:31.784231 3815 cni.go:424] [openshift-image-registry/image-registry-65b68b76bc-c6rcx f7b1466d346649e43e6616141728a2372300818fe5987be7dd8106bc89e2b6d9 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-65b68b76bc-c6rcx f7b1466d346649e43e6616141728a2372300818fe5987be7dd8106bc89e2b6d9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:18:c9:62:80:49\",\"name\":\"f7b1466d346649e\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e1e95ca4-62af-48cb-8762-9a71ba7e49c0\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:10:32.051125 3815 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:32.051146 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:32.051151 3815 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:32.348440 3815 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 15:10:33.059279 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xqg2p] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-xqg2p for pod on switch ip-10-0-0-192.ec2.internal I0327 15:10:33.059400 3815 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xqg2p I0327 15:10:33.074948 3815 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xqg2p] pod update took 15.573285ms I0327 15:10:33.074971 3815 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xqg2p] addLogicalPort annotation time took 15.601374ms I0327 15:10:33.076273 3815 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xqg2p] addLogicalPort took 17.013247ms, libovsdb time 998.056µs I0327 15:10:34.067203 3815 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xqg2p 7f9e5f4e60571e89c1f300796b84ace8c88bb5a874685e5d70dd940a3b2f366d network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xqg2p 7f9e5f4e60571e89c1f300796b84ace8c88bb5a874685e5d70dd940a3b2f366d network default NAD default] I0327 15:10:34.075456 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-xqg2p, hostIfaceName: 7f9e5f4e60571e8, network: default, NAD default, SandboxID: "7f9e5f4e60571e89c1f300796b84ace8c88bb5a874685e5d70dd940a3b2f366d", PCI device ID: , UID: "4cc49bdf-f0d0-4721-b274-ff134af7d682", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0327 15:10:34.117437 3815 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xqg2p 7f9e5f4e60571e89c1f300796b84ace8c88bb5a874685e5d70dd940a3b2f366d network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xqg2p 7f9e5f4e60571e89c1f300796b84ace8c88bb5a874685e5d70dd940a3b2f366d network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:8f:01:c2:15:19\",\"name\":\"7f9e5f4e60571e8\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d317ec4f-b4c6-489a-82b9-0e8dadcd51f2\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:10:35.841974 3815 cni.go:403] [openshift-dns/dns-default-kkz7f 03ea98e4d61dc1549aa50ce171a8771b91c5aea70bc1f983181c30eaa6e94a60 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-kkz7f 03ea98e4d61dc1549aa50ce171a8771b91c5aea70bc1f983181c30eaa6e94a60 network default NAD default] I0327 15:10:35.851294 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-kkz7f, hostIfaceName: 03ea98e4d61dc15, network: default, NAD default, SandboxID: "03ea98e4d61dc1549aa50ce171a8771b91c5aea70bc1f983181c30eaa6e94a60", PCI device ID: , UID: "6db12263-1328-4912-bcb2-1aa0dcf138fd", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0327 15:10:35.893310 3815 cni.go:424] [openshift-dns/dns-default-kkz7f 03ea98e4d61dc1549aa50ce171a8771b91c5aea70bc1f983181c30eaa6e94a60 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-kkz7f 03ea98e4d61dc1549aa50ce171a8771b91c5aea70bc1f983181c30eaa6e94a60 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:dd:2f:b8:46:f2\",\"name\":\"03ea98e4d61dc15\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1a020b46-b1f2-4163-83e8-174674e1dd12\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:10:36.114614 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0327 15:10:37.049411 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:10:37.049437 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0327 15:10:38.136918 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-c56ff685-zpjmn] creating logical port openshift-monitoring_prometheus-operator-c56ff685-zpjmn for pod on switch ip-10-0-0-192.ec2.internal I0327 15:10:38.137015 3815 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-c56ff685-zpjmn I0327 15:10:38.154840 3815 pod.go:62] [openshift-monitoring/prometheus-operator-c56ff685-zpjmn] pod update took 17.851837ms I0327 15:10:38.154862 3815 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-c56ff685-zpjmn] addLogicalPort annotation time took 17.881008ms I0327 15:10:38.155971 3815 pods.go:271] [openshift-monitoring/prometheus-operator-c56ff685-zpjmn] addLogicalPort took 19.0731ms, libovsdb time 805.826µs I0327 15:10:38.493827 3815 cni.go:403] [openshift-monitoring/prometheus-operator-c56ff685-zpjmn 64196727e79db981f3ace29e95d3172584cb182d3541e1e34554fd711f4f1d54 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-c56ff685-zpjmn 64196727e79db981f3ace29e95d3172584cb182d3541e1e34554fd711f4f1d54 network default NAD default] I0327 15:10:38.511399 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-c56ff685-zpjmn, hostIfaceName: 64196727e79db98, network: default, NAD default, SandboxID: "64196727e79db981f3ace29e95d3172584cb182d3541e1e34554fd711f4f1d54", PCI device ID: , UID: "fab3f69f-0bf2-46e1-96f1-4d995025e5ac", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0327 15:10:38.555176 3815 cni.go:424] [openshift-monitoring/prometheus-operator-c56ff685-zpjmn 64196727e79db981f3ace29e95d3172584cb182d3541e1e34554fd711f4f1d54 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-c56ff685-zpjmn 64196727e79db981f3ace29e95d3172584cb182d3541e1e34554fd711f4f1d54 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:54:6f:8f:26:c5\",\"name\":\"64196727e79db98\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e8bad6db-ceea-4cf4-8ac3-d4dfe9c20dae\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:10:42.066853 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:42.066879 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:42.066885 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:42.066897 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:42.066901 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:42.066906 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default I0327 15:10:43.777176 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:10:43.777348 3815 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0327 15:10:43.805934 3815 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 28.653126ms I0327 15:10:43.805959 3815 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 28.685984ms I0327 15:10:43.807226 3815 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 30.066183ms, libovsdb time 951.6µs I0327 15:10:44.140393 3815 cni.go:403] [openshift-monitoring/alertmanager-main-0 098198dda32316e312b5c2751c40e5d945c99221e2f2b937438c263f857273aa network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 098198dda32316e312b5c2751c40e5d945c99221e2f2b937438c263f857273aa network default NAD default] I0327 15:10:44.148801 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 098198dda32316e, network: default, NAD default, SandboxID: "098198dda32316e312b5c2751c40e5d945c99221e2f2b937438c263f857273aa", PCI device ID: , UID: "63ccc52a-8543-420d-98da-75b665e82a96", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0327 15:10:44.194043 3815 cni.go:424] [openshift-monitoring/alertmanager-main-0 098198dda32316e312b5c2751c40e5d945c99221e2f2b937438c263f857273aa network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 098198dda32316e312b5c2751c40e5d945c99221e2f2b937438c263f857273aa network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:93:c9:b6:30:74\",\"name\":\"098198dda32316e\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/72ce4dba-504c-4cc3-9329-4de709dfc7b6\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:10:46.454657 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:10:46.454678 3815 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:10:46.454683 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 15:10:47.403425 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-84d869bc95-cx5c9] creating logical port openshift-monitoring_metrics-server-84d869bc95-cx5c9 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:10:47.403540 3815 kube.go:256] Updating pod openshift-monitoring/metrics-server-84d869bc95-cx5c9 I0327 15:10:47.423394 3815 pod.go:62] [openshift-monitoring/metrics-server-84d869bc95-cx5c9] pod update took 19.873529ms I0327 15:10:47.423421 3815 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-84d869bc95-cx5c9] addLogicalPort annotation time took 19.908328ms I0327 15:10:47.424570 3815 pods.go:271] [openshift-monitoring/metrics-server-84d869bc95-cx5c9] addLogicalPort took 21.163931ms, libovsdb time 830.506µs I0327 15:10:47.759089 3815 cni.go:403] [openshift-monitoring/metrics-server-84d869bc95-cx5c9 f116d6ec689f471fbb9855464c335123fbd39d109b77ad918e6a015ea3680436 network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-84d869bc95-cx5c9 f116d6ec689f471fbb9855464c335123fbd39d109b77ad918e6a015ea3680436 network default NAD default] I0327 15:10:47.767346 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-84d869bc95-cx5c9, hostIfaceName: f116d6ec689f471, network: default, NAD default, SandboxID: "f116d6ec689f471fbb9855464c335123fbd39d109b77ad918e6a015ea3680436", PCI device ID: , UID: "09f78386-3725-42b7-894b-217f7b9f8012", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0327 15:10:47.813204 3815 cni.go:424] [openshift-monitoring/metrics-server-84d869bc95-cx5c9 f116d6ec689f471fbb9855464c335123fbd39d109b77ad918e6a015ea3680436 network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-84d869bc95-cx5c9 f116d6ec689f471fbb9855464c335123fbd39d109b77ad918e6a015ea3680436 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:a4:b3:6a:ca:56\",\"name\":\"f116d6ec689f471\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/58ce228f-dac0-4593-8e6a-520f8d1b52b0\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:10:47.935123 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-6d4c9f54f6-7bhrf] creating logical port openshift-monitoring_telemeter-client-6d4c9f54f6-7bhrf for pod on switch ip-10-0-0-192.ec2.internal I0327 15:10:47.935243 3815 kube.go:256] Updating pod openshift-monitoring/telemeter-client-6d4c9f54f6-7bhrf I0327 15:10:47.959120 3815 pod.go:62] [openshift-monitoring/telemeter-client-6d4c9f54f6-7bhrf] pod update took 23.916225ms I0327 15:10:47.959145 3815 base_network_controller_pods.go:937] [default/openshift-monitoring/telemeter-client-6d4c9f54f6-7bhrf] addLogicalPort annotation time took 23.947523ms I0327 15:10:47.960938 3815 pods.go:271] [openshift-monitoring/telemeter-client-6d4c9f54f6-7bhrf] addLogicalPort took 25.836585ms, libovsdb time 1.174456ms W0327 15:10:48.233672 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:10:48.233692 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:10:48.233699 3815 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 15:10:48.298377 3815 cni.go:403] [openshift-monitoring/telemeter-client-6d4c9f54f6-7bhrf 0d8006919022a2709432f179c769ddb94073aef246c3d3b020eff93aefb82081 network default NAD default] ADD starting CNI request [openshift-monitoring/telemeter-client-6d4c9f54f6-7bhrf 0d8006919022a2709432f179c769ddb94073aef246c3d3b020eff93aefb82081 network default NAD default] I0327 15:10:48.307050 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-6d4c9f54f6-7bhrf, hostIfaceName: 0d8006919022a27, network: default, NAD default, SandboxID: "0d8006919022a2709432f179c769ddb94073aef246c3d3b020eff93aefb82081", PCI device ID: , UID: "8c7f143f-fa7f-4ce4-ae63-20b0d7ad61a9", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0327 15:10:48.353354 3815 cni.go:424] [openshift-monitoring/telemeter-client-6d4c9f54f6-7bhrf 0d8006919022a2709432f179c769ddb94073aef246c3d3b020eff93aefb82081 network default NAD default] ADD finished CNI request [openshift-monitoring/telemeter-client-6d4c9f54f6-7bhrf 0d8006919022a2709432f179c769ddb94073aef246c3d3b020eff93aefb82081 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:02:b8:5f:6d:b5\",\"name\":\"0d8006919022a27\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/39347786-457b-44f4-bd0a-0b17ca43fd65\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:10:49.424696 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:49.424724 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:49.424733 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:10:50.496455 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:10:51.145009 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:10:51.204364 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:10:51.204386 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:10:51.234327 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:51.234348 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:51.234354 3815 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:51.234360 3815 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:51.289255 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:10:51.289277 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:10:52.161679 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:52.161700 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:52.161705 3815 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:52.161709 3815 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:10:53.363540 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0327 15:10:53.758418 3815 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-xpkhm] creating logical port openshift-console_downloads-56d45c498b-xpkhm for pod on switch ip-10-0-0-192.ec2.internal I0327 15:10:53.758518 3815 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-xpkhm I0327 15:10:53.778051 3815 pod.go:62] [openshift-console/downloads-56d45c498b-xpkhm] pod update took 19.557209ms I0327 15:10:53.778073 3815 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-xpkhm] addLogicalPort annotation time took 19.583612ms I0327 15:10:53.779247 3815 pods.go:271] [openshift-console/downloads-56d45c498b-xpkhm] addLogicalPort took 20.8453ms, libovsdb time 769.722µs I0327 15:10:54.116361 3815 cni.go:403] [openshift-console/downloads-56d45c498b-xpkhm 648f1c277433a653b935e55bba35ff8d0892b1fd5672add054b56d725fd05f1e network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-xpkhm 648f1c277433a653b935e55bba35ff8d0892b1fd5672add054b56d725fd05f1e network default NAD default] I0327 15:10:54.124741 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-xpkhm, hostIfaceName: 648f1c277433a65, network: default, NAD default, SandboxID: "648f1c277433a653b935e55bba35ff8d0892b1fd5672add054b56d725fd05f1e", PCI device ID: , UID: "f410825e-3a99-4a13-a67b-643b5e57cbb3", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0327 15:10:54.168085 3815 cni.go:424] [openshift-console/downloads-56d45c498b-xpkhm 648f1c277433a653b935e55bba35ff8d0892b1fd5672add054b56d725fd05f1e network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-xpkhm 648f1c277433a653b935e55bba35ff8d0892b1fd5672add054b56d725fd05f1e network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:09:91:90:1e:b9\",\"name\":\"648f1c277433a65\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/54a49d25-b716-4e7e-969f-1f3ec2006ad0\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:10:55.056991 3815 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7f4b95875b-9lvtg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:55.057558 3815 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7f4b95875b-9lvtg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:55.188151 3815 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7f4b95875b-9lvtg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:55.188174 3815 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7f4b95875b-9lvtg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:55.198107 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7f4b95875b-9lvtg I0327 15:10:55.198132 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7f4b95875b-9lvtg I0327 15:10:58.369854 3815 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:58.370425 3815 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:58.370464 3815 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:10:58.370473 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0327 15:10:58.370484 3815 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:10:58.370489 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0327 15:10:59.396040 3815 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:59.396064 3815 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:59.396082 3815 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:59.396090 3815 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390nwpxn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:02.882027 3815 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 15:11:02.890925 3815 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 15:11:04.212966 3815 base_network_controller_pods.go:487] [default/openshift-console/console-5dbf498488-b8htm] creating logical port openshift-console_console-5dbf498488-b8htm for pod on switch ip-10-0-0-192.ec2.internal I0327 15:11:04.213093 3815 kube.go:256] Updating pod openshift-console/console-5dbf498488-b8htm I0327 15:11:04.227344 3815 pod.go:62] [openshift-console/console-5dbf498488-b8htm] pod update took 14.277485ms I0327 15:11:04.227371 3815 base_network_controller_pods.go:937] [default/openshift-console/console-5dbf498488-b8htm] addLogicalPort annotation time took 14.310095ms I0327 15:11:04.228768 3815 pods.go:271] [openshift-console/console-5dbf498488-b8htm] addLogicalPort took 15.817913ms, libovsdb time 938.415µs I0327 15:11:08.592019 3815 cni.go:403] [openshift-console/console-5dbf498488-b8htm 3dd5887c683b864d13af7df723c12fb4bc39cd5986aa5596646a6674d6f2e854 network default NAD default] ADD starting CNI request [openshift-console/console-5dbf498488-b8htm 3dd5887c683b864d13af7df723c12fb4bc39cd5986aa5596646a6674d6f2e854 network default NAD default] I0327 15:11:08.602304 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5dbf498488-b8htm, hostIfaceName: 3dd5887c683b864, network: default, NAD default, SandboxID: "3dd5887c683b864d13af7df723c12fb4bc39cd5986aa5596646a6674d6f2e854", PCI device ID: , UID: "c90a0a47-8181-466d-baf7-277bf018de46", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0327 15:11:08.651707 3815 cni.go:424] [openshift-console/console-5dbf498488-b8htm 3dd5887c683b864d13af7df723c12fb4bc39cd5986aa5596646a6674d6f2e854 network default NAD default] ADD finished CNI request [openshift-console/console-5dbf498488-b8htm 3dd5887c683b864d13af7df723c12fb4bc39cd5986aa5596646a6674d6f2e854 network default NAD default], result "{\"interfaces\":[{\"mac\":\"be:d9:93:44:9d:be\",\"name\":\"3dd5887c683b864\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4f3da06f-6d1d-4e74-b116-36f75f829a25\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:11:09.252551 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0327 15:11:09.530004 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0327 15:11:09.554207 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads I0327 15:11:10.175970 3815 cni.go:403] [openshift-multus/network-metrics-daemon-dr7nn 21ab0ffacea2b65477fb5d2a0574d040963bbefa110688bbb41a83fe3c02d1ea network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-dr7nn 21ab0ffacea2b65477fb5d2a0574d040963bbefa110688bbb41a83fe3c02d1ea network default NAD default] I0327 15:11:10.187248 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-dr7nn, hostIfaceName: 21ab0ffacea2b65, network: default, NAD default, SandboxID: "21ab0ffacea2b65477fb5d2a0574d040963bbefa110688bbb41a83fe3c02d1ea", PCI device ID: , UID: "bf9112e3-b299-480c-a330-d7ca17e331e4", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0327 15:11:10.240943 3815 cni.go:424] [openshift-multus/network-metrics-daemon-dr7nn 21ab0ffacea2b65477fb5d2a0574d040963bbefa110688bbb41a83fe3c02d1ea network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-dr7nn 21ab0ffacea2b65477fb5d2a0574d040963bbefa110688bbb41a83fe3c02d1ea network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:fe:e2:b7:6a:9f\",\"name\":\"21ab0ffacea2b65\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dda88d70-69f4-4a96-b5db-3770e8dc2cf4\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:11:11.461622 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:11:12.262596 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 15:11:12.551480 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 15:11:12.692907 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 15:11:13.556933 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0327 15:11:13.556950 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0327 15:11:13.557000 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0327 15:11:14.194682 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:11:14.194712 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:11:14.194720 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:11:14.591925 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0327 15:11:14.619596 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 15:11:18.686846 3815 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7f4b95875b-7mwt8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:18.687457 3815 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7f4b95875b-7mwt8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:19.334919 3815 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7f4b95875b-7mwt8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:19.334945 3815 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7f4b95875b-7mwt8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:19.337868 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7f4b95875b-7mwt8 I0327 15:11:19.337893 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7f4b95875b-7mwt8 W0327 15:11:23.605114 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 15:11:23.605117 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 15:11:23.605164 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 15:11:23.635327 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 15:11:23.635341 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 15:11:23.635370 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 15:11:24.613450 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 15:11:24.637392 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0327 15:11:39.713753 3815 cni.go:403] [openshift-console/console-5dbf498488-b8htm 3dd5887c683b864d13af7df723c12fb4bc39cd5986aa5596646a6674d6f2e854 network default NAD default] DEL starting CNI request [openshift-console/console-5dbf498488-b8htm 3dd5887c683b864d13af7df723c12fb4bc39cd5986aa5596646a6674d6f2e854 network default NAD default] I0327 15:11:39.788129 3815 cni.go:424] [openshift-console/console-5dbf498488-b8htm 3dd5887c683b864d13af7df723c12fb4bc39cd5986aa5596646a6674d6f2e854 network default NAD default] DEL finished CNI request [openshift-console/console-5dbf498488-b8htm 3dd5887c683b864d13af7df723c12fb4bc39cd5986aa5596646a6674d6f2e854 network default NAD default], result "{}", err I0327 15:11:39.843659 3815 obj_retry.go:464] Detected object openshift-console/console-5dbf498488-b8htm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:39.843701 3815 pods.go:176] Deleting pod: openshift-console/console-5dbf498488-b8htm I0327 15:11:39.845174 3815 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5dbf498488-b8htm, ips: 10.129.0.25 I0327 15:11:39.845203 3815 obj_retry.go:464] Detected object openshift-console/console-5dbf498488-b8htm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:39.853673 3815 base_network_controller_pods.go:487] [default/openshift-console/console-7d8bc75c77-c54lq] creating logical port openshift-console_console-7d8bc75c77-c54lq for pod on switch ip-10-0-0-192.ec2.internal I0327 15:11:39.853780 3815 kube.go:256] Updating pod openshift-console/console-7d8bc75c77-c54lq I0327 15:11:39.866011 3815 pod.go:62] [openshift-console/console-7d8bc75c77-c54lq] pod update took 12.266062ms I0327 15:11:39.866039 3815 base_network_controller_pods.go:937] [default/openshift-console/console-7d8bc75c77-c54lq] addLogicalPort annotation time took 12.297405ms I0327 15:11:39.867402 3815 pods.go:271] [openshift-console/console-7d8bc75c77-c54lq] addLogicalPort took 13.74371ms, libovsdb time 910.465µs I0327 15:11:40.243224 3815 cni.go:403] [openshift-console/console-7d8bc75c77-c54lq 188bf4491a187cd83a591ac4cc11e829cd938b36c5857b48b3cec1d587188ee4 network default NAD default] ADD starting CNI request [openshift-console/console-7d8bc75c77-c54lq 188bf4491a187cd83a591ac4cc11e829cd938b36c5857b48b3cec1d587188ee4 network default NAD default] I0327 15:11:40.252008 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-7d8bc75c77-c54lq, hostIfaceName: 188bf4491a187cd, network: default, NAD default, SandboxID: "188bf4491a187cd83a591ac4cc11e829cd938b36c5857b48b3cec1d587188ee4", PCI device ID: , UID: "7baa335d-d8fd-40d1-ae66-a8471add05b0", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0327 15:11:40.295778 3815 cni.go:424] [openshift-console/console-7d8bc75c77-c54lq 188bf4491a187cd83a591ac4cc11e829cd938b36c5857b48b3cec1d587188ee4 network default NAD default] ADD finished CNI request [openshift-console/console-7d8bc75c77-c54lq 188bf4491a187cd83a591ac4cc11e829cd938b36c5857b48b3cec1d587188ee4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:46:8c:64:b7:e0\",\"name\":\"188bf4491a187cd\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8bead8a7-38b4-45d7-a9eb-ad3619ff313e\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:11:40.697565 3815 obj_retry.go:458] Detected object openshift-console/console-5dbf498488-b8htm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:40.697605 3815 obj_retry.go:458] Detected object openshift-console/console-5dbf498488-b8htm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:40.704813 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5dbf498488-b8htm I0327 15:11:40.704844 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5dbf498488-b8htm I0327 15:11:47.412107 3815 obj_retry.go:464] Detected object openshift-ingress/router-default-859d5bccd5-cr7vc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:47.412684 3815 obj_retry.go:464] Detected object openshift-ingress/router-default-859d5bccd5-cr7vc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:48.408959 3815 obj_retry.go:458] Detected object openshift-ingress/router-default-859d5bccd5-cr7vc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:48.408989 3815 obj_retry.go:458] Detected object openshift-ingress/router-default-859d5bccd5-cr7vc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:48.413986 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-859d5bccd5-cr7vc I0327 15:11:48.414013 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-859d5bccd5-cr7vc I0327 15:11:48.740000 3815 obj_retry.go:464] Detected object openshift-console/console-5dbf498488-265nt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:48.740597 3815 obj_retry.go:464] Detected object openshift-console/console-5dbf498488-265nt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:49.414879 3815 obj_retry.go:458] Detected object openshift-console/console-5dbf498488-265nt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:49.414905 3815 obj_retry.go:458] Detected object openshift-console/console-5dbf498488-265nt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:49.421812 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5dbf498488-265nt I0327 15:11:49.421841 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5dbf498488-265nt W0327 15:11:50.744856 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 15:12:04.465959 3815 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:04.466507 3815 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:04.536475 3815 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:04.536500 3815 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:04.543062 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0327 15:12:04.543088 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0327 15:12:08.751301 3815 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:08.751838 3815 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:09.573358 3815 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:09.573382 3815 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:09.580617 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0327 15:12:09.580646 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0327 15:12:30.053626 3815 cni.go:403] [openshift-console/console-7d8bc75c77-c54lq 188bf4491a187cd83a591ac4cc11e829cd938b36c5857b48b3cec1d587188ee4 network default NAD default] DEL starting CNI request [openshift-console/console-7d8bc75c77-c54lq 188bf4491a187cd83a591ac4cc11e829cd938b36c5857b48b3cec1d587188ee4 network default NAD default] I0327 15:12:30.126417 3815 cni.go:424] [openshift-console/console-7d8bc75c77-c54lq 188bf4491a187cd83a591ac4cc11e829cd938b36c5857b48b3cec1d587188ee4 network default NAD default] DEL finished CNI request [openshift-console/console-7d8bc75c77-c54lq 188bf4491a187cd83a591ac4cc11e829cd938b36c5857b48b3cec1d587188ee4 network default NAD default], result "{}", err I0327 15:12:30.179677 3815 obj_retry.go:464] Detected object openshift-console/console-7d8bc75c77-c54lq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:30.179713 3815 pods.go:176] Deleting pod: openshift-console/console-7d8bc75c77-c54lq I0327 15:12:30.181221 3815 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7d8bc75c77-c54lq, ips: 10.129.0.26 I0327 15:12:30.181316 3815 obj_retry.go:464] Detected object openshift-console/console-7d8bc75c77-c54lq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:30.189309 3815 base_network_controller_pods.go:487] [default/openshift-console/console-d7b8685d9-kjpxv] creating logical port openshift-console_console-d7b8685d9-kjpxv for pod on switch ip-10-0-0-192.ec2.internal I0327 15:12:30.189437 3815 kube.go:256] Updating pod openshift-console/console-d7b8685d9-kjpxv I0327 15:12:30.204167 3815 pod.go:62] [openshift-console/console-d7b8685d9-kjpxv] pod update took 14.768103ms I0327 15:12:30.204193 3815 base_network_controller_pods.go:937] [default/openshift-console/console-d7b8685d9-kjpxv] addLogicalPort annotation time took 14.802121ms I0327 15:12:30.205418 3815 pods.go:271] [openshift-console/console-d7b8685d9-kjpxv] addLogicalPort took 16.129549ms, libovsdb time 846.279µs I0327 15:12:30.549276 3815 cni.go:403] [openshift-console/console-d7b8685d9-kjpxv 727c4e2a09e41a0b7e306982800c11ec607e0b82d773b6e83810fbd106584136 network default NAD default] ADD starting CNI request [openshift-console/console-d7b8685d9-kjpxv 727c4e2a09e41a0b7e306982800c11ec607e0b82d773b6e83810fbd106584136 network default NAD default] I0327 15:12:30.558308 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-d7b8685d9-kjpxv, hostIfaceName: 727c4e2a09e41a0, network: default, NAD default, SandboxID: "727c4e2a09e41a0b7e306982800c11ec607e0b82d773b6e83810fbd106584136", PCI device ID: , UID: "bf71986f-8de3-4808-9ddf-cb1cb7ef5a14", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0327 15:12:30.604998 3815 cni.go:424] [openshift-console/console-d7b8685d9-kjpxv 727c4e2a09e41a0b7e306982800c11ec607e0b82d773b6e83810fbd106584136 network default NAD default] ADD finished CNI request [openshift-console/console-d7b8685d9-kjpxv 727c4e2a09e41a0b7e306982800c11ec607e0b82d773b6e83810fbd106584136 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:c4:81:8f:d7:1a\",\"name\":\"727c4e2a09e41a0\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4aa3d0ee-aaa7-47e7-88fe-9b60b479e66a\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:12:30.913831 3815 obj_retry.go:458] Detected object openshift-console/console-7d8bc75c77-c54lq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:30.913855 3815 obj_retry.go:458] Detected object openshift-console/console-7d8bc75c77-c54lq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:30.917014 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d8bc75c77-c54lq I0327 15:12:30.917040 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d8bc75c77-c54lq I0327 15:12:36.145750 3815 cni.go:403] [openshift-monitoring/alertmanager-main-0 098198dda32316e312b5c2751c40e5d945c99221e2f2b937438c263f857273aa network default NAD default] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 098198dda32316e312b5c2751c40e5d945c99221e2f2b937438c263f857273aa network default NAD default] I0327 15:12:36.223329 3815 cni.go:424] [openshift-monitoring/alertmanager-main-0 098198dda32316e312b5c2751c40e5d945c99221e2f2b937438c263f857273aa network default NAD default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 098198dda32316e312b5c2751c40e5d945c99221e2f2b937438c263f857273aa network default NAD default], result "{}", err I0327 15:12:36.281981 3815 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:36.282024 3815 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0327 15:12:36.283451 3815 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.129.0.21 I0327 15:12:36.283480 3815 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:36.939255 3815 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:36.939283 3815 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:36.946117 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0327 15:12:36.946143 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0327 15:12:36.982145 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:12:36.982294 3815 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0327 15:12:37.005718 3815 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 23.46629ms I0327 15:12:37.005742 3815 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 23.495623ms I0327 15:12:37.007026 3815 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 24.899163ms, libovsdb time 900.756µs I0327 15:12:37.341969 3815 cni.go:403] [openshift-monitoring/alertmanager-main-0 a9d4ea86b51f87385b11b4f6fead5d85aafa3b2c6cb519aadfe358c6e9f0be74 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 a9d4ea86b51f87385b11b4f6fead5d85aafa3b2c6cb519aadfe358c6e9f0be74 network default NAD default] I0327 15:12:37.350877 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: a9d4ea86b51f873, network: default, NAD default, SandboxID: "a9d4ea86b51f87385b11b4f6fead5d85aafa3b2c6cb519aadfe358c6e9f0be74", PCI device ID: , UID: "7a165d80-885a-4167-a18c-a331f4c78024", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0327 15:12:37.397327 3815 cni.go:424] [openshift-monitoring/alertmanager-main-0 a9d4ea86b51f87385b11b4f6fead5d85aafa3b2c6cb519aadfe358c6e9f0be74 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 a9d4ea86b51f87385b11b4f6fead5d85aafa3b2c6cb519aadfe358c6e9f0be74 network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:28:69:ac:18:4d\",\"name\":\"a9d4ea86b51f873\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2cbcebaf-5265-43b9-83fc-24381db36da3\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:12:40.863273 3815 obj_retry.go:464] Detected object openshift-console/console-7d8bc75c77-8mglp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:12:40.863889 3815 obj_retry.go:464] Detected object openshift-console/console-7d8bc75c77-8mglp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 15:12:40.971974 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 15:12:41.709710 3815 obj_retry.go:458] Detected object openshift-console/console-7d8bc75c77-8mglp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:41.709733 3815 obj_retry.go:458] Detected object openshift-console/console-7d8bc75c77-8mglp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:12:41.712132 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d8bc75c77-8mglp I0327 15:12:41.712157 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d8bc75c77-8mglp W0327 15:12:57.393016 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:12:57.393042 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:12:57.393048 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 15:13:12.177785 3815 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:13:12.178342 3815 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:13:12.749487 3815 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:13:12.749515 3815 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:13:12.756052 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0327 15:13:12.756074 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0327 15:13:59.211138 3815 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 15:14:52.200410 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=56 I0327 15:15:04.308985 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0327 15:15:48.303541 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0327 15:15:49.287616 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0327 15:15:54.751165 3815 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 1eb1dbfa6d534efa9a388a719b1b9d45097f534ac146505e2a5a8478b495bd2a network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 1eb1dbfa6d534efa9a388a719b1b9d45097f534ac146505e2a5a8478b495bd2a network default NAD default] I0327 15:15:54.833379 3815 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 1eb1dbfa6d534efa9a388a719b1b9d45097f534ac146505e2a5a8478b495bd2a network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 1eb1dbfa6d534efa9a388a719b1b9d45097f534ac146505e2a5a8478b495bd2a network default NAD default], result "{}", err I0327 15:15:54.875982 3815 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:15:54.876014 3815 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 I0327 15:15:54.877352 3815 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6, ips: 10.129.0.16 I0327 15:15:54.877378 3815 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:15:55.901331 3815 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:15:55.901364 3815 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:04.323605 3815 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-67c57] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-67c57 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:16:04.323755 3815 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-67c57 I0327 15:16:04.340764 3815 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-67c57] pod update took 17.03818ms I0327 15:16:04.340794 3815 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-67c57] addLogicalPort annotation time took 17.074394ms I0327 15:16:04.341977 3815 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-67c57] addLogicalPort took 18.407065ms, libovsdb time 855.574µs I0327 15:16:04.680355 3815 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-67c57 7d74c433c4ee9b3088cc354a22fad1b4b7f42a65699d9f82dd30d6891213ae55 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-67c57 7d74c433c4ee9b3088cc354a22fad1b4b7f42a65699d9f82dd30d6891213ae55 network default NAD default] I0327 15:16:04.688476 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-67c57, hostIfaceName: 7d74c433c4ee9b3, network: default, NAD default, SandboxID: "7d74c433c4ee9b3088cc354a22fad1b4b7f42a65699d9f82dd30d6891213ae55", PCI device ID: , UID: "363e614c-2a7c-4b42-b660-0d4cc28e24b7", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0327 15:16:04.732452 3815 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-67c57 7d74c433c4ee9b3088cc354a22fad1b4b7f42a65699d9f82dd30d6891213ae55 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-67c57 7d74c433c4ee9b3088cc354a22fad1b4b7f42a65699d9f82dd30d6891213ae55 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:e0:87:54:70:c4\",\"name\":\"7d74c433c4ee9b3\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7745762f-8cd0-4433-b951-fcf1279a9a20\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:16:04.877854 3815 namespace.go:98] [tsf] adding namespace I0327 15:16:04.878774 3815 namespace.go:102] [tsf] adding namespace took 900.194µs I0327 15:16:04.894023 3815 namespace.go:142] [tsf] updating namespace I0327 15:16:04.903571 3815 namespace.go:142] [tsf] updating namespace I0327 15:16:07.202738 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=17 W0327 15:16:07.804687 3815 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:16:08.847760 3815 namespace.go:98] [konflux-ui] adding namespace I0327 15:16:08.848656 3815 namespace.go:102] [konflux-ui] adding namespace took 872.126µs I0327 15:16:08.852470 3815 namespace.go:98] [tsf-keycloak] adding namespace I0327 15:16:08.852499 3815 namespace.go:98] [tsf-tas] adding namespace I0327 15:16:08.853245 3815 namespace.go:102] [tsf-keycloak] adding namespace took 758.003µs I0327 15:16:08.853822 3815 namespace.go:102] [tsf-tas] adding namespace took 1.308976ms I0327 15:16:08.856469 3815 namespace.go:98] [konflux-operator] adding namespace I0327 15:16:08.856609 3815 namespace.go:98] [tsf-tpa] adding namespace I0327 15:16:08.856768 3815 namespace.go:98] [rhtpa-operator] adding namespace I0327 15:16:08.856815 3815 namespace.go:98] [rhbk-operator] adding namespace I0327 15:16:08.857141 3815 namespace.go:102] [konflux-operator] adding namespace took 657.125µs I0327 15:16:08.857163 3815 namespace.go:98] [cert-manager-operator] adding namespace I0327 15:16:08.857665 3815 namespace.go:102] [cert-manager-operator] adding namespace took 493.838µs I0327 15:16:08.858239 3815 namespace.go:102] [tsf-tpa] adding namespace took 1.615051ms I0327 15:16:08.858856 3815 namespace.go:102] [rhtpa-operator] adding namespace took 2.068904ms I0327 15:16:08.859417 3815 namespace.go:102] [rhbk-operator] adding namespace took 2.589843ms I0327 15:16:08.865314 3815 namespace.go:142] [konflux-ui] updating namespace I0327 15:16:08.882297 3815 namespace.go:142] [konflux-ui] updating namespace I0327 15:16:08.891702 3815 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:08.905151 3815 namespace.go:142] [tsf-tas] updating namespace I0327 15:16:08.911488 3815 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:16:08.911570 3815 namespace.go:142] [konflux-operator] updating namespace I0327 15:16:08.911737 3815 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:08.919745 3815 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:08.919776 3815 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:16:08.928969 3815 namespace.go:142] [rhbk-operator] updating namespace I0327 15:16:08.929078 3815 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:08.929403 3815 namespace.go:142] [konflux-ui] updating namespace I0327 15:16:08.943427 3815 namespace.go:142] [konflux-operator] updating namespace I0327 15:16:08.955099 3815 namespace.go:142] [konflux-operator] updating namespace I0327 15:16:08.963202 3815 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:16:08.975603 3815 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:16:08.986816 3815 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:09.022928 3815 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:09.022988 3815 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:16:09.026943 3815 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:16:09.027136 3815 namespace.go:142] [rhbk-operator] updating namespace I0327 15:16:09.036712 3815 namespace.go:142] [rhbk-operator] updating namespace I0327 15:16:09.042497 3815 namespace.go:142] [tsf-tas] updating namespace I0327 15:16:09.054069 3815 namespace.go:142] [tsf-tas] updating namespace I0327 15:16:09.891860 3815 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:09.897145 3815 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:16.738390 3815 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.738993 3815 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.739024 3815 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.739036 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:16.739047 3815 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.739052 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:16.759989 3815 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.760407 3815 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.760435 3815 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.760446 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:16.760460 3815 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:16.760467 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:17.763211 3815 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.763243 3815 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.763263 3815 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.763293 3815 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vwn7g of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.788683 3815 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.788703 3815 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.788718 3815 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:17.788734 3815 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfsxnkj of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:16:18.813544 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:16:18.841634 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0327 15:16:18.841704 3815 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-4vskp IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-4vskp: no pod IPs found I0327 15:16:19.887194 3815 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:19.887721 3815 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:19.887752 3815 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:19.887760 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:19.887771 3815 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:19.887776 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:20.917716 3815 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:20.917745 3815 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:20.917768 3815 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:20.917777 3815 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8qggs2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:22.292182 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0327 15:16:22.748182 3815 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:22.748765 3815 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:22.748789 3815 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:22.748797 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:22.748812 3815 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:22.748816 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:23.164550 3815 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-8468f77bc5-6zzkn] creating logical port rhbk-operator_rhbk-operator-8468f77bc5-6zzkn for pod on switch ip-10-0-0-192.ec2.internal I0327 15:16:23.164676 3815 kube.go:256] Updating pod rhbk-operator/rhbk-operator-8468f77bc5-6zzkn I0327 15:16:23.184582 3815 pod.go:62] [rhbk-operator/rhbk-operator-8468f77bc5-6zzkn] pod update took 19.928091ms I0327 15:16:23.184634 3815 base_network_controller_pods.go:937] [default/rhbk-operator/rhbk-operator-8468f77bc5-6zzkn] addLogicalPort annotation time took 19.978897ms I0327 15:16:23.185981 3815 pods.go:271] [rhbk-operator/rhbk-operator-8468f77bc5-6zzkn] addLogicalPort took 21.444648ms, libovsdb time 879.341µs I0327 15:16:23.541419 3815 cni.go:403] [rhbk-operator/rhbk-operator-8468f77bc5-6zzkn 8fc29603d37c6534dfcb0aab3d842f16319f4fc22b00fa7dd05b056703d0ff64 network default NAD default] ADD starting CNI request [rhbk-operator/rhbk-operator-8468f77bc5-6zzkn 8fc29603d37c6534dfcb0aab3d842f16319f4fc22b00fa7dd05b056703d0ff64 network default NAD default] I0327 15:16:23.550741 3815 helper_linux.go:450] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-8468f77bc5-6zzkn, hostIfaceName: 8fc29603d37c653, network: default, NAD default, SandboxID: "8fc29603d37c6534dfcb0aab3d842f16319f4fc22b00fa7dd05b056703d0ff64", PCI device ID: , UID: "700240e0-446a-441c-96c2-fd238b3ef929", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0327 15:16:23.594674 3815 cni.go:424] [rhbk-operator/rhbk-operator-8468f77bc5-6zzkn 8fc29603d37c6534dfcb0aab3d842f16319f4fc22b00fa7dd05b056703d0ff64 network default NAD default] ADD finished CNI request [rhbk-operator/rhbk-operator-8468f77bc5-6zzkn 8fc29603d37c6534dfcb0aab3d842f16319f4fc22b00fa7dd05b056703d0ff64 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:dc:e3:10:41:5d\",\"name\":\"8fc29603d37c653\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a7f89408-da29-46d1-b806-9cb6b1599b2f\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:16:23.765035 3815 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:23.765555 3815 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:23.765609 3815 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:23.765620 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:23.765630 3815 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:23.765634 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:23.783743 3815 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:23.783766 3815 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:23.783780 3815 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:23.783791 3815 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nd6k2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:24.322300 3815 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0327 15:16:24.804353 3815 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:24.804382 3815 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:24.804395 3815 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:24.804403 3815 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafz5js5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:33.839061 3815 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:33.839632 3815 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:33.839660 3815 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:33.839671 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:33.839681 3815 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:33.839685 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:33.917919 3815 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd] creating logical port konflux-operator_konflux-operator-controller-manager-7db75c44-mb9wd for pod on switch ip-10-0-0-192.ec2.internal I0327 15:16:33.918048 3815 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd I0327 15:16:33.946428 3815 pod.go:62] [konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd] pod update took 28.404206ms I0327 15:16:33.946453 3815 base_network_controller_pods.go:937] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd] addLogicalPort annotation time took 28.435701ms I0327 15:16:33.947661 3815 pods.go:271] [konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd] addLogicalPort took 29.764228ms, libovsdb time 848.211µs I0327 15:16:34.274873 3815 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd 135e7b93f29ebf2890b80ab89776016c9ba2c566e19fa669356832a9da23803c network default NAD default] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd 135e7b93f29ebf2890b80ab89776016c9ba2c566e19fa669356832a9da23803c network default NAD default] I0327 15:16:34.284286 3815 helper_linux.go:450] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-7db75c44-mb9wd, hostIfaceName: 135e7b93f29ebf2, network: default, NAD default, SandboxID: "135e7b93f29ebf2890b80ab89776016c9ba2c566e19fa669356832a9da23803c", PCI device ID: , UID: "4f304968-65ff-4b17-8ed3-c9f31a275ee5", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0327 15:16:34.329862 3815 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd 135e7b93f29ebf2890b80ab89776016c9ba2c566e19fa669356832a9da23803c network default NAD default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd 135e7b93f29ebf2890b80ab89776016c9ba2c566e19fa669356832a9da23803c network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:ec:b7:61:72:9e\",\"name\":\"135e7b93f29ebf2\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b6a91d02-3148-4440-ba6d-261cab9ef9d7\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:16:34.712485 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 15:16:34.837824 3815 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:34.837851 3815 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:34.837867 3815 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:34.837878 3815 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad776rx4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:35.327240 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 W0327 15:16:36.959190 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:16:38.167655 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:16:38.187731 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:16:38.202632 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:16:38.214914 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0327 15:16:38.317377 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0327 15:16:41.253355 3815 namespace.go:98] [cert-manager] adding namespace I0327 15:16:41.254429 3815 namespace.go:102] [cert-manager] adding namespace took 1.04726ms I0327 15:16:41.268009 3815 namespace.go:142] [cert-manager] updating namespace I0327 15:16:41.276755 3815 namespace.go:142] [cert-manager] updating namespace I0327 15:16:44.860538 3815 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:44.861086 3815 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:16:44.861112 3815 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:44.861120 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:16:44.861131 3815 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:16:44.861140 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:16:45.885757 3815 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:45.885788 3815 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:45.885805 3815 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:45.885815 3815 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7zf82p of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:16:47.298615 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 W0327 15:16:47.955492 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:16:49.090220 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0327 15:16:49.090246 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0327 15:16:50.790900 3815 namespace.go:142] [openshift-operators] updating namespace I0327 15:16:50.808423 3815 namespace.go:142] [openshift-operators] updating namespace I0327 15:16:54.752743 3815 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-qxdgv] creating logical port openshift-operators_rhtas-operator-controller-manager-6987b7f98c-qxdgv for pod on switch ip-10-0-0-192.ec2.internal I0327 15:16:54.752859 3815 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-qxdgv I0327 15:16:54.780275 3815 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-qxdgv] pod update took 27.441815ms I0327 15:16:54.780299 3815 base_network_controller_pods.go:937] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-qxdgv] addLogicalPort annotation time took 27.471923ms I0327 15:16:54.781621 3815 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-qxdgv] addLogicalPort took 28.89001ms, libovsdb time 890.497µs I0327 15:16:55.113759 3815 cni.go:403] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-qxdgv 53adb47ba2dc9007e175f9499a24a606bdb58acc7dd13f729e3eb6844697beb5 network default NAD default] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-qxdgv 53adb47ba2dc9007e175f9499a24a606bdb58acc7dd13f729e3eb6844697beb5 network default NAD default] I0327 15:16:55.122064 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-6987b7f98c-qxdgv, hostIfaceName: 53adb47ba2dc900, network: default, NAD default, SandboxID: "53adb47ba2dc9007e175f9499a24a606bdb58acc7dd13f729e3eb6844697beb5", PCI device ID: , UID: "9e678433-9fd0-4b64-b0f8-8caea433f72e", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0327 15:16:55.167389 3815 cni.go:424] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-qxdgv 53adb47ba2dc9007e175f9499a24a606bdb58acc7dd13f729e3eb6844697beb5 network default NAD default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-qxdgv 53adb47ba2dc9007e175f9499a24a606bdb58acc7dd13f729e3eb6844697beb5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:ba:bc:2a:3d:3f\",\"name\":\"53adb47ba2dc900\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0d662058-1722-4295-8870-1d380eb6714a\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:16:58.649697 3815 namespace.go:98] [trusted-artifact-signer] adding namespace I0327 15:16:58.650506 3815 namespace.go:102] [trusted-artifact-signer] adding namespace took 786.124µs I0327 15:16:58.668883 3815 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:16:58.700605 3815 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:16:58.852902 3815 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-txxq6] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-txxq6 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:16:58.853042 3815 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-txxq6 I0327 15:16:58.872844 3815 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-txxq6] pod update took 19.826355ms I0327 15:16:58.872873 3815 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-txxq6] addLogicalPort annotation time took 19.861059ms I0327 15:16:58.874144 3815 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-txxq6] addLogicalPort took 21.26306ms, libovsdb time 848.769µs I0327 15:16:58.994504 3815 namespace.go:142] [openshift-operators] updating namespace W0327 15:16:59.066455 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 15:16:59.210751 3815 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-txxq6 4232931e8da67501cf3d9423d62381298912809c2b144556afbd9477936c25ce network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-txxq6 4232931e8da67501cf3d9423d62381298912809c2b144556afbd9477936c25ce network default NAD default] I0327 15:16:59.219134 3815 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-txxq6, hostIfaceName: 4232931e8da6750, network: default, NAD default, SandboxID: "4232931e8da67501cf3d9423d62381298912809c2b144556afbd9477936c25ce", PCI device ID: , UID: "f98c93d3-3823-4a21-bda5-6f0da1eb17c7", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0327 15:16:59.262475 3815 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-txxq6 4232931e8da67501cf3d9423d62381298912809c2b144556afbd9477936c25ce network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-txxq6 4232931e8da67501cf3d9423d62381298912809c2b144556afbd9477936c25ce network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:51:2b:bc:57:5a\",\"name\":\"4232931e8da6750\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3321b131-0df5-4d06-a6d7-6b7223c8fa96\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:16:59.866657 3815 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6684678d86-gs9f7] creating logical port openshift-operators_openshift-pipelines-operator-6684678d86-gs9f7 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:16:59.866790 3815 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6684678d86-gs9f7 I0327 15:16:59.897929 3815 pod.go:62] [openshift-operators/openshift-pipelines-operator-6684678d86-gs9f7] pod update took 31.165827ms I0327 15:16:59.897952 3815 base_network_controller_pods.go:937] [default/openshift-operators/openshift-pipelines-operator-6684678d86-gs9f7] addLogicalPort annotation time took 31.193394ms I0327 15:16:59.899232 3815 pods.go:271] [openshift-operators/openshift-pipelines-operator-6684678d86-gs9f7] addLogicalPort took 32.592179ms, libovsdb time 904.497µs I0327 15:17:00.252046 3815 cni.go:403] [openshift-operators/openshift-pipelines-operator-6684678d86-gs9f7 3c9878a76d2997ce9797db174db910e9eef3f350fddb182529ee0a23085bd5f8 network default NAD default] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-gs9f7 3c9878a76d2997ce9797db174db910e9eef3f350fddb182529ee0a23085bd5f8 network default NAD default] I0327 15:17:00.262701 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6684678d86-gs9f7, hostIfaceName: 3c9878a76d2997c, network: default, NAD default, SandboxID: "3c9878a76d2997ce9797db174db910e9eef3f350fddb182529ee0a23085bd5f8", PCI device ID: , UID: "6b1af3a8-6ce2-446c-a0d7-ae23f24d519e", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0327 15:17:00.319217 3815 cni.go:424] [openshift-operators/openshift-pipelines-operator-6684678d86-gs9f7 3c9878a76d2997ce9797db174db910e9eef3f350fddb182529ee0a23085bd5f8 network default NAD default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-gs9f7 3c9878a76d2997ce9797db174db910e9eef3f350fddb182529ee0a23085bd5f8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:48:7b:12:bb:a7\",\"name\":\"3c9878a76d2997c\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5997aa6c-1c97-49c7-af6d-500b607ec1d5\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:17:01.509537 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0327 15:17:01.521657 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0327 15:17:01.537101 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0327 15:17:01.544420 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 15:17:02.103756 3815 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59ccb4bb6b-wjqrm] creating logical port cert-manager_cert-manager-59ccb4bb6b-wjqrm for pod on switch ip-10-0-0-192.ec2.internal I0327 15:17:02.103854 3815 kube.go:256] Updating pod cert-manager/cert-manager-59ccb4bb6b-wjqrm I0327 15:17:02.125323 3815 pod.go:62] [cert-manager/cert-manager-59ccb4bb6b-wjqrm] pod update took 21.491315ms I0327 15:17:02.125353 3815 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-59ccb4bb6b-wjqrm] addLogicalPort annotation time took 21.529377ms I0327 15:17:02.126993 3815 pods.go:271] [cert-manager/cert-manager-59ccb4bb6b-wjqrm] addLogicalPort took 23.251571ms, libovsdb time 1.126839ms I0327 15:17:03.197082 3815 cni.go:403] [cert-manager/cert-manager-59ccb4bb6b-wjqrm be6fac0b5fc1cc314facaee100d549bec09985ebcdf08963504e0394bb52f00b network default NAD default] ADD starting CNI request [cert-manager/cert-manager-59ccb4bb6b-wjqrm be6fac0b5fc1cc314facaee100d549bec09985ebcdf08963504e0394bb52f00b network default NAD default] I0327 15:17:03.206690 3815 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59ccb4bb6b-wjqrm, hostIfaceName: be6fac0b5fc1cc3, network: default, NAD default, SandboxID: "be6fac0b5fc1cc314facaee100d549bec09985ebcdf08963504e0394bb52f00b", PCI device ID: , UID: "96f3f4ab-0587-4bed-9140-9f6d59726dfd", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0327 15:17:03.258887 3815 cni.go:424] [cert-manager/cert-manager-59ccb4bb6b-wjqrm be6fac0b5fc1cc314facaee100d549bec09985ebcdf08963504e0394bb52f00b network default NAD default] ADD finished CNI request [cert-manager/cert-manager-59ccb4bb6b-wjqrm be6fac0b5fc1cc314facaee100d549bec09985ebcdf08963504e0394bb52f00b network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:9f:dc:16:ef:e6\",\"name\":\"be6fac0b5fc1cc3\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2fbfabde-8b01-4bd7-bad6-9c287a667b94\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:17:09.201627 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=91 W0327 15:17:10.076984 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 15:17:11.861567 3815 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:11.862104 3815 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:11.885451 3815 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:11.885472 3815 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:11.896227 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0327 15:17:11.896259 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0327 15:17:14.614070 3815 namespace.go:98] [openshift-pipelines] adding namespace I0327 15:17:14.614966 3815 namespace.go:102] [openshift-pipelines] adding namespace took 870.613µs I0327 15:17:14.636455 3815 namespace.go:142] [openshift-pipelines] updating namespace I0327 15:17:15.327642 3815 namespace.go:142] [cert-manager] updating namespace I0327 15:17:15.525657 3815 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:17:15.926621 3815 namespace.go:142] [konflux-operator] updating namespace I0327 15:17:16.140155 3815 namespace.go:142] [konflux-ui] updating namespace I0327 15:17:16.330503 3815 namespace.go:142] [open-cluster-management-2pa4nkn99kaqqvukhcr451snphcdn29u] updating namespace I0327 15:17:16.732207 3815 namespace.go:142] [rhbk-operator] updating namespace I0327 15:17:16.936657 3815 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:17:17.133995 3815 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:17.348025 3815 namespace.go:142] [tsf] updating namespace I0327 15:17:17.557975 3815 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:17.765251 3815 namespace.go:142] [tsf-tas] updating namespace I0327 15:17:17.966412 3815 namespace.go:142] [tsf-tpa] updating namespace I0327 15:17:18.108185 3815 namespace.go:142] [cert-manager] updating namespace I0327 15:17:18.116818 3815 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:17:18.126783 3815 namespace.go:142] [default] updating namespace I0327 15:17:18.137999 3815 namespace.go:142] [konflux-operator] updating namespace I0327 15:17:18.148958 3815 namespace.go:142] [konflux-ui] updating namespace I0327 15:17:18.161510 3815 namespace.go:142] [open-cluster-management-2pa4nkn99kaqqvukhcr451snphcdn29u] updating namespace I0327 15:17:18.171393 3815 namespace.go:142] [openshift] updating namespace I0327 15:17:18.179892 3815 namespace.go:142] [rhbk-operator] updating namespace I0327 15:17:18.191200 3815 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:17:18.204640 3815 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:18.208334 3815 namespace.go:142] [tsf] updating namespace I0327 15:17:18.220459 3815 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:18.236544 3815 namespace.go:142] [tsf-tas] updating namespace I0327 15:17:18.256192 3815 namespace.go:142] [tsf-tpa] updating namespace I0327 15:17:18.359099 3815 namespace.go:142] [cert-manager] updating namespace I0327 15:17:18.469633 3815 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:17:18.571024 3815 namespace.go:142] [default] updating namespace I0327 15:17:18.658448 3815 namespace.go:142] [konflux-operator] updating namespace I0327 15:17:18.755754 3815 namespace.go:142] [konflux-ui] updating namespace I0327 15:17:18.858144 3815 namespace.go:142] [open-cluster-management-2pa4nkn99kaqqvukhcr451snphcdn29u] updating namespace I0327 15:17:18.955686 3815 namespace.go:142] [openshift] updating namespace I0327 15:17:19.057560 3815 namespace.go:142] [rhbk-operator] updating namespace I0327 15:17:19.160703 3815 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:17:19.254448 3815 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:19.356134 3815 namespace.go:142] [tsf] updating namespace I0327 15:17:19.456983 3815 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:19.563794 3815 namespace.go:142] [tsf-tas] updating namespace I0327 15:17:19.657616 3815 namespace.go:142] [tsf-tpa] updating namespace I0327 15:17:20.010538 3815 namespace.go:142] [openshift-pipelines] updating namespace W0327 15:17:23.793883 3815 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0327 15:17:27.471398 3815 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-b79bdd59-hm5cb] creating logical port openshift-pipelines_tekton-events-controller-b79bdd59-hm5cb for pod on switch ip-10-0-0-192.ec2.internal I0327 15:17:27.471491 3815 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-b79bdd59-hm5cb I0327 15:17:27.487683 3815 pod.go:62] [openshift-pipelines/tekton-events-controller-b79bdd59-hm5cb] pod update took 16.218652ms I0327 15:17:27.487707 3815 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-events-controller-b79bdd59-hm5cb] addLogicalPort annotation time took 16.247749ms I0327 15:17:27.489165 3815 pods.go:271] [openshift-pipelines/tekton-events-controller-b79bdd59-hm5cb] addLogicalPort took 17.783381ms, libovsdb time 1.030547ms I0327 15:17:27.673500 3815 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-w7wzs] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-w7wzs for pod on switch ip-10-0-0-192.ec2.internal I0327 15:17:27.673618 3815 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-w7wzs I0327 15:17:27.688719 3815 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-w7wzs] pod update took 15.152905ms I0327 15:17:27.688740 3815 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-w7wzs] addLogicalPort annotation time took 15.1781ms I0327 15:17:27.689985 3815 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-w7wzs] addLogicalPort took 16.502976ms, libovsdb time 836.812µs I0327 15:17:27.834463 3815 cni.go:403] [openshift-pipelines/tekton-events-controller-b79bdd59-hm5cb d21ad58f3261a128e4feaec24306bc97bf03ef80842301b3380d8cec5a228dea network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-hm5cb d21ad58f3261a128e4feaec24306bc97bf03ef80842301b3380d8cec5a228dea network default NAD default] I0327 15:17:27.843427 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-b79bdd59-hm5cb, hostIfaceName: d21ad58f3261a12, network: default, NAD default, SandboxID: "d21ad58f3261a128e4feaec24306bc97bf03ef80842301b3380d8cec5a228dea", PCI device ID: , UID: "26f5a60c-1547-4ca4-8944-c1d3c2d38a04", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0327 15:17:27.892454 3815 cni.go:424] [openshift-pipelines/tekton-events-controller-b79bdd59-hm5cb d21ad58f3261a128e4feaec24306bc97bf03ef80842301b3380d8cec5a228dea network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-hm5cb d21ad58f3261a128e4feaec24306bc97bf03ef80842301b3380d8cec5a228dea network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:89:28:5f:43:fa\",\"name\":\"d21ad58f3261a12\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2684168e-3c79-4dc8-bf48-e3c64c4df41b\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:17:28.036827 3815 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-w7wzs a446a161d5f871e19553d95a9bed8f3b5ef0cf5c30a50b2cc062c21b663abd9c network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-w7wzs a446a161d5f871e19553d95a9bed8f3b5ef0cf5c30a50b2cc062c21b663abd9c network default NAD default] I0327 15:17:28.046357 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-w7wzs, hostIfaceName: a446a161d5f871e, network: default, NAD default, SandboxID: "a446a161d5f871e19553d95a9bed8f3b5ef0cf5c30a50b2cc062c21b663abd9c", PCI device ID: , UID: "879e24a2-f4fc-41b7-a30f-88f38ca55415", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0327 15:17:28.093236 3815 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-w7wzs a446a161d5f871e19553d95a9bed8f3b5ef0cf5c30a50b2cc062c21b663abd9c network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-w7wzs a446a161d5f871e19553d95a9bed8f3b5ef0cf5c30a50b2cc062c21b663abd9c network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:7d:23:6a:db:e1\",\"name\":\"a446a161d5f871e\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c172be6a-eac5-4c9f-8189-3327f57c5717\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:17:31.235615 3815 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:17:31.235633 3815 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:17:31.235638 3815 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:17:32.033489 3815 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0327 15:17:32.257918 3815 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:17:32.257939 3815 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:17:32.257943 3815 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:17:32.257947 3815 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:17:34.039941 3815 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:17:34.039964 3815 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:17:34.039969 3815 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0327 15:17:39.326709 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 W0327 15:17:42.241771 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:17:42.241794 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:17:42.241801 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:17:43.242772 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:17:43.242791 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:17:43.242797 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:17:43.242801 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0327 15:17:51.089815 3815 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-qdkvq] creating logical port openshift-pipelines_tekton-triggers-controller-68564d75ff-qdkvq for pod on switch ip-10-0-0-192.ec2.internal I0327 15:17:51.089906 3815 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-68564d75ff-qdkvq I0327 15:17:51.103506 3815 pod.go:62] [openshift-pipelines/tekton-triggers-controller-68564d75ff-qdkvq] pod update took 13.625039ms I0327 15:17:51.103528 3815 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-qdkvq] addLogicalPort annotation time took 13.652545ms I0327 15:17:51.104935 3815 pods.go:271] [openshift-pipelines/tekton-triggers-controller-68564d75ff-qdkvq] addLogicalPort took 15.13663ms, libovsdb time 912.362µs I0327 15:17:51.189263 3815 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-glscs] creating logical port openshift-pipelines_tekton-triggers-webhook-55d94ff6c6-glscs for pod on switch ip-10-0-0-192.ec2.internal I0327 15:17:51.189357 3815 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-glscs I0327 15:17:51.202420 3815 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-glscs] pod update took 13.089425ms I0327 15:17:51.202456 3815 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-glscs] addLogicalPort annotation time took 13.124076ms I0327 15:17:51.203892 3815 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-glscs] addLogicalPort took 14.65471ms, libovsdb time 781.61µs I0327 15:17:51.290228 3815 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-pkrjc] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-8bb674894-pkrjc for pod on switch ip-10-0-0-192.ec2.internal I0327 15:17:51.290327 3815 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-pkrjc I0327 15:17:51.306791 3815 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-pkrjc] pod update took 16.493563ms I0327 15:17:51.306814 3815 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-pkrjc] addLogicalPort annotation time took 16.521399ms I0327 15:17:51.308155 3815 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-pkrjc] addLogicalPort took 17.946556ms, libovsdb time 867.098µs I0327 15:17:51.452454 3815 cni.go:403] [openshift-pipelines/tekton-triggers-controller-68564d75ff-qdkvq 2acb54ea1645cb173a1e75ba8a2ffc11b1100d35400c08ca87dcc8d32bcd5ecf network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-qdkvq 2acb54ea1645cb173a1e75ba8a2ffc11b1100d35400c08ca87dcc8d32bcd5ecf network default NAD default] I0327 15:17:51.461615 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-68564d75ff-qdkvq, hostIfaceName: 2acb54ea1645cb1, network: default, NAD default, SandboxID: "2acb54ea1645cb173a1e75ba8a2ffc11b1100d35400c08ca87dcc8d32bcd5ecf", PCI device ID: , UID: "43cd3ee5-b28f-4551-b853-3e43508a23b4", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0327 15:17:51.509330 3815 cni.go:424] [openshift-pipelines/tekton-triggers-controller-68564d75ff-qdkvq 2acb54ea1645cb173a1e75ba8a2ffc11b1100d35400c08ca87dcc8d32bcd5ecf network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-qdkvq 2acb54ea1645cb173a1e75ba8a2ffc11b1100d35400c08ca87dcc8d32bcd5ecf network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:82:68:08:1f:a0\",\"name\":\"2acb54ea1645cb1\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/efaf6d6d-f8e3-426e-9f3c-b48a26a03fa3\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:17:51.549017 3815 cni.go:403] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-glscs 9e1f973676f5b3995510cbd0c956ac50440b5edd3f39ac22d891a495e5ad84cb network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-glscs 9e1f973676f5b3995510cbd0c956ac50440b5edd3f39ac22d891a495e5ad84cb network default NAD default] I0327 15:17:51.558211 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-55d94ff6c6-glscs, hostIfaceName: 9e1f973676f5b39, network: default, NAD default, SandboxID: "9e1f973676f5b3995510cbd0c956ac50440b5edd3f39ac22d891a495e5ad84cb", PCI device ID: , UID: "1ca5c8e8-10a0-408f-bd10-64aecef80b65", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0327 15:17:51.603547 3815 cni.go:424] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-glscs 9e1f973676f5b3995510cbd0c956ac50440b5edd3f39ac22d891a495e5ad84cb network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-glscs 9e1f973676f5b3995510cbd0c956ac50440b5edd3f39ac22d891a495e5ad84cb network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:cb:8b:47:9f:1f\",\"name\":\"9e1f973676f5b39\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5cef2fce-11bf-4d2d-bda9-26bbf5a1ed86\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:17:51.649124 3815 cni.go:403] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-pkrjc 11b07b04b7e5ff94ccb913c33c02607a07b231552559f54884a66ea0eed9a7a0 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-pkrjc 11b07b04b7e5ff94ccb913c33c02607a07b231552559f54884a66ea0eed9a7a0 network default NAD default] I0327 15:17:51.666517 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-8bb674894-pkrjc, hostIfaceName: 11b07b04b7e5ff9, network: default, NAD default, SandboxID: "11b07b04b7e5ff94ccb913c33c02607a07b231552559f54884a66ea0eed9a7a0", PCI device ID: , UID: "288106b8-5491-4d03-9183-26d216fe49ec", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0327 15:17:51.713848 3815 cni.go:424] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-pkrjc 11b07b04b7e5ff94ccb913c33c02607a07b231552559f54884a66ea0eed9a7a0 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-pkrjc 11b07b04b7e5ff94ccb913c33c02607a07b231552559f54884a66ea0eed9a7a0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:8b:22:29:89:7d\",\"name\":\"11b07b04b7e5ff9\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/519e92e2-ef27-4f34-ba52-0fd47e75a6eb\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:17:55.409405 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0327 15:17:56.068158 3815 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:56.068767 3815 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:56.086243 3815 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:56.086267 3815 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:56.091728 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0327 15:17:56.091750 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0327 15:17:57.200499 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 W0327 15:18:02.971873 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:02.971903 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:02.976149 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 15:18:06.370872 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0327 15:18:10.201520 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=208 I0327 15:18:13.118215 3815 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 15:18:25.316912 3815 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:18:25.317477 3815 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:18:26.075250 3815 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:18:26.075273 3815 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:18:26.078439 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 15:18:26.078462 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0327 15:18:28.098266 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 15:18:28.098371 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:28.098390 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:39.162166 3815 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:18:39.162188 3815 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:18:39.162192 3815 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0327 15:18:55.209105 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=909 I0327 15:18:58.405130 3815 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:18:58.405743 3815 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:18:59.011453 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-34.ec2.internal I0327 15:18:59.011484 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-34.ec2.internal: 38.07µs I0327 15:18:59.011516 3815 egressfirewall.go:628] Syncing node "ip-10-0-2-34.ec2.internal" for egress firewall I0327 15:19:01.287867 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0327 15:19:05.439703 3815 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-29nwp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:05.440237 3815 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-29nwp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:06.470899 3815 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-29nwp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:06.470922 3815 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-29nwp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:06.814958 3815 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:06.815510 3815 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:07.309969 3815 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:07.309994 3815 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:07.316098 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 15:19:07.316120 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0327 15:19:08.251974 3815 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-shgnp] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-shgnp for pod on switch ip-10-0-0-192.ec2.internal I0327 15:19:08.252068 3815 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-shgnp I0327 15:19:08.267402 3815 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-shgnp] pod update took 15.357712ms I0327 15:19:08.267425 3815 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-shgnp] addLogicalPort annotation time took 15.386191ms I0327 15:19:08.268851 3815 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-shgnp] addLogicalPort took 16.8961ms, libovsdb time 949.731µs I0327 15:19:08.610500 3815 cni.go:403] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-shgnp d6efb27a337a4b53986046b079aa0938886b2e6507efa2a186614d456f02b9b4 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-shgnp d6efb27a337a4b53986046b079aa0938886b2e6507efa2a186614d456f02b9b4 network default NAD default] I0327 15:19:08.618978 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-shgnp, hostIfaceName: d6efb27a337a4b5, network: default, NAD default, SandboxID: "d6efb27a337a4b53986046b079aa0938886b2e6507efa2a186614d456f02b9b4", PCI device ID: , UID: "fdf77b69-dffd-4cef-9142-37f4546836b2", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0327 15:19:08.666540 3815 cni.go:424] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-shgnp d6efb27a337a4b53986046b079aa0938886b2e6507efa2a186614d456f02b9b4 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-shgnp d6efb27a337a4b53986046b079aa0938886b2e6507efa2a186614d456f02b9b4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:3a:45:ac:c7:11\",\"name\":\"d6efb27a337a4b5\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/500fe11b-60ac-40cd-ada0-4f82394387c4\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:19:09.332307 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:19:09.332323 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 15:19:09.332329 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:19:11.345992 3815 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 15:19:11.346017 3815 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 15:19:11.746487 3815 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0327 15:19:21.963425 3815 namespace.go:142] [openshift-pipelines] updating namespace W0327 15:19:22.768207 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0327 15:19:58.487576 3815 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:58.487629 3815 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:58.493275 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 15:19:58.493304 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 15:20:00.178636 3815 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29577080-tsqzk for pod on switch ip-10-0-0-192.ec2.internal I0327 15:20:00.178752 3815 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk I0327 15:20:00.191078 3815 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk] pod update took 12.346155ms I0327 15:20:00.191104 3815 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk] addLogicalPort annotation time took 12.376132ms I0327 15:20:00.192421 3815 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk] addLogicalPort took 13.801219ms, libovsdb time 907.704µs I0327 15:20:00.536887 3815 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk 2b50d5a0925a26c59fbf73f53f147b275cdc2fb542ca38ad52cc48334ebf8981 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk 2b50d5a0925a26c59fbf73f53f147b275cdc2fb542ca38ad52cc48334ebf8981 network default NAD default] I0327 15:20:00.545352 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29577080-tsqzk, hostIfaceName: 2b50d5a0925a26c, network: default, NAD default, SandboxID: "2b50d5a0925a26c59fbf73f53f147b275cdc2fb542ca38ad52cc48334ebf8981", PCI device ID: , UID: "26df295a-f7b4-4688-8a23-813c9805fbf2", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0327 15:20:00.593011 3815 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk 2b50d5a0925a26c59fbf73f53f147b275cdc2fb542ca38ad52cc48334ebf8981 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk 2b50d5a0925a26c59fbf73f53f147b275cdc2fb542ca38ad52cc48334ebf8981 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:ba:d9:b7:12:e6\",\"name\":\"2b50d5a0925a26c\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e7c99690-0b47-493b-b9df-5892d7c82c8f\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:20:04.015262 3815 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk 2b50d5a0925a26c59fbf73f53f147b275cdc2fb542ca38ad52cc48334ebf8981 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk 2b50d5a0925a26c59fbf73f53f147b275cdc2fb542ca38ad52cc48334ebf8981 network default NAD default] I0327 15:20:04.093005 3815 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk 2b50d5a0925a26c59fbf73f53f147b275cdc2fb542ca38ad52cc48334ebf8981 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk 2b50d5a0925a26c59fbf73f53f147b275cdc2fb542ca38ad52cc48334ebf8981 network default NAD default], result "{}", err I0327 15:20:04.142843 3815 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:04.142886 3815 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk I0327 15:20:04.144767 3815 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk, ips: 10.129.0.42 I0327 15:20:04.144854 3815 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:05.026251 3815 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:05.026274 3815 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-tsqzk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:07.766483 3815 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:07.767033 3815 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:07.787066 3815 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:07.787083 3815 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:07.792716 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 15:20:07.792742 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 15:20:09.372925 3815 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-k4vwr-kz9c2] creating logical port tsf-tas_segment-backup-installation-k4vwr-kz9c2 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:20:09.373047 3815 kube.go:256] Updating pod tsf-tas/segment-backup-installation-k4vwr-kz9c2 I0327 15:20:09.386527 3815 pod.go:62] [tsf-tas/segment-backup-installation-k4vwr-kz9c2] pod update took 13.50203ms I0327 15:20:09.386551 3815 base_network_controller_pods.go:937] [default/tsf-tas/segment-backup-installation-k4vwr-kz9c2] addLogicalPort annotation time took 13.533392ms I0327 15:20:09.387754 3815 pods.go:271] [tsf-tas/segment-backup-installation-k4vwr-kz9c2] addLogicalPort took 14.845088ms, libovsdb time 806.734µs I0327 15:20:09.732668 3815 cni.go:403] [tsf-tas/segment-backup-installation-k4vwr-kz9c2 5c872341dcadf1d3556ef3925b399d307b924cc4246a26bd22dfc13397be40b8 network default NAD default] ADD starting CNI request [tsf-tas/segment-backup-installation-k4vwr-kz9c2 5c872341dcadf1d3556ef3925b399d307b924cc4246a26bd22dfc13397be40b8 network default NAD default] I0327 15:20:09.741660 3815 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-k4vwr-kz9c2, hostIfaceName: 5c872341dcadf1d, network: default, NAD default, SandboxID: "5c872341dcadf1d3556ef3925b399d307b924cc4246a26bd22dfc13397be40b8", PCI device ID: , UID: "2e3cbde1-a0bd-4dc9-b621-bd3a21b9c6c9", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0327 15:20:09.790440 3815 cni.go:424] [tsf-tas/segment-backup-installation-k4vwr-kz9c2 5c872341dcadf1d3556ef3925b399d307b924cc4246a26bd22dfc13397be40b8 network default NAD default] ADD finished CNI request [tsf-tas/segment-backup-installation-k4vwr-kz9c2 5c872341dcadf1d3556ef3925b399d307b924cc4246a26bd22dfc13397be40b8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:b3:38:74:1e:c8\",\"name\":\"5c872341dcadf1d\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a049c2ee-7b64-4b65-b806-f8dbf0402de2\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:20:10.395119 3815 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-ms7lz-g4ghb] creating logical port tsf-tas_ctlog-createtree-job-ms7lz-g4ghb for pod on switch ip-10-0-0-192.ec2.internal I0327 15:20:10.395309 3815 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-ms7lz-g4ghb I0327 15:20:10.411782 3815 pod.go:62] [tsf-tas/ctlog-createtree-job-ms7lz-g4ghb] pod update took 16.53191ms I0327 15:20:10.411817 3815 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-createtree-job-ms7lz-g4ghb] addLogicalPort annotation time took 16.571001ms I0327 15:20:10.413721 3815 pods.go:271] [tsf-tas/ctlog-createtree-job-ms7lz-g4ghb] addLogicalPort took 18.620801ms, libovsdb time 1.150812ms I0327 15:20:10.778982 3815 cni.go:403] [tsf-tas/ctlog-createtree-job-ms7lz-g4ghb be1e3a0acd26b8ad8ea87d3909ba635e03981a3604b524efe6aba5cfddce5d81 network default NAD default] ADD starting CNI request [tsf-tas/ctlog-createtree-job-ms7lz-g4ghb be1e3a0acd26b8ad8ea87d3909ba635e03981a3604b524efe6aba5cfddce5d81 network default NAD default] I0327 15:20:10.789714 3815 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-ms7lz-g4ghb, hostIfaceName: be1e3a0acd26b8a, network: default, NAD default, SandboxID: "be1e3a0acd26b8ad8ea87d3909ba635e03981a3604b524efe6aba5cfddce5d81", PCI device ID: , UID: "b1a95461-292e-42bb-9ded-c1c11ed3f64f", MAC: 0a:58:0a:81:00:2c, IPs: [10.129.0.44/23] I0327 15:20:10.851890 3815 cni.go:424] [tsf-tas/ctlog-createtree-job-ms7lz-g4ghb be1e3a0acd26b8ad8ea87d3909ba635e03981a3604b524efe6aba5cfddce5d81 network default NAD default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-ms7lz-g4ghb be1e3a0acd26b8ad8ea87d3909ba635e03981a3604b524efe6aba5cfddce5d81 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:4d:0b:1e:43:eb\",\"name\":\"be1e3a0acd26b8a\"},{\"mac\":\"0a:58:0a:81:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/38f4220d-ceb7-4aaf-9315-d3b66d4dfa5b\"}],\"ips\":[{\"address\":\"10.129.0.44/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:20:13.193692 3815 base_network_controller_pods.go:487] [default/tsf-tas/trillian-db-569444776d-k5qlx] creating logical port tsf-tas_trillian-db-569444776d-k5qlx for pod on switch ip-10-0-0-192.ec2.internal I0327 15:20:13.193806 3815 kube.go:256] Updating pod tsf-tas/trillian-db-569444776d-k5qlx I0327 15:20:13.207026 3815 pod.go:62] [tsf-tas/trillian-db-569444776d-k5qlx] pod update took 13.256011ms I0327 15:20:13.207056 3815 base_network_controller_pods.go:937] [default/tsf-tas/trillian-db-569444776d-k5qlx] addLogicalPort annotation time took 13.289702ms I0327 15:20:13.208719 3815 pods.go:271] [tsf-tas/trillian-db-569444776d-k5qlx] addLogicalPort took 15.047689ms, libovsdb time 1.086727ms W0327 15:20:13.674831 3815 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0327 15:20:13.674932 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0327 15:20:17.967561 3815 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0327 15:20:17.990520 3815 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0327 15:20:22.118818 3815 cni.go:403] [tsf-tas/segment-backup-installation-k4vwr-kz9c2 5c872341dcadf1d3556ef3925b399d307b924cc4246a26bd22dfc13397be40b8 network default NAD default] DEL starting CNI request [tsf-tas/segment-backup-installation-k4vwr-kz9c2 5c872341dcadf1d3556ef3925b399d307b924cc4246a26bd22dfc13397be40b8 network default NAD default] I0327 15:20:22.205478 3815 cni.go:424] [tsf-tas/segment-backup-installation-k4vwr-kz9c2 5c872341dcadf1d3556ef3925b399d307b924cc4246a26bd22dfc13397be40b8 network default NAD default] DEL finished CNI request [tsf-tas/segment-backup-installation-k4vwr-kz9c2 5c872341dcadf1d3556ef3925b399d307b924cc4246a26bd22dfc13397be40b8 network default NAD default], result "{}", err I0327 15:20:22.253139 3815 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-k4vwr-kz9c2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:22.253176 3815 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-k4vwr-kz9c2 I0327 15:20:22.254486 3815 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-k4vwr-kz9c2, ips: 10.129.0.43 I0327 15:20:22.254522 3815 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-k4vwr-kz9c2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:23.273036 3815 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-k4vwr-kz9c2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:23.273064 3815 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-k4vwr-kz9c2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:23.469414 3815 cni.go:403] [tsf-tas/trillian-db-569444776d-k5qlx a5b1fc2db9d7d157bad9648c8c3b06481587b8635e886d87be55fdc247a9aea0 network default NAD default] ADD starting CNI request [tsf-tas/trillian-db-569444776d-k5qlx a5b1fc2db9d7d157bad9648c8c3b06481587b8635e886d87be55fdc247a9aea0 network default NAD default] I0327 15:20:23.477756 3815 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-db-569444776d-k5qlx, hostIfaceName: a5b1fc2db9d7d15, network: default, NAD default, SandboxID: "a5b1fc2db9d7d157bad9648c8c3b06481587b8635e886d87be55fdc247a9aea0", PCI device ID: , UID: "24f4ba26-5568-4284-9009-c698874ec8e8", MAC: 0a:58:0a:81:00:2d, IPs: [10.129.0.45/23] I0327 15:20:23.730074 3815 cni.go:424] [tsf-tas/trillian-db-569444776d-k5qlx a5b1fc2db9d7d157bad9648c8c3b06481587b8635e886d87be55fdc247a9aea0 network default NAD default] ADD finished CNI request [tsf-tas/trillian-db-569444776d-k5qlx a5b1fc2db9d7d157bad9648c8c3b06481587b8635e886d87be55fdc247a9aea0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:f7:14:07:b4:b2\",\"name\":\"a5b1fc2db9d7d15\"},{\"mac\":\"0a:58:0a:81:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1962cb7e-1f37-4339-b715-79d93b15d03e\"}],\"ips\":[{\"address\":\"10.129.0.45/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:20:26.141025 3815 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0327 15:20:37.190066 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 3306: empty IP address endpoints for service tsf-tas/trillian-mysql I0327 15:20:57.002771 3815 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:57.003317 3815 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:57.029913 3815 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:57.029935 3815 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:57.037706 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0327 15:20:57.037735 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0327 15:20:57.667944 3815 base_network_controller_pods.go:487] [default/tsf/patch-tekton-config-56jxt] creating logical port tsf_patch-tekton-config-56jxt for pod on switch ip-10-0-0-192.ec2.internal I0327 15:20:57.668039 3815 kube.go:256] Updating pod tsf/patch-tekton-config-56jxt I0327 15:20:57.680090 3815 pod.go:62] [tsf/patch-tekton-config-56jxt] pod update took 12.074447ms I0327 15:20:57.680116 3815 base_network_controller_pods.go:937] [default/tsf/patch-tekton-config-56jxt] addLogicalPort annotation time took 12.106961ms I0327 15:20:57.681323 3815 pods.go:271] [tsf/patch-tekton-config-56jxt] addLogicalPort took 13.402225ms, libovsdb time 813.772µs I0327 15:20:58.055208 3815 cni.go:403] [tsf/patch-tekton-config-56jxt c2700f7259d093e8303520227807e004140f2c541183c7b0fd59de148f52688f network default NAD default] ADD starting CNI request [tsf/patch-tekton-config-56jxt c2700f7259d093e8303520227807e004140f2c541183c7b0fd59de148f52688f network default NAD default] I0327 15:20:58.064073 3815 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: patch-tekton-config-56jxt, hostIfaceName: c2700f7259d093e, network: default, NAD default, SandboxID: "c2700f7259d093e8303520227807e004140f2c541183c7b0fd59de148f52688f", PCI device ID: , UID: "a6509c77-4faf-45ed-96eb-d0eee07b7db5", MAC: 0a:58:0a:81:00:2e, IPs: [10.129.0.46/23] I0327 15:20:58.110921 3815 cni.go:424] [tsf/patch-tekton-config-56jxt c2700f7259d093e8303520227807e004140f2c541183c7b0fd59de148f52688f network default NAD default] ADD finished CNI request [tsf/patch-tekton-config-56jxt c2700f7259d093e8303520227807e004140f2c541183c7b0fd59de148f52688f network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:ef:5c:fe:cd:dc\",\"name\":\"c2700f7259d093e\"},{\"mac\":\"0a:58:0a:81:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d57682e5-8e69-466e-b2a2-03dec5170e67\"}],\"ips\":[{\"address\":\"10.129.0.46/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:21:05.352640 3815 cni.go:403] [tsf/patch-tekton-config-56jxt c2700f7259d093e8303520227807e004140f2c541183c7b0fd59de148f52688f network default NAD default] DEL starting CNI request [tsf/patch-tekton-config-56jxt c2700f7259d093e8303520227807e004140f2c541183c7b0fd59de148f52688f network default NAD default] I0327 15:21:05.424297 3815 cni.go:424] [tsf/patch-tekton-config-56jxt c2700f7259d093e8303520227807e004140f2c541183c7b0fd59de148f52688f network default NAD default] DEL finished CNI request [tsf/patch-tekton-config-56jxt c2700f7259d093e8303520227807e004140f2c541183c7b0fd59de148f52688f network default NAD default], result "{}", err I0327 15:21:05.478369 3815 obj_retry.go:464] Detected object tsf/patch-tekton-config-56jxt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:05.478412 3815 pods.go:176] Deleting pod: tsf/patch-tekton-config-56jxt I0327 15:21:05.479797 3815 pods.go:236] Attempting to release IPs for pod: tsf/patch-tekton-config-56jxt, ips: 10.129.0.46 I0327 15:21:05.479834 3815 obj_retry.go:464] Detected object tsf/patch-tekton-config-56jxt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:06.499058 3815 obj_retry.go:458] Detected object tsf/patch-tekton-config-56jxt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:06.499083 3815 obj_retry.go:458] Detected object tsf/patch-tekton-config-56jxt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:08.398066 3815 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-mzns5-sgqj4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:08.398666 3815 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-mzns5-sgqj4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:09.272371 3815 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-mzns5-sgqj4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:09.272404 3815 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-mzns5-sgqj4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:10.385958 3815 cni.go:403] [tsf-tas/ctlog-createtree-job-ms7lz-g4ghb be1e3a0acd26b8ad8ea87d3909ba635e03981a3604b524efe6aba5cfddce5d81 network default NAD default] DEL starting CNI request [tsf-tas/ctlog-createtree-job-ms7lz-g4ghb be1e3a0acd26b8ad8ea87d3909ba635e03981a3604b524efe6aba5cfddce5d81 network default NAD default] I0327 15:21:10.460277 3815 cni.go:424] [tsf-tas/ctlog-createtree-job-ms7lz-g4ghb be1e3a0acd26b8ad8ea87d3909ba635e03981a3604b524efe6aba5cfddce5d81 network default NAD default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-ms7lz-g4ghb be1e3a0acd26b8ad8ea87d3909ba635e03981a3604b524efe6aba5cfddce5d81 network default NAD default], result "{}", err I0327 15:21:10.512324 3815 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-ms7lz-g4ghb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:10.512366 3815 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-ms7lz-g4ghb I0327 15:21:10.513791 3815 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-ms7lz-g4ghb, ips: 10.129.0.44 I0327 15:21:10.513822 3815 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-ms7lz-g4ghb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:11.295224 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0327 15:21:11.477831 3815 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-ms7lz-g4ghb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:11.477854 3815 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-ms7lz-g4ghb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:11.973517 3815 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-ddbdb897d-8fzct] creating logical port tsf-tas_rekor-redis-ddbdb897d-8fzct for pod on switch ip-10-0-0-192.ec2.internal I0327 15:21:11.973683 3815 kube.go:256] Updating pod tsf-tas/rekor-redis-ddbdb897d-8fzct I0327 15:21:11.986665 3815 pod.go:62] [tsf-tas/rekor-redis-ddbdb897d-8fzct] pod update took 13.008615ms I0327 15:21:11.986759 3815 base_network_controller_pods.go:937] [default/tsf-tas/rekor-redis-ddbdb897d-8fzct] addLogicalPort annotation time took 13.109624ms I0327 15:21:11.988266 3815 pods.go:271] [tsf-tas/rekor-redis-ddbdb897d-8fzct] addLogicalPort took 14.772651ms, libovsdb time 971.54µs I0327 15:21:12.073900 3815 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-wn9mw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:12.074475 3815 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-wn9mw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:12.933192 3815 cni.go:403] [tsf-tas/rekor-redis-ddbdb897d-8fzct b2efab72ebf344532cb580c651a7187c459247f38b6d95089feb4a7d228c7705 network default NAD default] ADD starting CNI request [tsf-tas/rekor-redis-ddbdb897d-8fzct b2efab72ebf344532cb580c651a7187c459247f38b6d95089feb4a7d228c7705 network default NAD default] I0327 15:21:12.941895 3815 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-ddbdb897d-8fzct, hostIfaceName: b2efab72ebf3445, network: default, NAD default, SandboxID: "b2efab72ebf344532cb580c651a7187c459247f38b6d95089feb4a7d228c7705", PCI device ID: , UID: "0ab1ce3c-2556-40d9-b221-f06e1a17755e", MAC: 0a:58:0a:81:00:2f, IPs: [10.129.0.47/23] I0327 15:21:12.990500 3815 cni.go:424] [tsf-tas/rekor-redis-ddbdb897d-8fzct b2efab72ebf344532cb580c651a7187c459247f38b6d95089feb4a7d228c7705 network default NAD default] ADD finished CNI request [tsf-tas/rekor-redis-ddbdb897d-8fzct b2efab72ebf344532cb580c651a7187c459247f38b6d95089feb4a7d228c7705 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:d8:9a:e0:12:57\",\"name\":\"b2efab72ebf3445\"},{\"mac\":\"0a:58:0a:81:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/20d9e1ba-e22c-42c0-8948-ab2bae820da2\"}],\"ips\":[{\"address\":\"10.129.0.47/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:21:13.093550 3815 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-wn9mw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:13.093577 3815 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-wn9mw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:13.144008 3815 obj_retry.go:458] Detected object tsf/patch-tekton-config-56jxt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:13.144039 3815 obj_retry.go:458] Detected object tsf/patch-tekton-config-56jxt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:13.148138 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-56jxt I0327 15:21:13.148172 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-56jxt I0327 15:21:13.264612 3815 base_network_controller_pods.go:487] [default/tsf/test-tsf-pipelines] creating logical port tsf_test-tsf-pipelines for pod on switch ip-10-0-0-192.ec2.internal I0327 15:21:13.264703 3815 kube.go:256] Updating pod tsf/test-tsf-pipelines I0327 15:21:13.279387 3815 pod.go:62] [tsf/test-tsf-pipelines] pod update took 14.707208ms I0327 15:21:13.279414 3815 base_network_controller_pods.go:937] [default/tsf/test-tsf-pipelines] addLogicalPort annotation time took 14.740041ms I0327 15:21:13.280709 3815 pods.go:271] [tsf/test-tsf-pipelines] addLogicalPort took 16.135703ms, libovsdb time 851.012µs I0327 15:21:13.781175 3815 cni.go:403] [tsf/test-tsf-pipelines f2af1d5ae0750a05fe873f771142727d708664ca0139041ec6f9a1cafd38df6d network default NAD default] ADD starting CNI request [tsf/test-tsf-pipelines f2af1d5ae0750a05fe873f771142727d708664ca0139041ec6f9a1cafd38df6d network default NAD default] I0327 15:21:13.790223 3815 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-pipelines, hostIfaceName: f2af1d5ae0750a0, network: default, NAD default, SandboxID: "f2af1d5ae0750a05fe873f771142727d708664ca0139041ec6f9a1cafd38df6d", PCI device ID: , UID: "5c082298-6eb8-429a-af37-274109fd6eca", MAC: 0a:58:0a:81:00:30, IPs: [10.129.0.48/23] I0327 15:21:13.842035 3815 cni.go:424] [tsf/test-tsf-pipelines f2af1d5ae0750a05fe873f771142727d708664ca0139041ec6f9a1cafd38df6d network default NAD default] ADD finished CNI request [tsf/test-tsf-pipelines f2af1d5ae0750a05fe873f771142727d708664ca0139041ec6f9a1cafd38df6d network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:19:50:74:bc:62\",\"name\":\"f2af1d5ae0750a0\"},{\"mac\":\"0a:58:0a:81:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4d8d1a67-299a-4904-9688-a231b1021860\"}],\"ips\":[{\"address\":\"10.129.0.48/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:21:14.404863 3815 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0327 15:21:14.666077 3815 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-7cd75898c7-skpv4] creating logical port tsf-tas_rekor-server-7cd75898c7-skpv4 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:21:14.666178 3815 kube.go:256] Updating pod tsf-tas/rekor-server-7cd75898c7-skpv4 I0327 15:21:14.679233 3815 pod.go:62] [tsf-tas/rekor-server-7cd75898c7-skpv4] pod update took 13.078079ms I0327 15:21:14.679259 3815 base_network_controller_pods.go:937] [default/tsf-tas/rekor-server-7cd75898c7-skpv4] addLogicalPort annotation time took 13.111099ms I0327 15:21:14.680655 3815 pods.go:271] [tsf-tas/rekor-server-7cd75898c7-skpv4] addLogicalPort took 14.595996ms, libovsdb time 893.837µs W0327 15:21:15.966988 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0327 15:21:18.318755 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0327 15:21:24.471428 3815 cni.go:403] [tsf/test-tsf-pipelines f2af1d5ae0750a05fe873f771142727d708664ca0139041ec6f9a1cafd38df6d network default NAD default] DEL starting CNI request [tsf/test-tsf-pipelines f2af1d5ae0750a05fe873f771142727d708664ca0139041ec6f9a1cafd38df6d network default NAD default] I0327 15:21:24.551045 3815 cni.go:424] [tsf/test-tsf-pipelines f2af1d5ae0750a05fe873f771142727d708664ca0139041ec6f9a1cafd38df6d network default NAD default] DEL finished CNI request [tsf/test-tsf-pipelines f2af1d5ae0750a05fe873f771142727d708664ca0139041ec6f9a1cafd38df6d network default NAD default], result "{}", err I0327 15:21:24.605803 3815 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:24.605841 3815 pods.go:176] Deleting pod: tsf/test-tsf-pipelines I0327 15:21:24.607236 3815 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-pipelines, ips: 10.129.0.48 I0327 15:21:24.607266 3815 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:24.621367 3815 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:24.621463 3815 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:24.626858 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0327 15:21:24.626960 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0327 15:21:24.629695 3815 cni.go:403] [tsf-tas/rekor-server-7cd75898c7-skpv4 7545d24d05e60ead492176997ebdbb9c3667cf443fdc0a8a1572425e376300a1 network default NAD default] ADD starting CNI request [tsf-tas/rekor-server-7cd75898c7-skpv4 7545d24d05e60ead492176997ebdbb9c3667cf443fdc0a8a1572425e376300a1 network default NAD default] I0327 15:21:24.638710 3815 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-7cd75898c7-skpv4, hostIfaceName: 7545d24d05e60ea, network: default, NAD default, SandboxID: "7545d24d05e60ead492176997ebdbb9c3667cf443fdc0a8a1572425e376300a1", PCI device ID: , UID: "88a6d7fd-7d39-4a5a-8f98-1bde8ece5e95", MAC: 0a:58:0a:81:00:31, IPs: [10.129.0.49/23] I0327 15:21:24.688853 3815 cni.go:424] [tsf-tas/rekor-server-7cd75898c7-skpv4 7545d24d05e60ead492176997ebdbb9c3667cf443fdc0a8a1572425e376300a1 network default NAD default] ADD finished CNI request [tsf-tas/rekor-server-7cd75898c7-skpv4 7545d24d05e60ead492176997ebdbb9c3667cf443fdc0a8a1572425e376300a1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:88:7f:4a:3f:ef\",\"name\":\"7545d24d05e60ea\"},{\"mac\":\"0a:58:0a:81:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9404129a-d6d0-4793-a305-fac48435804a\"}],\"ips\":[{\"address\":\"10.129.0.49/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:21:26.453529 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis W0327 15:21:26.483053 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0327 15:21:29.158515 3815 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:29.159123 3815 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:29.178667 3815 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:29.178694 3815 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:29.183947 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0327 15:21:29.183977 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0327 15:21:35.209664 3815 obj_retry.go:464] Detected object tsf-tpa/create-db-rmnrb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:35.210233 3815 obj_retry.go:464] Detected object tsf-tpa/create-db-rmnrb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:36.230793 3815 obj_retry.go:458] Detected object tsf-tpa/create-db-rmnrb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:36.230819 3815 obj_retry.go:458] Detected object tsf-tpa/create-db-rmnrb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:36.325640 3815 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-fwvnf] creating logical port tsf-tpa_migrate-db-fwvnf for pod on switch ip-10-0-0-192.ec2.internal I0327 15:21:36.325736 3815 kube.go:256] Updating pod tsf-tpa/migrate-db-fwvnf I0327 15:21:36.337520 3815 pod.go:62] [tsf-tpa/migrate-db-fwvnf] pod update took 11.804859ms I0327 15:21:36.337550 3815 base_network_controller_pods.go:937] [default/tsf-tpa/migrate-db-fwvnf] addLogicalPort annotation time took 11.842048ms I0327 15:21:36.338879 3815 pods.go:271] [tsf-tpa/migrate-db-fwvnf] addLogicalPort took 13.258465ms, libovsdb time 889.78µs I0327 15:21:36.973912 3815 cni.go:403] [tsf-tpa/migrate-db-fwvnf 15a9bd6c5d7fa064bf3fd5cde38bfa9b0b0573a789a592e52b159c3726a2bf4e network default NAD default] ADD starting CNI request [tsf-tpa/migrate-db-fwvnf 15a9bd6c5d7fa064bf3fd5cde38bfa9b0b0573a789a592e52b159c3726a2bf4e network default NAD default] I0327 15:21:36.982536 3815 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-fwvnf, hostIfaceName: 15a9bd6c5d7fa06, network: default, NAD default, SandboxID: "15a9bd6c5d7fa064bf3fd5cde38bfa9b0b0573a789a592e52b159c3726a2bf4e", PCI device ID: , UID: "1d803092-0b29-4a46-9809-8a01a9ed84b4", MAC: 0a:58:0a:81:00:32, IPs: [10.129.0.50/23] I0327 15:21:37.035407 3815 cni.go:424] [tsf-tpa/migrate-db-fwvnf 15a9bd6c5d7fa064bf3fd5cde38bfa9b0b0573a789a592e52b159c3726a2bf4e network default NAD default] ADD finished CNI request [tsf-tpa/migrate-db-fwvnf 15a9bd6c5d7fa064bf3fd5cde38bfa9b0b0573a789a592e52b159c3726a2bf4e network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:77:1a:5d:07:1e\",\"name\":\"15a9bd6c5d7fa06\"},{\"mac\":\"0a:58:0a:81:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/933a2bfd-1294-4876-81d5-29cbf53f06b9\"}],\"ips\":[{\"address\":\"10.129.0.50/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:21:37.494806 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0327 15:21:41.577280 3815 cni.go:403] [tsf-tpa/migrate-db-fwvnf 15a9bd6c5d7fa064bf3fd5cde38bfa9b0b0573a789a592e52b159c3726a2bf4e network default NAD default] DEL starting CNI request [tsf-tpa/migrate-db-fwvnf 15a9bd6c5d7fa064bf3fd5cde38bfa9b0b0573a789a592e52b159c3726a2bf4e network default NAD default] I0327 15:21:41.667854 3815 cni.go:424] [tsf-tpa/migrate-db-fwvnf 15a9bd6c5d7fa064bf3fd5cde38bfa9b0b0573a789a592e52b159c3726a2bf4e network default NAD default] DEL finished CNI request [tsf-tpa/migrate-db-fwvnf 15a9bd6c5d7fa064bf3fd5cde38bfa9b0b0573a789a592e52b159c3726a2bf4e network default NAD default], result "{}", err I0327 15:21:41.718308 3815 obj_retry.go:464] Detected object tsf-tpa/migrate-db-fwvnf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:41.718343 3815 pods.go:176] Deleting pod: tsf-tpa/migrate-db-fwvnf I0327 15:21:41.719658 3815 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-fwvnf, ips: 10.129.0.50 I0327 15:21:41.719681 3815 obj_retry.go:464] Detected object tsf-tpa/migrate-db-fwvnf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:42.589831 3815 obj_retry.go:458] Detected object tsf-tpa/migrate-db-fwvnf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:42.589854 3815 obj_retry.go:458] Detected object tsf-tpa/migrate-db-fwvnf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:42.758322 3815 base_network_controller_pods.go:487] [default/tsf-tpa/create-importers-lnpnk] creating logical port tsf-tpa_create-importers-lnpnk for pod on switch ip-10-0-0-192.ec2.internal I0327 15:21:42.758448 3815 kube.go:256] Updating pod tsf-tpa/create-importers-lnpnk I0327 15:21:42.770615 3815 pod.go:62] [tsf-tpa/create-importers-lnpnk] pod update took 12.190547ms I0327 15:21:42.770645 3815 base_network_controller_pods.go:937] [default/tsf-tpa/create-importers-lnpnk] addLogicalPort annotation time took 12.224924ms I0327 15:21:42.772024 3815 pods.go:271] [tsf-tpa/create-importers-lnpnk] addLogicalPort took 13.71901ms, libovsdb time 887.472µs I0327 15:21:43.118457 3815 cni.go:403] [tsf-tpa/create-importers-lnpnk feba3d9b9d118b699e5358327ab4dafc9b2661dba1f9e4e7da23d516bd9137cd network default NAD default] ADD starting CNI request [tsf-tpa/create-importers-lnpnk feba3d9b9d118b699e5358327ab4dafc9b2661dba1f9e4e7da23d516bd9137cd network default NAD default] I0327 15:21:43.127251 3815 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-importers-lnpnk, hostIfaceName: feba3d9b9d118b6, network: default, NAD default, SandboxID: "feba3d9b9d118b699e5358327ab4dafc9b2661dba1f9e4e7da23d516bd9137cd", PCI device ID: , UID: "50ab0a4a-30cf-49ec-8c9c-7ac2f3f8c0a0", MAC: 0a:58:0a:81:00:33, IPs: [10.129.0.51/23] I0327 15:21:43.177227 3815 cni.go:424] [tsf-tpa/create-importers-lnpnk feba3d9b9d118b699e5358327ab4dafc9b2661dba1f9e4e7da23d516bd9137cd network default NAD default] ADD finished CNI request [tsf-tpa/create-importers-lnpnk feba3d9b9d118b699e5358327ab4dafc9b2661dba1f9e4e7da23d516bd9137cd network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:eb:9d:9a:f1:83\",\"name\":\"feba3d9b9d118b6\"},{\"mac\":\"0a:58:0a:81:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/086b2804-cbe2-4e31-b4d1-36419cd715e7\"}],\"ips\":[{\"address\":\"10.129.0.51/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:21:44.599265 3815 cni.go:403] [tsf-tpa/create-importers-lnpnk feba3d9b9d118b699e5358327ab4dafc9b2661dba1f9e4e7da23d516bd9137cd network default NAD default] DEL starting CNI request [tsf-tpa/create-importers-lnpnk feba3d9b9d118b699e5358327ab4dafc9b2661dba1f9e4e7da23d516bd9137cd network default NAD default] I0327 15:21:44.678166 3815 cni.go:424] [tsf-tpa/create-importers-lnpnk feba3d9b9d118b699e5358327ab4dafc9b2661dba1f9e4e7da23d516bd9137cd network default NAD default] DEL finished CNI request [tsf-tpa/create-importers-lnpnk feba3d9b9d118b699e5358327ab4dafc9b2661dba1f9e4e7da23d516bd9137cd network default NAD default], result "{}", err I0327 15:21:44.733025 3815 obj_retry.go:464] Detected object tsf-tpa/create-importers-lnpnk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:44.733064 3815 pods.go:176] Deleting pod: tsf-tpa/create-importers-lnpnk I0327 15:21:44.734370 3815 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-importers-lnpnk, ips: 10.129.0.51 I0327 15:21:44.734396 3815 obj_retry.go:464] Detected object tsf-tpa/create-importers-lnpnk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:45.759104 3815 obj_retry.go:458] Detected object tsf-tpa/create-importers-lnpnk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:45.759127 3815 obj_retry.go:458] Detected object tsf-tpa/create-importers-lnpnk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:50.321857 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0327 15:21:52.300869 3815 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-7fzwb-wm8mk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:52.301420 3815 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-7fzwb-wm8mk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:53.182980 3815 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-7fzwb-wm8mk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:53.183008 3815 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-7fzwb-wm8mk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:21:54.560737 3815 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0327 15:21:57.323409 3815 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:57.324065 3815 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:57.342755 3815 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:57.342777 3815 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:57.353684 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0327 15:21:57.353711 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0327 15:21:58.079073 3815 namespace.go:98] [build-service] adding namespace I0327 15:21:58.079100 3815 namespace.go:98] [image-controller] adding namespace I0327 15:21:58.080181 3815 namespace.go:102] [build-service] adding namespace took 1.084264ms I0327 15:21:58.080815 3815 namespace.go:102] [image-controller] adding namespace took 1.69918ms I0327 15:21:58.081025 3815 namespace.go:98] [integration-service] adding namespace I0327 15:21:58.081624 3815 namespace.go:102] [integration-service] adding namespace took 587.105µs I0327 15:21:58.125502 3815 namespace.go:142] [build-service] updating namespace I0327 15:21:58.151208 3815 namespace.go:142] [build-service] updating namespace I0327 15:21:58.155318 3815 namespace.go:142] [image-controller] updating namespace I0327 15:21:58.175159 3815 namespace.go:142] [image-controller] updating namespace I0327 15:21:58.180702 3815 namespace.go:142] [integration-service] updating namespace I0327 15:21:58.204935 3815 namespace.go:142] [integration-service] updating namespace I0327 15:21:58.282261 3815 namespace.go:142] [build-service] updating namespace I0327 15:21:58.306721 3815 namespace.go:142] [integration-service] updating namespace I0327 15:21:58.332466 3815 namespace.go:98] [release-service] adding namespace I0327 15:21:58.333326 3815 namespace.go:102] [release-service] adding namespace took 836.003µs I0327 15:21:58.351478 3815 base_network_controller_pods.go:487] [default/konflux-ui/tsf-konflux-public-key-nwst2] creating logical port konflux-ui_tsf-konflux-public-key-nwst2 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:21:58.351574 3815 kube.go:256] Updating pod konflux-ui/tsf-konflux-public-key-nwst2 I0327 15:21:58.359023 3815 namespace.go:142] [release-service] updating namespace I0327 15:21:58.375396 3815 namespace.go:142] [release-service] updating namespace I0327 15:21:58.402316 3815 pod.go:62] [konflux-ui/tsf-konflux-public-key-nwst2] pod update took 50.767624ms I0327 15:21:58.402343 3815 base_network_controller_pods.go:937] [default/konflux-ui/tsf-konflux-public-key-nwst2] addLogicalPort annotation time took 50.802065ms I0327 15:21:58.403632 3815 pods.go:271] [konflux-ui/tsf-konflux-public-key-nwst2] addLogicalPort took 52.173259ms, libovsdb time 848.169µs I0327 15:21:58.429628 3815 namespace.go:142] [konflux-ui] updating namespace I0327 15:21:58.498342 3815 namespace.go:98] [konflux-info] adding namespace I0327 15:21:58.499310 3815 namespace.go:102] [konflux-info] adding namespace took 939.05µs I0327 15:21:58.531140 3815 namespace.go:142] [konflux-info] updating namespace I0327 15:21:58.551369 3815 namespace.go:98] [namespace-lister] adding namespace I0327 15:21:58.552304 3815 namespace.go:102] [namespace-lister] adding namespace took 907.372µs I0327 15:21:58.565809 3815 namespace.go:142] [konflux-info] updating namespace I0327 15:21:58.592612 3815 namespace.go:142] [namespace-lister] updating namespace I0327 15:21:58.592713 3815 namespace.go:98] [enterprise-contract-service] adding namespace I0327 15:21:58.593553 3815 namespace.go:102] [enterprise-contract-service] adding namespace took 821.059µs I0327 15:21:58.656153 3815 namespace.go:142] [namespace-lister] updating namespace I0327 15:21:58.656222 3815 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:21:58.657875 3815 namespace.go:142] [image-controller] updating namespace I0327 15:21:58.721686 3815 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:21:58.848544 3815 namespace.go:98] [default-tenant] adding namespace I0327 15:21:58.849570 3815 namespace.go:102] [default-tenant] adding namespace took 999.528µs I0327 15:21:58.873112 3815 namespace.go:142] [build-service] updating namespace I0327 15:21:59.012699 3815 namespace.go:98] [segment-bridge] adding namespace I0327 15:21:59.012799 3815 namespace.go:142] [default-tenant] updating namespace I0327 15:21:59.013526 3815 namespace.go:102] [segment-bridge] adding namespace took 799.764µs I0327 15:21:59.078188 3815 namespace.go:142] [default-tenant] updating namespace I0327 15:21:59.078261 3815 namespace.go:142] [segment-bridge] updating namespace I0327 15:21:59.085460 3815 namespace.go:142] [image-controller] updating namespace I0327 15:21:59.230500 3815 namespace.go:142] [segment-bridge] updating namespace I0327 15:21:59.268208 3815 namespace.go:142] [integration-service] updating namespace I0327 15:21:59.270391 3815 cni.go:403] [konflux-ui/tsf-konflux-public-key-nwst2 f11f310930ca85c1474c814150762a523cf38dd7a6c552a7b34fd7b154b72b7f network default NAD default] ADD starting CNI request [konflux-ui/tsf-konflux-public-key-nwst2 f11f310930ca85c1474c814150762a523cf38dd7a6c552a7b34fd7b154b72b7f network default NAD default] I0327 15:21:59.278898 3815 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: tsf-konflux-public-key-nwst2, hostIfaceName: f11f310930ca85c, network: default, NAD default, SandboxID: "f11f310930ca85c1474c814150762a523cf38dd7a6c552a7b34fd7b154b72b7f", PCI device ID: , UID: "d0ec000a-75cd-40d7-9fa6-fcac16672e74", MAC: 0a:58:0a:81:00:34, IPs: [10.129.0.52/23] I0327 15:21:59.330696 3815 cni.go:424] [konflux-ui/tsf-konflux-public-key-nwst2 f11f310930ca85c1474c814150762a523cf38dd7a6c552a7b34fd7b154b72b7f network default NAD default] ADD finished CNI request [konflux-ui/tsf-konflux-public-key-nwst2 f11f310930ca85c1474c814150762a523cf38dd7a6c552a7b34fd7b154b72b7f network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:13:0a:d2:0a:67\",\"name\":\"f11f310930ca85c\"},{\"mac\":\"0a:58:0a:81:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/01fd9581-d8a0-470c-8254-38bd32811f26\"}],\"ips\":[{\"address\":\"10.129.0.52/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:21:59.492559 3815 namespace.go:142] [release-service] updating namespace I0327 15:21:59.618497 3815 namespace.go:142] [build-service] updating namespace I0327 15:21:59.618988 3815 namespace.go:142] [image-controller] updating namespace I0327 15:21:59.619038 3815 namespace.go:142] [integration-service] updating namespace I0327 15:21:59.621931 3815 namespace.go:142] [release-service] updating namespace I0327 15:21:59.681162 3815 namespace.go:142] [build-service] updating namespace I0327 15:21:59.709768 3815 namespace.go:142] [image-controller] updating namespace I0327 15:21:59.833733 3815 namespace.go:142] [integration-service] updating namespace I0327 15:21:59.903013 3815 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-67655c4674-hp67t] creating logical port namespace-lister_namespace-lister-67655c4674-hp67t for pod on switch ip-10-0-0-192.ec2.internal I0327 15:21:59.903114 3815 kube.go:256] Updating pod namespace-lister/namespace-lister-67655c4674-hp67t I0327 15:21:59.961969 3815 namespace.go:142] [release-service] updating namespace I0327 15:22:00.084317 3815 pod.go:62] [namespace-lister/namespace-lister-67655c4674-hp67t] pod update took 181.230841ms I0327 15:22:00.084345 3815 base_network_controller_pods.go:937] [default/namespace-lister/namespace-lister-67655c4674-hp67t] addLogicalPort annotation time took 181.264214ms I0327 15:22:00.085681 3815 pods.go:271] [namespace-lister/namespace-lister-67655c4674-hp67t] addLogicalPort took 182.686144ms, libovsdb time 875.65µs I0327 15:22:00.444459 3815 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0327 15:22:00.445147 3815 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0327 15:22:00.445166 3815 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0327 15:22:00.448064 3815 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0327 15:22:00.577841 3815 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0327 15:22:00.577879 3815 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] W0327 15:22:00.578091 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0327 15:22:00.579246 3815 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0327 15:22:01.091714 3815 namespace.go:142] [default-tenant] updating namespace I0327 15:22:01.137473 3815 base_network_controller_pods.go:487] [default/konflux-ui/dex-69fc7b7ccf-z9g5b] creating logical port konflux-ui_dex-69fc7b7ccf-z9g5b for pod on switch ip-10-0-0-192.ec2.internal I0327 15:22:01.137567 3815 kube.go:256] Updating pod konflux-ui/dex-69fc7b7ccf-z9g5b I0327 15:22:01.170081 3815 pod.go:62] [konflux-ui/dex-69fc7b7ccf-z9g5b] pod update took 32.53901ms I0327 15:22:01.170113 3815 base_network_controller_pods.go:937] [default/konflux-ui/dex-69fc7b7ccf-z9g5b] addLogicalPort annotation time took 32.574951ms I0327 15:22:01.171715 3815 pods.go:271] [konflux-ui/dex-69fc7b7ccf-z9g5b] addLogicalPort took 34.26057ms, libovsdb time 1.126177ms I0327 15:22:01.280210 3815 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx] creating logical port image-controller_image-controller-controller-manager-5f8cd64d55-ks4fx for pod on switch ip-10-0-0-192.ec2.internal I0327 15:22:01.280305 3815 kube.go:256] Updating pod image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx I0327 15:22:01.328991 3815 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:22:01.343581 3815 pod.go:62] [image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx] pod update took 63.301439ms I0327 15:22:01.343635 3815 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx] addLogicalPort annotation time took 63.363226ms I0327 15:22:01.344996 3815 pods.go:271] [image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx] addLogicalPort took 64.800963ms, libovsdb time 877.448µs I0327 15:22:01.355354 3815 base_network_controller_pods.go:487] [default/konflux-ui/proxy-5f5844ff9b-p58tn] creating logical port konflux-ui_proxy-5f5844ff9b-p58tn for pod on switch ip-10-0-0-192.ec2.internal I0327 15:22:01.355453 3815 kube.go:256] Updating pod konflux-ui/proxy-5f5844ff9b-p58tn I0327 15:22:01.422878 3815 pod.go:62] [konflux-ui/proxy-5f5844ff9b-p58tn] pod update took 67.459296ms I0327 15:22:01.422904 3815 base_network_controller_pods.go:937] [default/konflux-ui/proxy-5f5844ff9b-p58tn] addLogicalPort annotation time took 67.490288ms I0327 15:22:01.424280 3815 pods.go:271] [konflux-ui/proxy-5f5844ff9b-p58tn] addLogicalPort took 68.9448ms, libovsdb time 895.049µs I0327 15:22:01.522846 3815 namespace.go:142] [konflux-info] updating namespace I0327 15:22:01.600521 3815 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-759977d69f-d4bhv] creating logical port integration-service_integration-service-controller-manager-759977d69f-d4bhv for pod on switch ip-10-0-0-192.ec2.internal I0327 15:22:01.600658 3815 kube.go:256] Updating pod integration-service/integration-service-controller-manager-759977d69f-d4bhv I0327 15:22:01.622130 3815 pod.go:62] [integration-service/integration-service-controller-manager-759977d69f-d4bhv] pod update took 21.505768ms I0327 15:22:01.622163 3815 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-759977d69f-d4bhv] addLogicalPort annotation time took 21.546006ms I0327 15:22:01.624074 3815 pods.go:271] [integration-service/integration-service-controller-manager-759977d69f-d4bhv] addLogicalPort took 23.573852ms, libovsdb time 1.184599ms I0327 15:22:01.656418 3815 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx 4c081e4dfbac69a386ca02f8b5e790b1491fec6c8da2f3cc0416b7b615264069 network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx 4c081e4dfbac69a386ca02f8b5e790b1491fec6c8da2f3cc0416b7b615264069 network default NAD default] I0327 15:22:01.668078 3815 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-5f8cd64d55-ks4fx, hostIfaceName: 4c081e4dfbac69a, network: default, NAD default, SandboxID: "4c081e4dfbac69a386ca02f8b5e790b1491fec6c8da2f3cc0416b7b615264069", PCI device ID: , UID: "e9cf7595-9de6-4948-8517-094ca197bb8d", MAC: 0a:58:0a:81:00:37, IPs: [10.129.0.55/23] I0327 15:22:01.710040 3815 namespace.go:142] [namespace-lister] updating namespace I0327 15:22:01.725845 3815 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx 4c081e4dfbac69a386ca02f8b5e790b1491fec6c8da2f3cc0416b7b615264069 network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx 4c081e4dfbac69a386ca02f8b5e790b1491fec6c8da2f3cc0416b7b615264069 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:f9:fd:16:53:5b\",\"name\":\"4c081e4dfbac69a\"},{\"mac\":\"0a:58:0a:81:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/69a07a62-ba64-409f-b3d6-857944c338d7\"}],\"ips\":[{\"address\":\"10.129.0.55/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:22:01.787704 3815 cni.go:403] [namespace-lister/namespace-lister-67655c4674-hp67t c62ab27dea8fbaeb8eb59bf2b677112105c250fce7ecebb95f3a25ef0d81af22 network default NAD default] ADD starting CNI request [namespace-lister/namespace-lister-67655c4674-hp67t c62ab27dea8fbaeb8eb59bf2b677112105c250fce7ecebb95f3a25ef0d81af22 network default NAD default] I0327 15:22:01.797019 3815 helper_linux.go:450] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-67655c4674-hp67t, hostIfaceName: c62ab27dea8fbae, network: default, NAD default, SandboxID: "c62ab27dea8fbaeb8eb59bf2b677112105c250fce7ecebb95f3a25ef0d81af22", PCI device ID: , UID: "e3a2dcb6-1a36-441f-83de-fe56b23cfb65", MAC: 0a:58:0a:81:00:35, IPs: [10.129.0.53/23] I0327 15:22:01.907952 3815 namespace.go:142] [segment-bridge] updating namespace I0327 15:22:02.032317 3815 namespace.go:142] [default-tenant] updating namespace I0327 15:22:02.052336 3815 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-854f775895-ndk8t] creating logical port release-service_release-service-controller-manager-854f775895-ndk8t for pod on switch ip-10-0-0-192.ec2.internal I0327 15:22:02.052448 3815 kube.go:256] Updating pod release-service/release-service-controller-manager-854f775895-ndk8t I0327 15:22:02.056010 3815 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:22:02.057925 3815 cni.go:424] [namespace-lister/namespace-lister-67655c4674-hp67t c62ab27dea8fbaeb8eb59bf2b677112105c250fce7ecebb95f3a25ef0d81af22 network default NAD default] ADD finished CNI request [namespace-lister/namespace-lister-67655c4674-hp67t c62ab27dea8fbaeb8eb59bf2b677112105c250fce7ecebb95f3a25ef0d81af22 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:a2:ea:1b:f6:0b\",\"name\":\"c62ab27dea8fbae\"},{\"mac\":\"0a:58:0a:81:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/56f023a0-5f36-4dbc-a779-bbf574e562be\"}],\"ips\":[{\"address\":\"10.129.0.53/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:22:02.073089 3815 pod.go:62] [release-service/release-service-controller-manager-854f775895-ndk8t] pod update took 20.667619ms I0327 15:22:02.073151 3815 base_network_controller_pods.go:937] [default/release-service/release-service-controller-manager-854f775895-ndk8t] addLogicalPort annotation time took 20.736707ms I0327 15:22:02.074527 3815 pods.go:271] [release-service/release-service-controller-manager-854f775895-ndk8t] addLogicalPort took 22.211064ms, libovsdb time 856.196µs I0327 15:22:02.077792 3815 namespace.go:142] [konflux-info] updating namespace I0327 15:22:02.097537 3815 namespace.go:142] [namespace-lister] updating namespace I0327 15:22:02.112623 3815 namespace.go:142] [segment-bridge] updating namespace I0327 15:22:02.206702 3815 namespace.go:142] [default-tenant] updating namespace I0327 15:22:02.273796 3815 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:22:02.349039 3815 namespace.go:142] [konflux-info] updating namespace I0327 15:22:02.415554 3815 namespace.go:142] [namespace-lister] updating namespace I0327 15:22:02.467716 3815 namespace.go:142] [segment-bridge] updating namespace I0327 15:22:02.715125 3815 cni.go:403] [konflux-ui/tsf-konflux-public-key-nwst2 f11f310930ca85c1474c814150762a523cf38dd7a6c552a7b34fd7b154b72b7f network default NAD default] DEL starting CNI request [konflux-ui/tsf-konflux-public-key-nwst2 f11f310930ca85c1474c814150762a523cf38dd7a6c552a7b34fd7b154b72b7f network default NAD default] I0327 15:22:02.800789 3815 cni.go:424] [konflux-ui/tsf-konflux-public-key-nwst2 f11f310930ca85c1474c814150762a523cf38dd7a6c552a7b34fd7b154b72b7f network default NAD default] DEL finished CNI request [konflux-ui/tsf-konflux-public-key-nwst2 f11f310930ca85c1474c814150762a523cf38dd7a6c552a7b34fd7b154b72b7f network default NAD default], result "{}", err I0327 15:22:02.885398 3815 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-nwst2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:02.885435 3815 pods.go:176] Deleting pod: konflux-ui/tsf-konflux-public-key-nwst2 I0327 15:22:02.887025 3815 pods.go:236] Attempting to release IPs for pod: konflux-ui/tsf-konflux-public-key-nwst2, ips: 10.129.0.52 I0327 15:22:02.887056 3815 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-nwst2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:03.904458 3815 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-nwst2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:03.904517 3815 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-nwst2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:22:05.247801 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0327 15:22:05.685668 3815 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-d4bhv b9d4b9e8f4523971bdb52d776fa45e2f5586bfe75a741c4061a03f02ef57f067 network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-759977d69f-d4bhv b9d4b9e8f4523971bdb52d776fa45e2f5586bfe75a741c4061a03f02ef57f067 network default NAD default] I0327 15:22:05.694739 3815 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-759977d69f-d4bhv, hostIfaceName: b9d4b9e8f452397, network: default, NAD default, SandboxID: "b9d4b9e8f4523971bdb52d776fa45e2f5586bfe75a741c4061a03f02ef57f067", PCI device ID: , UID: "70879ca9-1378-4fc8-8072-d58f1c0aa1d3", MAC: 0a:58:0a:81:00:39, IPs: [10.129.0.57/23] I0327 15:22:05.747551 3815 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-d4bhv b9d4b9e8f4523971bdb52d776fa45e2f5586bfe75a741c4061a03f02ef57f067 network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-759977d69f-d4bhv b9d4b9e8f4523971bdb52d776fa45e2f5586bfe75a741c4061a03f02ef57f067 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:f5:8c:56:c8:07\",\"name\":\"b9d4b9e8f452397\"},{\"mac\":\"0a:58:0a:81:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0b7c481e-1462-4621-8891-9c92739e4ecc\"}],\"ips\":[{\"address\":\"10.129.0.57/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:22:06.744096 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0327 15:22:06.766658 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0327 15:22:06.793718 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister I0327 15:22:09.818255 3815 cni.go:403] [konflux-ui/dex-69fc7b7ccf-z9g5b fc73bc3490c44198da67487a36a397edf3b0c137d1f2c917f1c53d621649aafd network default NAD default] ADD starting CNI request [konflux-ui/dex-69fc7b7ccf-z9g5b fc73bc3490c44198da67487a36a397edf3b0c137d1f2c917f1c53d621649aafd network default NAD default] I0327 15:22:09.826971 3815 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: dex-69fc7b7ccf-z9g5b, hostIfaceName: fc73bc3490c4419, network: default, NAD default, SandboxID: "fc73bc3490c44198da67487a36a397edf3b0c137d1f2c917f1c53d621649aafd", PCI device ID: , UID: "275a3314-c8fa-4983-8e74-c83f4fe07898", MAC: 0a:58:0a:81:00:36, IPs: [10.129.0.54/23] I0327 15:22:09.880510 3815 cni.go:424] [konflux-ui/dex-69fc7b7ccf-z9g5b fc73bc3490c44198da67487a36a397edf3b0c137d1f2c917f1c53d621649aafd network default NAD default] ADD finished CNI request [konflux-ui/dex-69fc7b7ccf-z9g5b fc73bc3490c44198da67487a36a397edf3b0c137d1f2c917f1c53d621649aafd network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:b0:a4:a8:a5:a5\",\"name\":\"fc73bc3490c4419\"},{\"mac\":\"0a:58:0a:81:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1054dd72-80dc-4f2e-b253-a8092d0777d0\"}],\"ips\":[{\"address\":\"10.129.0.54/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:22:10.218601 3815 cni.go:403] [release-service/release-service-controller-manager-854f775895-ndk8t 38d8ee6554edc20b45047817834b3aac1ead3eb15272348566db7653f438b9f1 network default NAD default] ADD starting CNI request [release-service/release-service-controller-manager-854f775895-ndk8t 38d8ee6554edc20b45047817834b3aac1ead3eb15272348566db7653f438b9f1 network default NAD default] I0327 15:22:10.227569 3815 helper_linux.go:450] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-854f775895-ndk8t, hostIfaceName: 38d8ee6554edc20, network: default, NAD default, SandboxID: "38d8ee6554edc20b45047817834b3aac1ead3eb15272348566db7653f438b9f1", PCI device ID: , UID: "58feeadf-c228-43df-934c-6da42d86da82", MAC: 0a:58:0a:81:00:3a, IPs: [10.129.0.58/23] I0327 15:22:10.279529 3815 cni.go:424] [release-service/release-service-controller-manager-854f775895-ndk8t 38d8ee6554edc20b45047817834b3aac1ead3eb15272348566db7653f438b9f1 network default NAD default] ADD finished CNI request [release-service/release-service-controller-manager-854f775895-ndk8t 38d8ee6554edc20b45047817834b3aac1ead3eb15272348566db7653f438b9f1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:c1:cb:2c:db:14\",\"name\":\"38d8ee6554edc20\"},{\"mac\":\"0a:58:0a:81:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/66e53ea8-0f5f-439d-8230-c8d9c06bdef2\"}],\"ips\":[{\"address\":\"10.129.0.58/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:22:10.428331 3815 cni.go:403] [konflux-ui/proxy-5f5844ff9b-p58tn 020b3758749948fa32b5ca3579d737742e1436b46389a3900391a44ce9bef1c6 network default NAD default] ADD starting CNI request [konflux-ui/proxy-5f5844ff9b-p58tn 020b3758749948fa32b5ca3579d737742e1436b46389a3900391a44ce9bef1c6 network default NAD default] I0327 15:22:10.436834 3815 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-5f5844ff9b-p58tn, hostIfaceName: 020b3758749948f, network: default, NAD default, SandboxID: "020b3758749948fa32b5ca3579d737742e1436b46389a3900391a44ce9bef1c6", PCI device ID: , UID: "efd93dea-9577-4ccb-b566-343a38eaa5cc", MAC: 0a:58:0a:81:00:38, IPs: [10.129.0.56/23] I0327 15:22:10.488544 3815 cni.go:424] [konflux-ui/proxy-5f5844ff9b-p58tn 020b3758749948fa32b5ca3579d737742e1436b46389a3900391a44ce9bef1c6 network default NAD default] ADD finished CNI request [konflux-ui/proxy-5f5844ff9b-p58tn 020b3758749948fa32b5ca3579d737742e1436b46389a3900391a44ce9bef1c6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:01:af:6f:6d:d0\",\"name\":\"020b3758749948f\"},{\"mac\":\"0a:58:0a:81:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2f7a2650-2669-4567-8ade-07eca577f045\"}],\"ips\":[{\"address\":\"10.129.0.56/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 15:22:10.778047 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 15:22:10.778056 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 15:22:12.796916 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0327 15:22:12.796918 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0327 15:22:12.819665 3815 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0327 15:22:14.313868 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 15:22:14.313865 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service I0327 15:22:15.294818 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 W0327 15:22:16.889029 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:16.889053 3815 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0327 15:22:17.205206 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 W0327 15:22:17.754176 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0327 15:22:18.344278 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:18.344300 3815 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:21.790518 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 15:22:21.791730 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 15:22:23.802701 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service W0327 15:22:23.808600 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service I0327 15:22:25.502652 3815 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-d4bhv b9d4b9e8f4523971bdb52d776fa45e2f5586bfe75a741c4061a03f02ef57f067 network default NAD default] DEL starting CNI request [integration-service/integration-service-controller-manager-759977d69f-d4bhv b9d4b9e8f4523971bdb52d776fa45e2f5586bfe75a741c4061a03f02ef57f067 network default NAD default] I0327 15:22:25.580987 3815 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-d4bhv b9d4b9e8f4523971bdb52d776fa45e2f5586bfe75a741c4061a03f02ef57f067 network default NAD default] DEL finished CNI request [integration-service/integration-service-controller-manager-759977d69f-d4bhv b9d4b9e8f4523971bdb52d776fa45e2f5586bfe75a741c4061a03f02ef57f067 network default NAD default], result "{}", err I0327 15:22:25.638634 3815 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-d4bhv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:25.638681 3815 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-759977d69f-d4bhv I0327 15:22:25.640029 3815 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-759977d69f-d4bhv, ips: 10.129.0.57 I0327 15:22:25.640061 3815 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-d4bhv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:25.886815 3815 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-d4bhv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:25.886846 3815 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-d4bhv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:25.893272 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-d4bhv I0327 15:22:25.893303 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-d4bhv I0327 15:22:26.302290 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 W0327 15:22:27.837424 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex I0327 15:22:34.203852 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=68 W0327 15:22:54.901864 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 15:22:54.901887 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-192.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0327 15:23:04.326070 3815 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0327 15:23:28.197506 3815 cni.go:403] [konflux-ui/proxy-5f5844ff9b-p58tn 020b3758749948fa32b5ca3579d737742e1436b46389a3900391a44ce9bef1c6 network default NAD default] DEL starting CNI request [konflux-ui/proxy-5f5844ff9b-p58tn 020b3758749948fa32b5ca3579d737742e1436b46389a3900391a44ce9bef1c6 network default NAD default] I0327 15:23:28.287024 3815 cni.go:424] [konflux-ui/proxy-5f5844ff9b-p58tn 020b3758749948fa32b5ca3579d737742e1436b46389a3900391a44ce9bef1c6 network default NAD default] DEL finished CNI request [konflux-ui/proxy-5f5844ff9b-p58tn 020b3758749948fa32b5ca3579d737742e1436b46389a3900391a44ce9bef1c6 network default NAD default], result "{}", err I0327 15:23:28.352686 3815 obj_retry.go:464] Detected object konflux-ui/proxy-5f5844ff9b-p58tn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:28.352724 3815 pods.go:176] Deleting pod: konflux-ui/proxy-5f5844ff9b-p58tn I0327 15:23:28.354205 3815 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-5f5844ff9b-p58tn, ips: 10.129.0.56 I0327 15:23:28.354235 3815 obj_retry.go:464] Detected object konflux-ui/proxy-5f5844ff9b-p58tn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:29.273558 3815 obj_retry.go:458] Detected object konflux-ui/proxy-5f5844ff9b-p58tn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:29.273597 3815 obj_retry.go:458] Detected object konflux-ui/proxy-5f5844ff9b-p58tn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:29.279349 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-5f5844ff9b-p58tn I0327 15:23:29.279371 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-5f5844ff9b-p58tn I0327 15:23:35.887311 3815 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:35.887938 3815 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:35.922518 3815 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:35.922550 3815 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:35.925041 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0327 15:23:35.925065 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0327 15:23:45.307198 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0327 15:24:06.713676 3815 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd 135e7b93f29ebf2890b80ab89776016c9ba2c566e19fa669356832a9da23803c network default NAD default] DEL starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd 135e7b93f29ebf2890b80ab89776016c9ba2c566e19fa669356832a9da23803c network default NAD default] I0327 15:24:06.795605 3815 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd 135e7b93f29ebf2890b80ab89776016c9ba2c566e19fa669356832a9da23803c network default NAD default] DEL finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd 135e7b93f29ebf2890b80ab89776016c9ba2c566e19fa669356832a9da23803c network default NAD default], result "{}", err I0327 15:24:06.859472 3815 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:06.859528 3815 pods.go:176] Deleting pod: konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd I0327 15:24:06.860859 3815 pods.go:236] Attempting to release IPs for pod: konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd, ips: 10.129.0.31 I0327 15:24:06.860888 3815 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:06.901007 3815 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-56b684c87-xlrw5] creating logical port image-controller_image-controller-controller-manager-56b684c87-xlrw5 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:24:06.901123 3815 kube.go:256] Updating pod image-controller/image-controller-controller-manager-56b684c87-xlrw5 I0327 15:24:06.913603 3815 pod.go:62] [image-controller/image-controller-controller-manager-56b684c87-xlrw5] pod update took 12.488335ms I0327 15:24:06.913625 3815 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-56b684c87-xlrw5] addLogicalPort annotation time took 12.533019ms I0327 15:24:06.915028 3815 pods.go:271] [image-controller/image-controller-controller-manager-56b684c87-xlrw5] addLogicalPort took 14.033758ms, libovsdb time 926.767µs I0327 15:24:07.260788 3815 cni.go:403] [image-controller/image-controller-controller-manager-56b684c87-xlrw5 473e162babc68d713ca1dd5b1b7f929c0c00287144768874cb34eb3767d6acbe network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-56b684c87-xlrw5 473e162babc68d713ca1dd5b1b7f929c0c00287144768874cb34eb3767d6acbe network default NAD default] I0327 15:24:07.269155 3815 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-56b684c87-xlrw5, hostIfaceName: 473e162babc68d7, network: default, NAD default, SandboxID: "473e162babc68d713ca1dd5b1b7f929c0c00287144768874cb34eb3767d6acbe", PCI device ID: , UID: "85549191-8bdc-4c9a-8e00-3132e6aaf9c0", MAC: 0a:58:0a:81:00:3b, IPs: [10.129.0.59/23] I0327 15:24:07.316473 3815 cni.go:424] [image-controller/image-controller-controller-manager-56b684c87-xlrw5 473e162babc68d713ca1dd5b1b7f929c0c00287144768874cb34eb3767d6acbe network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-56b684c87-xlrw5 473e162babc68d713ca1dd5b1b7f929c0c00287144768874cb34eb3767d6acbe network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:2a:a7:78:3d:95\",\"name\":\"473e162babc68d7\"},{\"mac\":\"0a:58:0a:81:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6edcb99a-d741-45a3-b9b6-22e2a48ba1a9\"}],\"ips\":[{\"address\":\"10.129.0.59/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:24:07.491511 3815 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:07.491534 3815 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:07.498699 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd I0327 15:24:07.498723 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-mb9wd I0327 15:24:10.311604 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=11 I0327 15:24:20.708205 3815 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx 4c081e4dfbac69a386ca02f8b5e790b1491fec6c8da2f3cc0416b7b615264069 network default NAD default] DEL starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx 4c081e4dfbac69a386ca02f8b5e790b1491fec6c8da2f3cc0416b7b615264069 network default NAD default] I0327 15:24:20.799709 3815 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx 4c081e4dfbac69a386ca02f8b5e790b1491fec6c8da2f3cc0416b7b615264069 network default NAD default] DEL finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx 4c081e4dfbac69a386ca02f8b5e790b1491fec6c8da2f3cc0416b7b615264069 network default NAD default], result "{}", err I0327 15:24:20.853026 3815 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:20.853058 3815 pods.go:176] Deleting pod: image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx I0327 15:24:20.854361 3815 pods.go:236] Attempting to release IPs for pod: image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx, ips: 10.129.0.55 I0327 15:24:20.854384 3815 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:21.584958 3815 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:21.584982 3815 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:21.592619 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx I0327 15:24:21.592644 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-ks4fx I0327 15:24:51.204788 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=120 I0327 15:24:52.203254 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0327 15:25:05.206666 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=154 I0327 15:25:13.331183 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0327 15:26:34.299990 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=5 I0327 15:26:37.291370 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0327 15:26:47.329652 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0327 15:27:46.211915 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=405 I0327 15:29:07.305445 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0327 15:29:07.329421 3815 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0327 15:29:09.206750 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=35 I0327 15:29:37.325360 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0327 15:30:00.201879 3815 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29577090-2bzzx for pod on switch ip-10-0-0-192.ec2.internal I0327 15:30:00.201998 3815 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx I0327 15:30:00.251709 3815 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx I0327 15:30:00.265560 3815 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx] pod update took 63.584338ms I0327 15:30:00.265607 3815 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx] addLogicalPort annotation time took 63.61491ms I0327 15:30:00.266854 3815 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx] addLogicalPort took 64.990902ms, libovsdb time 838.32µs I0327 15:30:00.561880 3815 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx 117fc01e2c31b30663ba4a59d89940ff4ddd1c052fd590e45175b23d285bd774 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx 117fc01e2c31b30663ba4a59d89940ff4ddd1c052fd590e45175b23d285bd774 network default NAD default] I0327 15:30:00.570247 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29577090-2bzzx, hostIfaceName: 117fc01e2c31b30, network: default, NAD default, SandboxID: "117fc01e2c31b30663ba4a59d89940ff4ddd1c052fd590e45175b23d285bd774", PCI device ID: , UID: "533e2997-65c6-4e1d-8e09-564cc6539b18", MAC: 0a:58:0a:81:00:3c, IPs: [10.129.0.60/23] I0327 15:30:00.617954 3815 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx 117fc01e2c31b30663ba4a59d89940ff4ddd1c052fd590e45175b23d285bd774 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx 117fc01e2c31b30663ba4a59d89940ff4ddd1c052fd590e45175b23d285bd774 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:f7:b0:42:34:41\",\"name\":\"117fc01e2c31b30\"},{\"mac\":\"0a:58:0a:81:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3cf85d8e-e986-4b09-a852-a1c049bc1a28\"}],\"ips\":[{\"address\":\"10.129.0.60/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:30:04.514267 3815 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx 117fc01e2c31b30663ba4a59d89940ff4ddd1c052fd590e45175b23d285bd774 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx 117fc01e2c31b30663ba4a59d89940ff4ddd1c052fd590e45175b23d285bd774 network default NAD default] I0327 15:30:04.592862 3815 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx 117fc01e2c31b30663ba4a59d89940ff4ddd1c052fd590e45175b23d285bd774 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx 117fc01e2c31b30663ba4a59d89940ff4ddd1c052fd590e45175b23d285bd774 network default NAD default], result "{}", err I0327 15:30:04.645887 3815 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:30:04.645955 3815 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx I0327 15:30:04.647449 3815 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx, ips: 10.129.0.60 I0327 15:30:04.647476 3815 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:30:05.524206 3815 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:30:05.524236 3815 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-2bzzx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:30:31.208231 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=10 I0327 15:30:37.298117 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0327 15:31:07.310153 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0327 15:31:27.314744 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0327 15:31:36.060230 3815 obj_retry.go:458] Detected object tsf-tpa/create-db-rmnrb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:36.060271 3815 obj_retry.go:458] Detected object tsf-tpa/create-db-rmnrb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:36.064111 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-rmnrb I0327 15:31:36.064131 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-rmnrb I0327 15:31:40.205860 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0327 15:31:42.028954 3815 obj_retry.go:458] Detected object tsf-tpa/migrate-db-fwvnf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:42.029014 3815 obj_retry.go:458] Detected object tsf-tpa/migrate-db-fwvnf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:42.031753 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-fwvnf I0327 15:31:42.031782 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-fwvnf I0327 15:31:45.041641 3815 obj_retry.go:458] Detected object tsf-tpa/create-importers-lnpnk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:45.041700 3815 obj_retry.go:458] Detected object tsf-tpa/create-importers-lnpnk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:45.046144 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-lnpnk I0327 15:31:45.046186 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-lnpnk I0327 15:31:57.210187 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=8 I0327 15:32:47.334719 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0327 15:33:21.956618 3815 namespace.go:98] [default-tenant-managed] adding namespace I0327 15:33:21.957489 3815 namespace.go:102] [default-tenant-managed] adding namespace took 845.781µs I0327 15:33:22.000237 3815 namespace.go:142] [default-tenant-managed] updating namespace I0327 15:33:22.014841 3815 namespace.go:142] [default-tenant-managed] updating namespace I0327 15:33:22.964538 3815 namespace.go:142] [default-tenant-managed] updating namespace I0327 15:33:23.093188 3815 namespace.go:142] [default-tenant-managed] updating namespace I0327 15:33:23.194798 3815 namespace.go:142] [default-tenant-managed] updating namespace I0327 15:34:12.596919 3815 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-init-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:34:12.597108 3815 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod I0327 15:34:12.614487 3815 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod] pod update took 17.416574ms I0327 15:34:12.614513 3815 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod] addLogicalPort annotation time took 17.448966ms I0327 15:34:12.615808 3815 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod] addLogicalPort took 18.907064ms, libovsdb time 873.629µs I0327 15:34:12.960634 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod c1acc8409557667a1f5556d6677d2fcfaf16e90431a088a8ab8cfde9ac495b25 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod c1acc8409557667a1f5556d6677d2fcfaf16e90431a088a8ab8cfde9ac495b25 network default NAD default] I0327 15:34:12.969264 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-init-pod, hostIfaceName: c1acc8409557667, network: default, NAD default, SandboxID: "c1acc8409557667a1f5556d6677d2fcfaf16e90431a088a8ab8cfde9ac495b25", PCI device ID: , UID: "912382c1-d0da-4871-82c4-e161ce2bec1b", MAC: 0a:58:0a:81:00:3d, IPs: [10.129.0.61/23] I0327 15:34:13.018306 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod c1acc8409557667a1f5556d6677d2fcfaf16e90431a088a8ab8cfde9ac495b25 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod c1acc8409557667a1f5556d6677d2fcfaf16e90431a088a8ab8cfde9ac495b25 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:aa:1c:e6:db:a8\",\"name\":\"c1acc8409557667\"},{\"mac\":\"0a:58:0a:81:00:3d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8cfb1b5a-2c8d-41c6-ac4d-aa079d98c0b8\"}],\"ips\":[{\"address\":\"10.129.0.61/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:34:28.207375 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=12 I0327 15:34:28.332853 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0327 15:34:28.987438 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod c1acc8409557667a1f5556d6677d2fcfaf16e90431a088a8ab8cfde9ac495b25 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod c1acc8409557667a1f5556d6677d2fcfaf16e90431a088a8ab8cfde9ac495b25 network default NAD default] I0327 15:34:29.063642 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod c1acc8409557667a1f5556d6677d2fcfaf16e90431a088a8ab8cfde9ac495b25 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod c1acc8409557667a1f5556d6677d2fcfaf16e90431a088a8ab8cfde9ac495b25 network default NAD default], result "{}", err I0327 15:34:29.131605 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:34:29.131668 3815 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod I0327 15:34:29.132983 3815 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod, ips: 10.129.0.61 I0327 15:34:29.133012 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:34:31.268808 3815 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-clone-repository-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:34:31.268952 3815 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod I0327 15:34:31.293071 3815 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod] pod update took 24.149213ms I0327 15:34:31.293095 3815 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod] addLogicalPort annotation time took 24.178364ms I0327 15:34:31.294411 3815 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod] addLogicalPort took 25.618374ms, libovsdb time 919.809µs I0327 15:34:31.706066 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod 6f1205367bfbe78e58200f11c4ed0386dfd678b982225f82de852dffdecbc854 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod 6f1205367bfbe78e58200f11c4ed0386dfd678b982225f82de852dffdecbc854 network default NAD default] I0327 15:34:31.714823 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-clone-repository-pod, hostIfaceName: 6f1205367bfbe78, network: default, NAD default, SandboxID: "6f1205367bfbe78e58200f11c4ed0386dfd678b982225f82de852dffdecbc854", PCI device ID: , UID: "b86a04ac-de2c-4178-acce-317fa24921cb", MAC: 0a:58:0a:81:00:3e, IPs: [10.129.0.62/23] I0327 15:34:31.763534 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod 6f1205367bfbe78e58200f11c4ed0386dfd678b982225f82de852dffdecbc854 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod 6f1205367bfbe78e58200f11c4ed0386dfd678b982225f82de852dffdecbc854 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:77:c5:c3:4c:75\",\"name\":\"6f1205367bfbe78\"},{\"mac\":\"0a:58:0a:81:00:3e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/96324f4f-7785-4a21-9a45-4abc221da035\"}],\"ips\":[{\"address\":\"10.129.0.62/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:34:57.168214 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod 6f1205367bfbe78e58200f11c4ed0386dfd678b982225f82de852dffdecbc854 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod 6f1205367bfbe78e58200f11c4ed0386dfd678b982225f82de852dffdecbc854 network default NAD default] I0327 15:34:57.247922 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod 6f1205367bfbe78e58200f11c4ed0386dfd678b982225f82de852dffdecbc854 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod 6f1205367bfbe78e58200f11c4ed0386dfd678b982225f82de852dffdecbc854 network default NAD default], result "{}", err I0327 15:34:57.323285 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:34:57.323342 3815 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod I0327 15:34:57.324830 3815 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod, ips: 10.129.0.62 I0327 15:34:57.324858 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:34:57.423627 3815 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:34:57.423752 3815 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod I0327 15:34:57.458618 3815 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod] pod update took 34.905842ms I0327 15:34:57.458641 3815 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod] addLogicalPort annotation time took 34.934329ms I0327 15:34:57.460036 3815 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod] addLogicalPort took 36.424156ms, libovsdb time 940.681µs I0327 15:34:57.786457 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod 367c0fe526db4fb601310e552c1065d506c9261ef869d936e00c417e9efda9e5 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod 367c0fe526db4fb601310e552c1065d506c9261ef869d936e00c417e9efda9e5 network default NAD default] I0327 15:34:57.795552 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod, hostIfaceName: 367c0fe526db4fb, network: default, NAD default, SandboxID: "367c0fe526db4fb601310e552c1065d506c9261ef869d936e00c417e9efda9e5", PCI device ID: , UID: "83dbeb01-ae1f-4fa7-a476-b6dc1f66d8a3", MAC: 0a:58:0a:81:00:3f, IPs: [10.129.0.63/23] I0327 15:34:57.839982 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod 367c0fe526db4fb601310e552c1065d506c9261ef869d936e00c417e9efda9e5 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod 367c0fe526db4fb601310e552c1065d506c9261ef869d936e00c417e9efda9e5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:0e:98:0e:49:70\",\"name\":\"367c0fe526db4fb\"},{\"mac\":\"0a:58:0a:81:00:3f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/10fdcef8-0840-4938-9a2a-aff75de0feea\"}],\"ips\":[{\"address\":\"10.129.0.63/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:35:08.215134 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=61 I0327 15:35:14.303088 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0327 15:35:20.345952 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod 367c0fe526db4fb601310e552c1065d506c9261ef869d936e00c417e9efda9e5 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod 367c0fe526db4fb601310e552c1065d506c9261ef869d936e00c417e9efda9e5 network default NAD default] I0327 15:35:20.419001 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod 367c0fe526db4fb601310e552c1065d506c9261ef869d936e00c417e9efda9e5 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod 367c0fe526db4fb601310e552c1065d506c9261ef869d936e00c417e9efda9e5 network default NAD default], result "{}", err I0327 15:35:20.541176 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:35:20.541211 3815 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod I0327 15:35:20.542642 3815 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod, ips: 10.129.0.63 I0327 15:35:20.542673 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:35:21.334178 3815 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-build-container-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:35:21.334334 3815 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod I0327 15:35:21.385807 3815 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod] pod update took 51.511978ms I0327 15:35:21.385839 3815 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod] addLogicalPort annotation time took 51.548909ms I0327 15:35:21.387285 3815 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod] addLogicalPort took 53.118254ms, libovsdb time 886.327µs I0327 15:35:21.705030 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod 45dfce04ebb0bc0b1328539f731e449c0680189ef0b5625d57960dad0f00fd7c network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod 45dfce04ebb0bc0b1328539f731e449c0680189ef0b5625d57960dad0f00fd7c network default NAD default] I0327 15:35:21.713537 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-build-container-pod, hostIfaceName: 45dfce04ebb0bc0, network: default, NAD default, SandboxID: "45dfce04ebb0bc0b1328539f731e449c0680189ef0b5625d57960dad0f00fd7c", PCI device ID: , UID: "83f8c6bf-2226-4ec9-8e4e-adda18e7ce69", MAC: 0a:58:0a:81:00:40, IPs: [10.129.0.64/23] I0327 15:35:21.762244 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod 45dfce04ebb0bc0b1328539f731e449c0680189ef0b5625d57960dad0f00fd7c network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod 45dfce04ebb0bc0b1328539f731e449c0680189ef0b5625d57960dad0f00fd7c network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:a6:04:2e:f3:f4\",\"name\":\"45dfce04ebb0bc0\"},{\"mac\":\"0a:58:0a:81:00:40\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f971cbf4-f7bf-46b4-ad95-ba676d652004\"}],\"ips\":[{\"address\":\"10.129.0.64/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:35:49.294525 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=11 I0327 15:36:00.332467 3815 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0327 15:36:15.209717 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=40 I0327 15:37:29.308833 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0327 15:37:35.318177 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0327 15:37:37.287706 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod 45dfce04ebb0bc0b1328539f731e449c0680189ef0b5625d57960dad0f00fd7c network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod 45dfce04ebb0bc0b1328539f731e449c0680189ef0b5625d57960dad0f00fd7c network default NAD default] I0327 15:37:37.379193 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod 45dfce04ebb0bc0b1328539f731e449c0680189ef0b5625d57960dad0f00fd7c network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod 45dfce04ebb0bc0b1328539f731e449c0680189ef0b5625d57960dad0f00fd7c network default NAD default], result "{}", err I0327 15:37:37.502947 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:37:37.502980 3815 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod I0327 15:37:37.504517 3815 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod, ips: 10.129.0.64 I0327 15:37:37.504546 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:38:03.213228 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=14 I0327 15:38:07.328018 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0327 15:38:20.801166 3815 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:38:20.801328 3815 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod I0327 15:38:20.823098 3815 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod] pod update took 21.800925ms I0327 15:38:20.823127 3815 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod] addLogicalPort annotation time took 21.832429ms I0327 15:38:20.824406 3815 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod] addLogicalPort took 23.266518ms, libovsdb time 857.936µs I0327 15:38:20.862285 3815 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-clair-scan-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:38:20.862495 3815 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod I0327 15:38:20.893799 3815 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod] pod update took 31.355257ms I0327 15:38:20.893830 3815 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod] addLogicalPort annotation time took 31.389453ms I0327 15:38:20.895171 3815 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod] addLogicalPort took 32.91703ms, libovsdb time 868.108µs I0327 15:38:21.163796 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod b72875eb4c9fb9c0c0c84aeb376e20fd3f459c6675b9d88e814df0c67cf1ebc4 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod b72875eb4c9fb9c0c0c84aeb376e20fd3f459c6675b9d88e814df0c67cf1ebc4 network default NAD default] I0327 15:38:21.172676 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod, hostIfaceName: b72875eb4c9fb9c, network: default, NAD default, SandboxID: "b72875eb4c9fb9c0c0c84aeb376e20fd3f459c6675b9d88e814df0c67cf1ebc4", PCI device ID: , UID: "7db5e25b-2161-4535-8105-d05989998143", MAC: 0a:58:0a:81:00:41, IPs: [10.129.0.65/23] I0327 15:38:21.223994 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod b72875eb4c9fb9c0c0c84aeb376e20fd3f459c6675b9d88e814df0c67cf1ebc4 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod b72875eb4c9fb9c0c0c84aeb376e20fd3f459c6675b9d88e814df0c67cf1ebc4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:ba:9e:ee:68:28\",\"name\":\"b72875eb4c9fb9c\"},{\"mac\":\"0a:58:0a:81:00:41\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/899ef511-78ff-4d8e-ba4a-41f2f2963100\"}],\"ips\":[{\"address\":\"10.129.0.65/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:38:21.235905 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod fee1c3c092ce3579351d0cdd641ecdfe9387121b8d43396475795ede551bd32f network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod fee1c3c092ce3579351d0cdd641ecdfe9387121b8d43396475795ede551bd32f network default NAD default] I0327 15:38:21.244323 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-clair-scan-pod, hostIfaceName: fee1c3c092ce357, network: default, NAD default, SandboxID: "fee1c3c092ce3579351d0cdd641ecdfe9387121b8d43396475795ede551bd32f", PCI device ID: , UID: "9a398a92-a118-4b4f-85c4-b1e664c841fe", MAC: 0a:58:0a:81:00:42, IPs: [10.129.0.66/23] I0327 15:38:21.293823 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod fee1c3c092ce3579351d0cdd641ecdfe9387121b8d43396475795ede551bd32f network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod fee1c3c092ce3579351d0cdd641ecdfe9387121b8d43396475795ede551bd32f network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:f3:69:0e:51:d4\",\"name\":\"fee1c3c092ce357\"},{\"mac\":\"0a:58:0a:81:00:42\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a9f1de7a-04ba-4dd4-b74e-6b954c8d5624\"}],\"ips\":[{\"address\":\"10.129.0.66/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:38:21.416631 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:38:21.417168 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:38:21.694174 3815 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod] creating logical port default-tenant_tsf-comp-apkw-on-push-mp89z-clamav-scan-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:38:21.694339 3815 kube.go:256] Updating pod default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod I0327 15:38:21.716972 3815 pod.go:62] [default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod] pod update took 22.666506ms I0327 15:38:21.717000 3815 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod] addLogicalPort annotation time took 22.697648ms I0327 15:38:21.718456 3815 pods.go:271] [default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod] addLogicalPort took 24.298296ms, libovsdb time 1.000475ms I0327 15:38:22.105286 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod 35f9aa7bcc9efddc5870296135a0be96b52b5e58d4baad4c9d26babaa673c6f7 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod 35f9aa7bcc9efddc5870296135a0be96b52b5e58d4baad4c9d26babaa673c6f7 network default NAD default] I0327 15:38:22.114217 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-apkw-on-push-mp89z-clamav-scan-pod, hostIfaceName: 35f9aa7bcc9efdd, network: default, NAD default, SandboxID: "35f9aa7bcc9efddc5870296135a0be96b52b5e58d4baad4c9d26babaa673c6f7", PCI device ID: , UID: "04b1220f-75f6-4654-be00-43b2499b6aca", MAC: 0a:58:0a:81:00:43, IPs: [10.129.0.67/23] I0327 15:38:22.161476 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod 35f9aa7bcc9efddc5870296135a0be96b52b5e58d4baad4c9d26babaa673c6f7 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod 35f9aa7bcc9efddc5870296135a0be96b52b5e58d4baad4c9d26babaa673c6f7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:a3:28:d7:1e:85\",\"name\":\"35f9aa7bcc9efdd\"},{\"mac\":\"0a:58:0a:81:00:43\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6203a6ea-1600-4e9d-b79d-e63cb5a9b754\"}],\"ips\":[{\"address\":\"10.129.0.67/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:38:42.810257 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-apply-tags-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:38:42.810881 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-apply-tags-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:38:47.313629 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0327 15:39:03.818715 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:03.819402 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-coverity-availability-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:04.956092 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:04.956720 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-sast-snyk-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:21.081452 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod b72875eb4c9fb9c0c0c84aeb376e20fd3f459c6675b9d88e814df0c67cf1ebc4 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod b72875eb4c9fb9c0c0c84aeb376e20fd3f459c6675b9d88e814df0c67cf1ebc4 network default NAD default] I0327 15:39:21.178506 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod b72875eb4c9fb9c0c0c84aeb376e20fd3f459c6675b9d88e814df0c67cf1ebc4 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod b72875eb4c9fb9c0c0c84aeb376e20fd3f459c6675b9d88e814df0c67cf1ebc4 network default NAD default], result "{}", err I0327 15:39:21.286091 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:21.286200 3815 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod I0327 15:39:21.287815 3815 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod, ips: 10.129.0.65 I0327 15:39:21.287848 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:22.011242 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:22.011884 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:22.090261 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod 35f9aa7bcc9efddc5870296135a0be96b52b5e58d4baad4c9d26babaa673c6f7 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod 35f9aa7bcc9efddc5870296135a0be96b52b5e58d4baad4c9d26babaa673c6f7 network default NAD default] I0327 15:39:22.183068 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod 35f9aa7bcc9efddc5870296135a0be96b52b5e58d4baad4c9d26babaa673c6f7 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod 35f9aa7bcc9efddc5870296135a0be96b52b5e58d4baad4c9d26babaa673c6f7 network default NAD default], result "{}", err I0327 15:39:22.283713 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:22.283761 3815 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod I0327 15:39:22.285355 3815 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod, ips: 10.129.0.67 I0327 15:39:22.285387 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:25.212001 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=10 I0327 15:39:35.054138 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:35.054773 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:51.122118 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:51.122705 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-push-dockerfile-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:52.173119 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:52.173757 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:39:53.301551 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0327 15:40:00.193490 3815 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29577100-p5ddg for pod on switch ip-10-0-0-192.ec2.internal I0327 15:40:00.193608 3815 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg I0327 15:40:00.209297 3815 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg] pod update took 15.729078ms I0327 15:40:00.209324 3815 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg] addLogicalPort annotation time took 15.759183ms I0327 15:40:00.210663 3815 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg] addLogicalPort took 17.195169ms, libovsdb time 851.697µs I0327 15:40:00.287040 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:40:00.287610 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-ecosystem-cert-preflight-checks-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:40:00.625571 3815 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg c563e73294837aefaaed80918ade95f8ddc8a5c5b4990100f8ff2c896bf13c16 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg c563e73294837aefaaed80918ade95f8ddc8a5c5b4990100f8ff2c896bf13c16 network default NAD default] I0327 15:40:00.634161 3815 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29577100-p5ddg, hostIfaceName: c563e73294837ae, network: default, NAD default, SandboxID: "c563e73294837aefaaed80918ade95f8ddc8a5c5b4990100f8ff2c896bf13c16", PCI device ID: , UID: "fa1c19ca-f699-4672-9f5c-a1644bc13bb9", MAC: 0a:58:0a:81:00:44, IPs: [10.129.0.68/23] I0327 15:40:00.682996 3815 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg c563e73294837aefaaed80918ade95f8ddc8a5c5b4990100f8ff2c896bf13c16 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg c563e73294837aefaaed80918ade95f8ddc8a5c5b4990100f8ff2c896bf13c16 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:b1:7b:c8:eb:2e\",\"name\":\"c563e73294837ae\"},{\"mac\":\"0a:58:0a:81:00:44\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e4af6832-1f56-42fd-a241-d05e7d111359\"}],\"ips\":[{\"address\":\"10.129.0.68/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:40:04.402121 3815 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg c563e73294837aefaaed80918ade95f8ddc8a5c5b4990100f8ff2c896bf13c16 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg c563e73294837aefaaed80918ade95f8ddc8a5c5b4990100f8ff2c896bf13c16 network default NAD default] I0327 15:40:04.494153 3815 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg c563e73294837aefaaed80918ade95f8ddc8a5c5b4990100f8ff2c896bf13c16 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg c563e73294837aefaaed80918ade95f8ddc8a5c5b4990100f8ff2c896bf13c16 network default NAD default], result "{}", err I0327 15:40:04.572051 3815 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:40:04.572092 3815 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg I0327 15:40:04.573419 3815 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg, ips: 10.129.0.68 I0327 15:40:04.573450 3815 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:40:05.407707 3815 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:40:05.407737 3815 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577100-p5ddg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:40:05.446542 3815 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:40:05.446567 3815 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:40:05.450213 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 I0327 15:40:05.450239 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-zrrj6 I0327 15:40:19.212200 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0327 15:41:05.337990 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0327 15:41:42.072073 3815 cni.go:403] [default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod fee1c3c092ce3579351d0cdd641ecdfe9387121b8d43396475795ede551bd32f network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod fee1c3c092ce3579351d0cdd641ecdfe9387121b8d43396475795ede551bd32f network default NAD default] I0327 15:41:42.160731 3815 cni.go:424] [default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod fee1c3c092ce3579351d0cdd641ecdfe9387121b8d43396475795ede551bd32f network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod fee1c3c092ce3579351d0cdd641ecdfe9387121b8d43396475795ede551bd32f network default NAD default], result "{}", err I0327 15:41:42.245711 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:41:42.245746 3815 pods.go:176] Deleting pod: default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod I0327 15:41:42.247360 3815 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod, ips: 10.129.0.66 I0327 15:41:42.247395 3815 obj_retry.go:464] Detected object default-tenant/tsf-comp-apkw-on-push-mp89z-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:42:10.068082 3815 obj_retry.go:464] Detected object default-tenant/my-integration-test-fxdi-x94ws-verify-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:42:10.068632 3815 obj_retry.go:464] Detected object default-tenant/my-integration-test-fxdi-x94ws-verify-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:42:22.613867 3815 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod] creating logical port default-tenant-managed_managed-fmmtf-verify-access-to-resources-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:42:22.614013 3815 kube.go:256] Updating pod default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod I0327 15:42:22.634203 3815 pod.go:62] [default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod] pod update took 20.217097ms I0327 15:42:22.634291 3815 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod] addLogicalPort annotation time took 20.310437ms I0327 15:42:22.635689 3815 pods.go:271] [default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod] addLogicalPort took 21.838318ms, libovsdb time 991.973µs I0327 15:42:22.980511 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod 4c20c41bbda9a6b27efce015d7ace22793ebb263170a5053fb5c1a5afb058333 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod 4c20c41bbda9a6b27efce015d7ace22793ebb263170a5053fb5c1a5afb058333 network default NAD default] I0327 15:42:22.989235 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-fmmtf-verify-access-to-resources-pod, hostIfaceName: 4c20c41bbda9a6b, network: default, NAD default, SandboxID: "4c20c41bbda9a6b27efce015d7ace22793ebb263170a5053fb5c1a5afb058333", PCI device ID: , UID: "263eadbe-add0-4beb-99e6-5899899a939f", MAC: 0a:58:0a:81:00:45, IPs: [10.129.0.69/23] I0327 15:42:23.039765 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod 4c20c41bbda9a6b27efce015d7ace22793ebb263170a5053fb5c1a5afb058333 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod 4c20c41bbda9a6b27efce015d7ace22793ebb263170a5053fb5c1a5afb058333 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:03:46:c2:95:d3\",\"name\":\"4c20c41bbda9a6b\"},{\"mac\":\"0a:58:0a:81:00:45\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eef76028-f1f9-460a-8397-d13d377a13b3\"}],\"ips\":[{\"address\":\"10.129.0.69/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:42:25.481578 3815 base_network_controller_pods.go:487] [default/default-tenant-managed/affinity-assistant-0426a73a44-0] creating logical port default-tenant-managed_affinity-assistant-0426a73a44-0 for pod on switch ip-10-0-0-192.ec2.internal I0327 15:42:25.481761 3815 kube.go:256] Updating pod default-tenant-managed/affinity-assistant-0426a73a44-0 I0327 15:42:25.494932 3815 pod.go:62] [default-tenant-managed/affinity-assistant-0426a73a44-0] pod update took 13.206269ms I0327 15:42:25.495020 3815 base_network_controller_pods.go:937] [default/default-tenant-managed/affinity-assistant-0426a73a44-0] addLogicalPort annotation time took 13.300103ms I0327 15:42:25.496841 3815 pods.go:271] [default-tenant-managed/affinity-assistant-0426a73a44-0] addLogicalPort took 15.287521ms, libovsdb time 1.246873ms I0327 15:42:25.852229 3815 cni.go:403] [default-tenant-managed/affinity-assistant-0426a73a44-0 9167622677b9051ebed7c17b6c5158f5e671255376619d253d71998cbee4d270 network default NAD default] ADD starting CNI request [default-tenant-managed/affinity-assistant-0426a73a44-0 9167622677b9051ebed7c17b6c5158f5e671255376619d253d71998cbee4d270 network default NAD default] I0327 15:42:25.862420 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: affinity-assistant-0426a73a44-0, hostIfaceName: 9167622677b9051, network: default, NAD default, SandboxID: "9167622677b9051ebed7c17b6c5158f5e671255376619d253d71998cbee4d270", PCI device ID: , UID: "a17524aa-0e64-47f3-8c1f-a09f84cf985d", MAC: 0a:58:0a:81:00:46, IPs: [10.129.0.70/23] I0327 15:42:25.922296 3815 cni.go:424] [default-tenant-managed/affinity-assistant-0426a73a44-0 9167622677b9051ebed7c17b6c5158f5e671255376619d253d71998cbee4d270 network default NAD default] ADD finished CNI request [default-tenant-managed/affinity-assistant-0426a73a44-0 9167622677b9051ebed7c17b6c5158f5e671255376619d253d71998cbee4d270 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:d5:8f:3e:69:5c\",\"name\":\"9167622677b9051\"},{\"mac\":\"0a:58:0a:81:00:46\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/74a2790a-fead-4b17-99f7-83eee7eebf06\"}],\"ips\":[{\"address\":\"10.129.0.70/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:42:35.312307 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0327 15:42:40.489766 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod 4c20c41bbda9a6b27efce015d7ace22793ebb263170a5053fb5c1a5afb058333 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod 4c20c41bbda9a6b27efce015d7ace22793ebb263170a5053fb5c1a5afb058333 network default NAD default] I0327 15:42:40.570597 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod 4c20c41bbda9a6b27efce015d7ace22793ebb263170a5053fb5c1a5afb058333 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod 4c20c41bbda9a6b27efce015d7ace22793ebb263170a5053fb5c1a5afb058333 network default NAD default], result "{}", err I0327 15:42:40.713221 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:42:40.713255 3815 pods.go:176] Deleting pod: default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod I0327 15:42:40.714638 3815 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod, ips: 10.129.0.69 I0327 15:42:40.714669 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-verify-access-to-resources-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:42:42.206075 3815 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-fmmtf-collect-data-pod] creating logical port default-tenant-managed_managed-fmmtf-collect-data-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:42:42.206227 3815 kube.go:256] Updating pod default-tenant-managed/managed-fmmtf-collect-data-pod I0327 15:42:42.232626 3815 pod.go:62] [default-tenant-managed/managed-fmmtf-collect-data-pod] pod update took 26.431186ms I0327 15:42:42.232648 3815 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-fmmtf-collect-data-pod] addLogicalPort annotation time took 26.460028ms I0327 15:42:42.234058 3815 pods.go:271] [default-tenant-managed/managed-fmmtf-collect-data-pod] addLogicalPort took 28.025792ms, libovsdb time 1.011661ms I0327 15:42:42.698453 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-collect-data-pod eeabe6324ef9255c672d108f7dc7e157afd57b51de40d304174070a222cd42a6 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-fmmtf-collect-data-pod eeabe6324ef9255c672d108f7dc7e157afd57b51de40d304174070a222cd42a6 network default NAD default] I0327 15:42:42.707453 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-fmmtf-collect-data-pod, hostIfaceName: eeabe6324ef9255, network: default, NAD default, SandboxID: "eeabe6324ef9255c672d108f7dc7e157afd57b51de40d304174070a222cd42a6", PCI device ID: , UID: "0d51f8cd-ff48-40ea-9d45-2683401b52d1", MAC: 0a:58:0a:81:00:47, IPs: [10.129.0.71/23] I0327 15:42:42.756190 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-collect-data-pod eeabe6324ef9255c672d108f7dc7e157afd57b51de40d304174070a222cd42a6 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-fmmtf-collect-data-pod eeabe6324ef9255c672d108f7dc7e157afd57b51de40d304174070a222cd42a6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:28:a5:0a:29:4e\",\"name\":\"eeabe6324ef9255\"},{\"mac\":\"0a:58:0a:81:00:47\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cc3948dc-c0a2-47bc-bb42-436aab1bd16e\"}],\"ips\":[{\"address\":\"10.129.0.71/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:43:01.218949 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=224 I0327 15:43:06.688387 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-collect-data-pod eeabe6324ef9255c672d108f7dc7e157afd57b51de40d304174070a222cd42a6 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-fmmtf-collect-data-pod eeabe6324ef9255c672d108f7dc7e157afd57b51de40d304174070a222cd42a6 network default NAD default] I0327 15:43:06.774288 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-collect-data-pod eeabe6324ef9255c672d108f7dc7e157afd57b51de40d304174070a222cd42a6 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-fmmtf-collect-data-pod eeabe6324ef9255c672d108f7dc7e157afd57b51de40d304174070a222cd42a6 network default NAD default], result "{}", err I0327 15:43:06.916845 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-collect-data-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:06.916910 3815 pods.go:176] Deleting pod: default-tenant-managed/managed-fmmtf-collect-data-pod I0327 15:43:06.918264 3815 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-fmmtf-collect-data-pod, ips: 10.129.0.71 I0327 15:43:06.918297 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-collect-data-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:08.102543 3815 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-fmmtf-collect-task-params-pod] creating logical port default-tenant-managed_managed-fmmtf-collect-task-params-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:43:08.102744 3815 kube.go:256] Updating pod default-tenant-managed/managed-fmmtf-collect-task-params-pod I0327 15:43:08.128119 3815 pod.go:62] [default-tenant-managed/managed-fmmtf-collect-task-params-pod] pod update took 25.411457ms I0327 15:43:08.128146 3815 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-fmmtf-collect-task-params-pod] addLogicalPort annotation time took 25.447346ms I0327 15:43:08.129512 3815 pods.go:271] [default-tenant-managed/managed-fmmtf-collect-task-params-pod] addLogicalPort took 26.987138ms, libovsdb time 916.338µs I0327 15:43:08.474871 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-collect-task-params-pod fecdaa6609271f62e1b7ffae72b071a12bbdcf726bb35aad21d9b82c339a9286 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-fmmtf-collect-task-params-pod fecdaa6609271f62e1b7ffae72b071a12bbdcf726bb35aad21d9b82c339a9286 network default NAD default] I0327 15:43:08.484317 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-fmmtf-collect-task-params-pod, hostIfaceName: fecdaa6609271f6, network: default, NAD default, SandboxID: "fecdaa6609271f62e1b7ffae72b071a12bbdcf726bb35aad21d9b82c339a9286", PCI device ID: , UID: "24a835b7-172c-452f-a312-45430d644e22", MAC: 0a:58:0a:81:00:48, IPs: [10.129.0.72/23] I0327 15:43:08.534091 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-collect-task-params-pod fecdaa6609271f62e1b7ffae72b071a12bbdcf726bb35aad21d9b82c339a9286 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-fmmtf-collect-task-params-pod fecdaa6609271f62e1b7ffae72b071a12bbdcf726bb35aad21d9b82c339a9286 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:9b:6b:52:00:a1\",\"name\":\"fecdaa6609271f6\"},{\"mac\":\"0a:58:0a:81:00:48\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bd3ea09e-1a7c-4bc9-9112-5de5c3310d8c\"}],\"ips\":[{\"address\":\"10.129.0.72/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:43:09.595566 3815 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod] creating logical port default-tenant-managed_managed-fmmtf-collect-registry-token-secret-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:43:09.595719 3815 kube.go:256] Updating pod default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod I0327 15:43:09.617783 3815 pod.go:62] [default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod] pod update took 22.107208ms I0327 15:43:09.617807 3815 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod] addLogicalPort annotation time took 22.137358ms I0327 15:43:09.619110 3815 pods.go:271] [default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod] addLogicalPort took 23.563662ms, libovsdb time 864.545µs I0327 15:43:09.959444 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod 3e5cf2d45a44cc0325841dae5fe7eaba370721b7041dd8bf9d619c6d6df7447b network default NAD default] ADD starting CNI request [default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod 3e5cf2d45a44cc0325841dae5fe7eaba370721b7041dd8bf9d619c6d6df7447b network default NAD default] I0327 15:43:09.969142 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-fmmtf-collect-registry-token-secret-pod, hostIfaceName: 3e5cf2d45a44cc0, network: default, NAD default, SandboxID: "3e5cf2d45a44cc0325841dae5fe7eaba370721b7041dd8bf9d619c6d6df7447b", PCI device ID: , UID: "2a600057-707a-45ef-b5d4-e404cfa84005", MAC: 0a:58:0a:81:00:49, IPs: [10.129.0.73/23] I0327 15:43:10.019241 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod 3e5cf2d45a44cc0325841dae5fe7eaba370721b7041dd8bf9d619c6d6df7447b network default NAD default] ADD finished CNI request [default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod 3e5cf2d45a44cc0325841dae5fe7eaba370721b7041dd8bf9d619c6d6df7447b network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:5f:b8:ae:a2:9c\",\"name\":\"3e5cf2d45a44cc0\"},{\"mac\":\"0a:58:0a:81:00:49\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b429361e-8f34-405c-9aa1-e95975d00bd9\"}],\"ips\":[{\"address\":\"10.129.0.73/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:43:10.510477 3815 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-fmmtf-check-data-keys-pod] creating logical port default-tenant-managed_managed-fmmtf-check-data-keys-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:43:10.510650 3815 kube.go:256] Updating pod default-tenant-managed/managed-fmmtf-check-data-keys-pod I0327 15:43:10.563503 3815 pod.go:62] [default-tenant-managed/managed-fmmtf-check-data-keys-pod] pod update took 52.912469ms I0327 15:43:10.563531 3815 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-fmmtf-check-data-keys-pod] addLogicalPort annotation time took 52.947064ms I0327 15:43:10.564862 3815 pods.go:271] [default-tenant-managed/managed-fmmtf-check-data-keys-pod] addLogicalPort took 54.398593ms, libovsdb time 868.812µs I0327 15:43:10.881608 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-check-data-keys-pod cc23ef7df14a4d909380d70b7ad84184bb1cdd43d1c0c7e0f51fc5a2c8e0a10a network default NAD default] ADD starting CNI request [default-tenant-managed/managed-fmmtf-check-data-keys-pod cc23ef7df14a4d909380d70b7ad84184bb1cdd43d1c0c7e0f51fc5a2c8e0a10a network default NAD default] I0327 15:43:10.890011 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-fmmtf-check-data-keys-pod, hostIfaceName: cc23ef7df14a4d9, network: default, NAD default, SandboxID: "cc23ef7df14a4d909380d70b7ad84184bb1cdd43d1c0c7e0f51fc5a2c8e0a10a", PCI device ID: , UID: "19ddbb6c-53a6-49bf-b0eb-aae4596ddb2d", MAC: 0a:58:0a:81:00:4a, IPs: [10.129.0.74/23] I0327 15:43:10.939439 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-check-data-keys-pod cc23ef7df14a4d909380d70b7ad84184bb1cdd43d1c0c7e0f51fc5a2c8e0a10a network default NAD default] ADD finished CNI request [default-tenant-managed/managed-fmmtf-check-data-keys-pod cc23ef7df14a4d909380d70b7ad84184bb1cdd43d1c0c7e0f51fc5a2c8e0a10a network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:17:4b:0f:d2:1c\",\"name\":\"cc23ef7df14a4d9\"},{\"mac\":\"0a:58:0a:81:00:4a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b51f505d-8648-4921-aa2a-0353e7e5feb5\"}],\"ips\":[{\"address\":\"10.129.0.74/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:43:11.070338 3815 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-fmmtf-reduce-snapshot-pod] creating logical port default-tenant-managed_managed-fmmtf-reduce-snapshot-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:43:11.070477 3815 kube.go:256] Updating pod default-tenant-managed/managed-fmmtf-reduce-snapshot-pod I0327 15:43:11.118673 3815 pod.go:62] [default-tenant-managed/managed-fmmtf-reduce-snapshot-pod] pod update took 48.229461ms I0327 15:43:11.118699 3815 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-fmmtf-reduce-snapshot-pod] addLogicalPort annotation time took 48.262187ms I0327 15:43:11.120215 3815 pods.go:271] [default-tenant-managed/managed-fmmtf-reduce-snapshot-pod] addLogicalPort took 49.889275ms, libovsdb time 1.069665ms I0327 15:43:11.321278 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0327 15:43:11.462792 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-reduce-snapshot-pod c573bba2d1fea492978c37d11d04f6fa6f768d94d0dd901e8345cc1c2f1ee106 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-fmmtf-reduce-snapshot-pod c573bba2d1fea492978c37d11d04f6fa6f768d94d0dd901e8345cc1c2f1ee106 network default NAD default] I0327 15:43:11.471818 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-fmmtf-reduce-snapshot-pod, hostIfaceName: c573bba2d1fea49, network: default, NAD default, SandboxID: "c573bba2d1fea492978c37d11d04f6fa6f768d94d0dd901e8345cc1c2f1ee106", PCI device ID: , UID: "7f62fadb-849f-4378-96be-bda4df557759", MAC: 0a:58:0a:81:00:4b, IPs: [10.129.0.75/23] I0327 15:43:11.534679 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-reduce-snapshot-pod c573bba2d1fea492978c37d11d04f6fa6f768d94d0dd901e8345cc1c2f1ee106 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-fmmtf-reduce-snapshot-pod c573bba2d1fea492978c37d11d04f6fa6f768d94d0dd901e8345cc1c2f1ee106 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:64:a1:24:cb:bd\",\"name\":\"c573bba2d1fea49\"},{\"mac\":\"0a:58:0a:81:00:4b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5cb68163-fac2-4b50-95ca-732f0474d4d3\"}],\"ips\":[{\"address\":\"10.129.0.75/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:43:14.779986 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-collect-task-params-pod fecdaa6609271f62e1b7ffae72b071a12bbdcf726bb35aad21d9b82c339a9286 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-fmmtf-collect-task-params-pod fecdaa6609271f62e1b7ffae72b071a12bbdcf726bb35aad21d9b82c339a9286 network default NAD default] I0327 15:43:14.865256 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-collect-task-params-pod fecdaa6609271f62e1b7ffae72b071a12bbdcf726bb35aad21d9b82c339a9286 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-fmmtf-collect-task-params-pod fecdaa6609271f62e1b7ffae72b071a12bbdcf726bb35aad21d9b82c339a9286 network default NAD default], result "{}", err I0327 15:43:14.947135 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-collect-task-params-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:14.947172 3815 pods.go:176] Deleting pod: default-tenant-managed/managed-fmmtf-collect-task-params-pod I0327 15:43:14.948769 3815 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-fmmtf-collect-task-params-pod, ips: 10.129.0.72 I0327 15:43:14.948806 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-collect-task-params-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:17.819804 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod 3e5cf2d45a44cc0325841dae5fe7eaba370721b7041dd8bf9d619c6d6df7447b network default NAD default] DEL starting CNI request [default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod 3e5cf2d45a44cc0325841dae5fe7eaba370721b7041dd8bf9d619c6d6df7447b network default NAD default] I0327 15:43:17.908548 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod 3e5cf2d45a44cc0325841dae5fe7eaba370721b7041dd8bf9d619c6d6df7447b network default NAD default] DEL finished CNI request [default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod 3e5cf2d45a44cc0325841dae5fe7eaba370721b7041dd8bf9d619c6d6df7447b network default NAD default], result "{}", err I0327 15:43:17.984064 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:17.984108 3815 pods.go:176] Deleting pod: default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod I0327 15:43:17.985723 3815 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod, ips: 10.129.0.73 I0327 15:43:17.985754 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-collect-registry-token-secret-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:22.210489 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=9 I0327 15:43:31.950056 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-check-data-keys-pod cc23ef7df14a4d909380d70b7ad84184bb1cdd43d1c0c7e0f51fc5a2c8e0a10a network default NAD default] DEL starting CNI request [default-tenant-managed/managed-fmmtf-check-data-keys-pod cc23ef7df14a4d909380d70b7ad84184bb1cdd43d1c0c7e0f51fc5a2c8e0a10a network default NAD default] I0327 15:43:32.037637 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-check-data-keys-pod cc23ef7df14a4d909380d70b7ad84184bb1cdd43d1c0c7e0f51fc5a2c8e0a10a network default NAD default] DEL finished CNI request [default-tenant-managed/managed-fmmtf-check-data-keys-pod cc23ef7df14a4d909380d70b7ad84184bb1cdd43d1c0c7e0f51fc5a2c8e0a10a network default NAD default], result "{}", err I0327 15:43:32.115668 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-check-data-keys-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:32.115713 3815 pods.go:176] Deleting pod: default-tenant-managed/managed-fmmtf-check-data-keys-pod I0327 15:43:32.117134 3815 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-fmmtf-check-data-keys-pod, ips: 10.129.0.74 I0327 15:43:32.117163 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-check-data-keys-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:42.027118 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-reduce-snapshot-pod c573bba2d1fea492978c37d11d04f6fa6f768d94d0dd901e8345cc1c2f1ee106 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-fmmtf-reduce-snapshot-pod c573bba2d1fea492978c37d11d04f6fa6f768d94d0dd901e8345cc1c2f1ee106 network default NAD default] I0327 15:43:42.110253 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-reduce-snapshot-pod c573bba2d1fea492978c37d11d04f6fa6f768d94d0dd901e8345cc1c2f1ee106 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-fmmtf-reduce-snapshot-pod c573bba2d1fea492978c37d11d04f6fa6f768d94d0dd901e8345cc1c2f1ee106 network default NAD default], result "{}", err I0327 15:43:42.199822 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-reduce-snapshot-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:42.199851 3815 pods.go:176] Deleting pod: default-tenant-managed/managed-fmmtf-reduce-snapshot-pod I0327 15:43:42.201203 3815 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-fmmtf-reduce-snapshot-pod, ips: 10.129.0.75 I0327 15:43:42.201233 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-reduce-snapshot-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:43:45.023501 3815 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-fmmtf-apply-mapping-pod] creating logical port default-tenant-managed_managed-fmmtf-apply-mapping-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:43:45.023667 3815 kube.go:256] Updating pod default-tenant-managed/managed-fmmtf-apply-mapping-pod I0327 15:43:45.050324 3815 pod.go:62] [default-tenant-managed/managed-fmmtf-apply-mapping-pod] pod update took 26.699243ms I0327 15:43:45.050349 3815 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-fmmtf-apply-mapping-pod] addLogicalPort annotation time took 26.727799ms I0327 15:43:45.051668 3815 pods.go:271] [default-tenant-managed/managed-fmmtf-apply-mapping-pod] addLogicalPort took 28.160726ms, libovsdb time 875.288µs I0327 15:43:45.393347 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-apply-mapping-pod 264b00c242a5b068b3adea8c8ad6ad107b5df785acb16e605c087668c555d076 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-fmmtf-apply-mapping-pod 264b00c242a5b068b3adea8c8ad6ad107b5df785acb16e605c087668c555d076 network default NAD default] I0327 15:43:45.402249 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-fmmtf-apply-mapping-pod, hostIfaceName: 264b00c242a5b06, network: default, NAD default, SandboxID: "264b00c242a5b068b3adea8c8ad6ad107b5df785acb16e605c087668c555d076", PCI device ID: , UID: "23bafd95-7294-4afd-abcc-6115a17c5b95", MAC: 0a:58:0a:81:00:4c, IPs: [10.129.0.76/23] I0327 15:43:45.451504 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-apply-mapping-pod 264b00c242a5b068b3adea8c8ad6ad107b5df785acb16e605c087668c555d076 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-fmmtf-apply-mapping-pod 264b00c242a5b068b3adea8c8ad6ad107b5df785acb16e605c087668c555d076 network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:c5:cd:85:a6:d7\",\"name\":\"264b00c242a5b06\"},{\"mac\":\"0a:58:0a:81:00:4c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f7738c9c-b879-42e2-b3dd-28085b0b4150\"}],\"ips\":[{\"address\":\"10.129.0.76/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:43:59.335800 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0327 15:44:07.236339 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-apply-mapping-pod 264b00c242a5b068b3adea8c8ad6ad107b5df785acb16e605c087668c555d076 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-fmmtf-apply-mapping-pod 264b00c242a5b068b3adea8c8ad6ad107b5df785acb16e605c087668c555d076 network default NAD default] I0327 15:44:07.330899 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-apply-mapping-pod 264b00c242a5b068b3adea8c8ad6ad107b5df785acb16e605c087668c555d076 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-fmmtf-apply-mapping-pod 264b00c242a5b068b3adea8c8ad6ad107b5df785acb16e605c087668c555d076 network default NAD default], result "{}", err I0327 15:44:07.413738 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-apply-mapping-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:07.413790 3815 pods.go:176] Deleting pod: default-tenant-managed/managed-fmmtf-apply-mapping-pod I0327 15:44:07.415501 3815 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-fmmtf-apply-mapping-pod, ips: 10.129.0.76 I0327 15:44:07.415534 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-apply-mapping-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:09.231551 3815 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-fmmtf-filter-already-released-images-pod] creating logical port default-tenant-managed_managed-fmmtf-filter-already-released-images-pod for pod on switch ip-10-0-0-192.ec2.internal I0327 15:44:09.231705 3815 kube.go:256] Updating pod default-tenant-managed/managed-fmmtf-filter-already-released-images-pod I0327 15:44:09.256699 3815 pod.go:62] [default-tenant-managed/managed-fmmtf-filter-already-released-images-pod] pod update took 25.031107ms I0327 15:44:09.256725 3815 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-fmmtf-filter-already-released-images-pod] addLogicalPort annotation time took 25.063975ms I0327 15:44:09.258139 3815 pods.go:271] [default-tenant-managed/managed-fmmtf-filter-already-released-images-pod] addLogicalPort took 26.605604ms, libovsdb time 928.789µs I0327 15:44:09.609068 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-filter-already-released-images-pod ddc143c6fd3bd379047440d667b7de5d65693a2372a964077967302423a44232 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-fmmtf-filter-already-released-images-pod ddc143c6fd3bd379047440d667b7de5d65693a2372a964077967302423a44232 network default NAD default] I0327 15:44:09.617445 3815 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-fmmtf-filter-already-released-images-pod, hostIfaceName: ddc143c6fd3bd37, network: default, NAD default, SandboxID: "ddc143c6fd3bd379047440d667b7de5d65693a2372a964077967302423a44232", PCI device ID: , UID: "58e8c10a-9c0f-4b5a-91bf-b036179e9e8e", MAC: 0a:58:0a:81:00:4d, IPs: [10.129.0.77/23] I0327 15:44:09.668658 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-filter-already-released-images-pod ddc143c6fd3bd379047440d667b7de5d65693a2372a964077967302423a44232 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-fmmtf-filter-already-released-images-pod ddc143c6fd3bd379047440d667b7de5d65693a2372a964077967302423a44232 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:62:37:02:57:e9\",\"name\":\"ddc143c6fd3bd37\"},{\"mac\":\"0a:58:0a:81:00:4d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6b14add5-10b1-4b9a-97f7-873b23674218\"}],\"ips\":[{\"address\":\"10.129.0.77/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 15:44:16.685324 3815 cni.go:403] [default-tenant-managed/affinity-assistant-0426a73a44-0 9167622677b9051ebed7c17b6c5158f5e671255376619d253d71998cbee4d270 network default NAD default] DEL starting CNI request [default-tenant-managed/affinity-assistant-0426a73a44-0 9167622677b9051ebed7c17b6c5158f5e671255376619d253d71998cbee4d270 network default NAD default] I0327 15:44:16.770635 3815 cni.go:424] [default-tenant-managed/affinity-assistant-0426a73a44-0 9167622677b9051ebed7c17b6c5158f5e671255376619d253d71998cbee4d270 network default NAD default] DEL finished CNI request [default-tenant-managed/affinity-assistant-0426a73a44-0 9167622677b9051ebed7c17b6c5158f5e671255376619d253d71998cbee4d270 network default NAD default], result "{}", err I0327 15:44:16.827450 3815 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-0426a73a44-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:16.827482 3815 pods.go:176] Deleting pod: default-tenant-managed/affinity-assistant-0426a73a44-0 I0327 15:44:16.829072 3815 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/affinity-assistant-0426a73a44-0, ips: 10.129.0.70 I0327 15:44:16.829100 3815 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-0426a73a44-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:17.321832 3815 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-0426a73a44-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:44:17.321866 3815 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-0426a73a44-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:44:17.324711 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod default-tenant-managed/affinity-assistant-0426a73a44-0 I0327 15:44:17.324739 3815 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod default-tenant-managed/affinity-assistant-0426a73a44-0 I0327 15:44:17.333393 3815 cni.go:403] [default-tenant-managed/managed-fmmtf-filter-already-released-images-pod ddc143c6fd3bd379047440d667b7de5d65693a2372a964077967302423a44232 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-fmmtf-filter-already-released-images-pod ddc143c6fd3bd379047440d667b7de5d65693a2372a964077967302423a44232 network default NAD default] I0327 15:44:17.421779 3815 cni.go:424] [default-tenant-managed/managed-fmmtf-filter-already-released-images-pod ddc143c6fd3bd379047440d667b7de5d65693a2372a964077967302423a44232 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-fmmtf-filter-already-released-images-pod ddc143c6fd3bd379047440d667b7de5d65693a2372a964077967302423a44232 network default NAD default], result "{}", err I0327 15:44:17.488015 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-filter-already-released-images-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:17.488055 3815 pods.go:176] Deleting pod: default-tenant-managed/managed-fmmtf-filter-already-released-images-pod I0327 15:44:17.489340 3815 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-fmmtf-filter-already-released-images-pod, ips: 10.129.0.77 I0327 15:44:17.489367 3815 obj_retry.go:464] Detected object default-tenant-managed/managed-fmmtf-filter-already-released-images-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:44:26.335761 3815 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0327 15:44:42.216296 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=7