+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-167.ec2.internal ++ [[ -n ip-10-0-1-167.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-167.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0326 23:17:29.377308533 - disable conntrack on geneve port' I0326 23:17:29.377308533 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0326 23:17:29.394956434 - starting ovnkube-node + echo 'I0326 23:17:29.394956434 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-167.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-167.ec2.internal --init-node ip-10-0-1-167.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-167.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0326 23:17:29.434924 3917 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0326 23:17:29.434974 3917 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-5f307f4c3f.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0326 23:17:29.436466 3917 kube.go:419] Waiting for certificate I0326 23:17:29.436510 3917 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0326 23:17:29.436544 3917 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0326 23:17:29.437349 3917 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0326 23:17:29.468880 3917 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:17:29.469293 3917 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:17:29.472511 3917 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:17:29.472624 3917 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-scwq8" I0326 23:17:29.476508 3917 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-scwq8" I0326 23:17:29.476577 3917 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:17:30.437157 3917 kube.go:426] Certificate found I0326 23:17:30.437720 3917 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0326 23:17:30.438856 3917 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0326 23:17:30.508117 3917 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0326 23:17:30.525315 3917 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0326 23:17:30.525380 3917 factory.go:531] Starting watch factory I0326 23:17:30.525476 3917 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.525495 3917 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.525580 3917 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.525595 3917 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.525596 3917 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.525613 3917 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.525633 3917 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.525682 3917 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.525852 3917 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.525936 3917 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.525989 3917 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.526157 3917 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.526197 3917 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-27 23:12:29 +0000 UTC" deadline="2026-03-27 19:44:01.516181342 +0000 UTC" I0326 23:17:30.526230 3917 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h26m30.989963849s" I0326 23:17:30.526535 3917 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0326 23:17:30.531512 3917 controller_manager.go:368] Starting the ovnkube controller I0326 23:17:30.531566 3917 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-167.ec2.internal I0326 23:17:30.531682 3917 controller_manager.go:393] NBDB zone sync took: 105.761µs I0326 23:17:30.531694 3917 factory.go:531] Starting watch factory I0326 23:17:30.545632 3917 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0326 23:17:30.549047 3917 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0326 23:17:30.549077 3917 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0326 23:17:30.550360 3917 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.551575 3917 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.553123 3917 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.553212 3917 ovn_northd.go:121] Registering OVN northd metrics I0326 23:17:30.553816 3917 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.558677 3917 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.564544 3917 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:17:30.568793 3917 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0326 23:17:30.630846 3917 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0326 23:17:30.630884 3917 factory.go:1890] *v1.Namespace informer cache synced successfully I0326 23:17:30.630890 3917 factory.go:1890] *v1.Node informer cache synced successfully I0326 23:17:30.630894 3917 factory.go:1890] *v1.Service informer cache synced successfully I0326 23:17:30.630898 3917 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0326 23:17:30.630902 3917 factory.go:1890] *v1.Pod informer cache synced successfully I0326 23:17:30.631008 3917 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:17:30.631035 3917 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:17:30.631077 3917 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:17:30.631117 3917 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:17:30.635075 3917 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:17:30.635078 3917 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:17:30.636841 3917 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0326 23:17:30.636862 3917 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0326 23:17:30.636956 3917 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.636982 3917 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.637759 3917 factory.go:1890] *v1.Namespace informer cache synced successfully I0326 23:17:30.637771 3917 factory.go:1890] *v1.Node informer cache synced successfully I0326 23:17:30.637775 3917 factory.go:1890] *v1.Service informer cache synced successfully I0326 23:17:30.637778 3917 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0326 23:17:30.637781 3917 factory.go:1890] *v1.Pod informer cache synced successfully I0326 23:17:30.637785 3917 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0326 23:17:30.640224 3917 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.642201 3917 factory.go:1890] *v1.EgressIP informer cache synced successfully I0326 23:17:30.642275 3917 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.642287 3917 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.642833 3917 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0326 23:17:30.642843 3917 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0326 23:17:30.645389 3917 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.647289 3917 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0326 23:17:30.647377 3917 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.647395 3917 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.647928 3917 factory.go:1890] *v1.EgressIP informer cache synced successfully I0326 23:17:30.650551 3917 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.652540 3917 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0326 23:17:30.652621 3917 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.652651 3917 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.653226 3917 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0326 23:17:30.655575 3917 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.658274 3917 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0326 23:17:30.658286 3917 factory.go:1890] *v1.EgressService informer cache synced successfully I0326 23:17:30.658790 3917 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.658809 3917 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.661822 3917 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.663724 3917 factory.go:1890] *v1.EgressService informer cache synced successfully I0326 23:17:30.663740 3917 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0326 23:17:30.663911 3917 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:17:30.663925 3917 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:17:30.666842 3917 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:17:30.669362 3917 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0326 23:17:30.669371 3917 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0326 23:17:30.669486 3917 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.669496 3917 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.669504 3917 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.669513 3917 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.672829 3917 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.672906 3917 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:17:30.674790 3917 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0326 23:17:30.674803 3917 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0326 23:17:30.674811 3917 factory.go:636] Watch Factory start up complete, took: 149.433456ms I0326 23:17:30.674872 3917 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0326 23:17:30.674800 3917 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0326 23:17:30.680892 3917 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0326 23:17:30.680907 3917 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0326 23:17:30.680914 3917 factory.go:636] Watch Factory start up complete, took: 149.222445ms I0326 23:17:30.680923 3917 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-1-167.ec2.internal" zone I0326 23:17:30.727020 3917 default_node_network_controller.go:709] Initializing the default node network controller I0326 23:17:30.784589 3917 udn_isolation.go:101] Starting UDN host isolation manager I0326 23:17:30.791613 3917 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0326 23:17:30.909637 3917 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0326 23:17:30.909793 3917 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0326 23:17:30.909813 3917 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0326 23:17:30.945470 3917 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0326 23:17:30.945543 3917 default_node_network_controller.go:808] Node ip-10-0-1-167.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0326 23:17:30.951262 3917 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.167"] k8s.ovn.org/zone-name:ip-10-0-1-167.ec2.internal] on node ip-10-0-1-167.ec2.internal I0326 23:17:30.992366 3917 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0326 23:17:30.997907 3917 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0326 23:17:30.998041 3917 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0326 23:17:30.998202 3917 gateway_init.go:263] Preparing Gateway I0326 23:17:30.998213 3917 gateway_shared_intf.go:1658] Creating new gateway I0326 23:17:31.095391 3917 iptables.go:108] Creating table: filter chain: FORWARD I0326 23:17:31.102138 3917 iptables.go:108] Creating table: filter chain: OUTPUT I0326 23:17:31.108827 3917 gateway_shared_intf.go:1787] Gateway Creation Complete I0326 23:17:31.108841 3917 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-167.ec2.internal","mac-address":"0a:ff:e3:d1:7b:89","ip-addresses":["10.0.1.167/24"],"ip-address":"10.0.1.167/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:2d25d4a6-3835-4fa6-befb-eb368b234a2a k8s.ovn.org/node-encap-ips:["10.0.1.167"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.167/24"} k8s.ovn.org/zone-name:ip-10-0-1-167.ec2.internal] on node ip-10-0-1-167.ec2.internal I0326 23:17:31.147360 3917 default_node_network_controller.go:518] Node connection status = connected I0326 23:17:31.181957 3917 controller_manager.go:427] Waiting for node in zone sync took: 501.00916ms I0326 23:17:31.202369 3917 controller_manager.go:608] Switching to transit router for layer2 networks I0326 23:17:31.202387 3917 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-167.ec2.internal I0326 23:17:31.225883 3917 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0326 23:17:31.232171 3917 controller_manager.go:288] SCTP support detected in OVN I0326 23:17:31.232771 3917 services_controller.go:69] Creating services controller for network=default I0326 23:17:31.232837 3917 udn_enabled_svc.go:80] Waiting for service informer to sync I0326 23:17:31.232950 3917 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0326 23:17:31.232990 3917 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0326 23:17:31.233011 3917 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0326 23:17:31.233025 3917 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0326 23:17:31.233228 3917 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0326 23:17:31.233287 3917 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0326 23:17:31.233302 3917 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 42.413µs I0326 23:17:31.233326 3917 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 28.7µs I0326 23:17:31.233338 3917 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 50.067µs I0326 23:17:31.233346 3917 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0326 23:17:31.233359 3917 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0326 23:17:31.233373 3917 default_network_controller.go:321] Starting the default network controller I0326 23:17:31.233442 3917 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 11.53µs I0326 23:17:31.233481 3917 udn_enabled_svc.go:95] Performing full resync I0326 23:17:31.233841 3917 udn_enabled_svc.go:99] Waiting for handler to sync I0326 23:17:31.233856 3917 udn_enabled_svc.go:104] Starting worker I0326 23:17:31.242264 3917 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0326 23:17:31.242359 3917 repair.go:33] Syncing exgw routes took 80.929µs I0326 23:17:31.242386 3917 default_network_controller.go:407] Starting all the Watchers... I0326 23:17:31.242640 3917 namespace.go:98] [openshift-marketplace] adding namespace I0326 23:17:31.242657 3917 namespace.go:98] [openshift-dns] adding namespace I0326 23:17:31.242667 3917 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0326 23:17:31.242680 3917 namespace.go:98] [openshift-operators] adding namespace I0326 23:17:31.242692 3917 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0326 23:17:31.242696 3917 namespace.go:98] [openshift-service-ca-operator] adding namespace I0326 23:17:31.242697 3917 namespace.go:98] [openshift-network-operator] adding namespace I0326 23:17:31.242708 3917 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0326 23:17:31.242713 3917 namespace.go:98] [openshift-backplane] adding namespace I0326 23:17:31.242715 3917 namespace.go:98] [openshift-ingress-operator] adding namespace I0326 23:17:31.242722 3917 namespace.go:98] [openshift-backplane-mobb] adding namespace I0326 23:17:31.242725 3917 namespace.go:98] [kube-system] adding namespace I0326 23:17:31.242737 3917 namespace.go:98] [openshift-apiserver] adding namespace I0326 23:17:31.242760 3917 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0326 23:17:31.242782 3917 namespace.go:98] [openshift-config-managed] adding namespace I0326 23:17:31.243393 3917 namespace.go:102] [openshift-marketplace] adding namespace took 729.618µs I0326 23:17:31.243414 3917 namespace.go:98] [openshift-network-node-identity] adding namespace I0326 23:17:31.244051 3917 namespace.go:102] [openshift-network-node-identity] adding namespace took 629.868µs I0326 23:17:31.244071 3917 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0326 23:17:31.244767 3917 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 690.146µs I0326 23:17:31.244781 3917 namespace.go:98] [openshift-config-operator] adding namespace I0326 23:17:31.245502 3917 namespace.go:102] [openshift-dns] adding namespace took 2.834546ms I0326 23:17:31.245517 3917 namespace.go:98] [openshift-kube-apiserver] adding namespace I0326 23:17:31.246061 3917 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 3.385531ms I0326 23:17:31.246072 3917 namespace.go:98] [openshift-authentication-operator] adding namespace I0326 23:17:31.246692 3917 namespace.go:102] [openshift-operators] adding namespace took 4.005214ms I0326 23:17:31.246702 3917 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0326 23:17:31.247232 3917 namespace.go:102] [openshift-service-ca-operator] adding namespace took 4.531128ms I0326 23:17:31.247243 3917 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0326 23:17:31.247763 3917 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 5.066019ms I0326 23:17:31.247775 3917 namespace.go:98] [openshift-network-console] adding namespace I0326 23:17:31.248321 3917 namespace.go:102] [openshift-network-operator] adding namespace took 5.610948ms I0326 23:17:31.248338 3917 namespace.go:98] [openshift-ingress-canary] adding namespace I0326 23:17:31.248777 3917 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.056501ms I0326 23:17:31.248791 3917 namespace.go:98] [kube-node-lease] adding namespace I0326 23:17:31.249348 3917 namespace.go:102] [openshift-backplane] adding namespace took 6.627276ms I0326 23:17:31.249363 3917 namespace.go:98] [openshift-backplane-cse] adding namespace I0326 23:17:31.249876 3917 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.152655ms I0326 23:17:31.249889 3917 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0326 23:17:31.250422 3917 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.69346ms I0326 23:17:31.250437 3917 namespace.go:98] [openshift-console] adding namespace W0326 23:17:31.250501 3917 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-s8pvt: pod kube-system/global-pull-secret-syncer-s8pvt: no pod IPs found I0326 23:17:31.251184 3917 namespace.go:102] [kube-system] adding namespace took 8.451276ms I0326 23:17:31.251193 3917 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0326 23:17:31.251880 3917 namespace.go:102] [openshift-apiserver] adding namespace took 9.116838ms I0326 23:17:31.251901 3917 namespace.go:98] [openshift-host-network] adding namespace I0326 23:17:31.252521 3917 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 9.746467ms I0326 23:17:31.252538 3917 namespace.go:98] [openshift-kube-scheduler] adding namespace I0326 23:17:31.253224 3917 namespace.go:102] [openshift-config-managed] adding namespace took 10.434155ms I0326 23:17:31.253239 3917 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0326 23:17:31.253642 3917 namespace.go:102] [openshift-config-operator] adding namespace took 8.856317ms I0326 23:17:31.253652 3917 namespace.go:98] [openshift-insights] adding namespace I0326 23:17:31.254170 3917 namespace.go:102] [openshift-kube-apiserver] adding namespace took 8.648266ms I0326 23:17:31.254182 3917 namespace.go:98] [openshift-multus] adding namespace I0326 23:17:31.254684 3917 namespace.go:102] [openshift-authentication-operator] adding namespace took 8.608758ms I0326 23:17:31.254694 3917 namespace.go:98] [openshift-ingress] adding namespace I0326 23:17:31.255158 3917 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 8.449755ms I0326 23:17:31.255178 3917 namespace.go:98] [openshift-must-gather-operator] adding namespace I0326 23:17:31.255610 3917 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 8.360931ms I0326 23:17:31.255625 3917 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0326 23:17:31.256113 3917 namespace.go:102] [openshift-network-console] adding namespace took 8.333652ms I0326 23:17:31.256124 3917 namespace.go:98] [openshift-apiserver-operator] adding namespace I0326 23:17:31.256648 3917 namespace.go:102] [openshift-ingress-canary] adding namespace took 8.303517ms I0326 23:17:31.256666 3917 namespace.go:98] [openshift-image-registry] adding namespace I0326 23:17:31.257201 3917 namespace.go:102] [kube-node-lease] adding namespace took 8.402465ms I0326 23:17:31.257212 3917 namespace.go:98] [openshift-dns-operator] adding namespace I0326 23:17:31.258054 3917 namespace.go:102] [openshift-backplane-cse] adding namespace took 8.686696ms I0326 23:17:31.258065 3917 namespace.go:98] [openshift-config] adding namespace I0326 23:17:31.258460 3917 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 8.565621ms I0326 23:17:31.258475 3917 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0326 23:17:31.259015 3917 namespace.go:102] [openshift-console] adding namespace took 8.573191ms I0326 23:17:31.259027 3917 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0326 23:17:31.259523 3917 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 8.324475ms I0326 23:17:31.259544 3917 namespace.go:98] [openshift-backplane-srep] adding namespace I0326 23:17:31.260064 3917 namespace.go:102] [openshift-host-network] adding namespace took 8.154777ms I0326 23:17:31.260077 3917 namespace.go:98] [openshift-backplane-cee] adding namespace I0326 23:17:31.260554 3917 namespace.go:102] [openshift-kube-scheduler] adding namespace took 8.01146ms I0326 23:17:31.260568 3917 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0326 23:17:31.261057 3917 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.813584ms I0326 23:17:31.261069 3917 namespace.go:98] [openshift-backplane-csm] adding namespace W0326 23:17:31.261459 3917 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-5lbkg: pod openshift-multus/network-metrics-daemon-5lbkg: no pod IPs found W0326 23:17:31.261471 3917 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-ctl7v: pod openshift-multus/network-metrics-daemon-ctl7v: no pod IPs found I0326 23:17:31.261504 3917 namespace.go:102] [openshift-insights] adding namespace took 7.847304ms I0326 23:17:31.262093 3917 namespace.go:102] [openshift-multus] adding namespace took 7.905307ms I0326 23:17:31.262108 3917 namespace.go:98] [openshift-console-user-settings] adding namespace I0326 23:17:31.262629 3917 namespace.go:102] [openshift-ingress] adding namespace took 7.930674ms I0326 23:17:31.262641 3917 namespace.go:98] [openshift-console-operator] adding namespace I0326 23:17:31.263060 3917 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.875229ms I0326 23:17:31.263072 3917 namespace.go:98] [openshift-network-diagnostics] adding namespace I0326 23:17:31.263480 3917 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.85019ms I0326 23:17:31.263489 3917 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0326 23:17:31.263983 3917 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.855346ms I0326 23:17:31.263992 3917 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0326 23:17:31.264495 3917 namespace.go:102] [openshift-image-registry] adding namespace took 7.824023ms I0326 23:17:31.264503 3917 namespace.go:98] [openshift] adding namespace I0326 23:17:31.264999 3917 namespace.go:102] [openshift-dns-operator] adding namespace took 7.781293ms I0326 23:17:31.265015 3917 namespace.go:98] [openshift-authentication] adding namespace I0326 23:17:31.265446 3917 namespace.go:102] [openshift-config] adding namespace took 7.376313ms I0326 23:17:31.265460 3917 namespace.go:98] [kube-public] adding namespace I0326 23:17:31.265890 3917 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.410429ms I0326 23:17:31.265904 3917 namespace.go:98] [openshift-controller-manager] adding namespace I0326 23:17:31.266310 3917 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.276883ms I0326 23:17:31.266324 3917 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0326 23:17:31.266775 3917 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.225096ms I0326 23:17:31.266785 3917 namespace.go:98] [default] adding namespace I0326 23:17:31.267225 3917 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.143387ms I0326 23:17:31.267238 3917 namespace.go:98] [openshift-logging] adding namespace I0326 23:17:31.267595 3917 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.023671ms I0326 23:17:31.267605 3917 namespace.go:98] [openshift-customer-monitoring] adding namespace I0326 23:17:31.268100 3917 namespace.go:102] [openshift-backplane-csm] adding namespace took 7.026429ms I0326 23:17:31.268112 3917 namespace.go:98] [openshift-machine-config-operator] adding namespace I0326 23:17:31.268502 3917 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.390457ms I0326 23:17:31.268511 3917 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0326 23:17:31.269071 3917 namespace.go:102] [openshift-console-operator] adding namespace took 6.422355ms W0326 23:17:31.269073 3917 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-22h4b: pod openshift-network-diagnostics/network-check-target-22h4b: no pod IPs found I0326 23:17:31.269090 3917 namespace.go:98] [dedicated-admin] adding namespace W0326 23:17:31.269123 3917 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-xhk27: pod openshift-network-diagnostics/network-check-target-xhk27: no pod IPs found I0326 23:17:31.269772 3917 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.693673ms I0326 23:17:31.269792 3917 namespace.go:98] [openshift-infra] adding namespace I0326 23:17:31.270136 3917 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.641665ms I0326 23:17:31.270152 3917 namespace.go:98] [openshift-cluster-version] adding namespace I0326 23:17:31.270620 3917 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.622078ms I0326 23:17:31.270636 3917 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0326 23:17:31.271030 3917 namespace.go:102] [openshift] adding namespace took 6.522388ms I0326 23:17:31.271045 3917 namespace.go:98] [openshift-backplane-tam] adding namespace I0326 23:17:31.271404 3917 namespace.go:102] [openshift-authentication] adding namespace took 6.384631ms I0326 23:17:31.271417 3917 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0326 23:17:31.271819 3917 namespace.go:102] [kube-public] adding namespace took 6.354766ms I0326 23:17:31.271833 3917 namespace.go:98] [openshift-node] adding namespace I0326 23:17:31.272236 3917 namespace.go:102] [openshift-controller-manager] adding namespace took 6.325888ms I0326 23:17:31.272250 3917 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0326 23:17:31.272630 3917 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.301784ms I0326 23:17:31.272644 3917 namespace.go:98] [openshift-package-operator] adding namespace I0326 23:17:31.273076 3917 namespace.go:102] [default] adding namespace took 6.287178ms I0326 23:17:31.273462 3917 namespace.go:102] [openshift-logging] adding namespace took 6.219188ms I0326 23:17:31.273868 3917 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.257513ms I0326 23:17:31.273885 3917 namespace.go:98] [open-cluster-management-2p9mplstndhmerb1mp1pd72ivq8vo3pp] adding namespace I0326 23:17:31.274284 3917 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.167469ms I0326 23:17:31.274295 3917 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0326 23:17:31.275137 3917 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.620931ms I0326 23:17:31.275549 3917 namespace.go:102] [dedicated-admin] adding namespace took 6.454192ms I0326 23:17:31.275561 3917 namespace.go:98] [openshift-monitoring] adding namespace I0326 23:17:31.275929 3917 namespace.go:102] [openshift-infra] adding namespace took 6.130241ms I0326 23:17:31.276357 3917 namespace.go:102] [openshift-cluster-version] adding namespace took 6.201483ms I0326 23:17:31.276736 3917 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.093751ms I0326 23:17:31.276765 3917 namespace.go:98] [openshift-route-controller-manager] adding namespace I0326 23:17:31.277239 3917 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.186445ms I0326 23:17:31.277261 3917 namespace.go:98] [openshift-machine-api] adding namespace I0326 23:17:31.277812 3917 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.387494ms I0326 23:17:31.278367 3917 namespace.go:102] [openshift-node] adding namespace took 6.525042ms I0326 23:17:31.278389 3917 namespace.go:98] [openshift-etcd] adding namespace I0326 23:17:31.278826 3917 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.569676ms I0326 23:17:31.278844 3917 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0326 23:17:31.279247 3917 namespace.go:102] [openshift-package-operator] adding namespace took 6.595036ms I0326 23:17:31.279265 3917 namespace.go:98] [openshift-operators-redhat] adding namespace I0326 23:17:31.279660 3917 namespace.go:102] [open-cluster-management-2p9mplstndhmerb1mp1pd72ivq8vo3pp] adding namespace took 5.768318ms I0326 23:17:31.280082 3917 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 5.781851ms I0326 23:17:31.280610 3917 namespace.go:102] [openshift-monitoring] adding namespace took 5.043424ms I0326 23:17:31.281124 3917 namespace.go:102] [openshift-route-controller-manager] adding namespace took 4.35434ms I0326 23:17:31.281532 3917 namespace.go:102] [openshift-machine-api] adding namespace took 4.265145ms I0326 23:17:31.282133 3917 namespace.go:102] [openshift-etcd] adding namespace took 3.736465ms I0326 23:17:31.282579 3917 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 3.727954ms I0326 23:17:31.283077 3917 namespace.go:102] [openshift-operators-redhat] adding namespace took 3.807529ms I0326 23:17:31.283506 3917 zone_ic_handler.go:186] Time taken to create transit switch: 324.511µs I0326 23:17:31.283601 3917 master.go:532] Adding or Updating local node "ip-10-0-1-167.ec2.internal" for network "default" I0326 23:17:31.283693 3917 obj_retry.go:544] Failed to create *v1.Node ip-10-0-2-111.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-111.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-111.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-111.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-111.ec2.internal I0326 23:17:31.284045 3917 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-75.ec2.internal for the network default I0326 23:17:31.285257 3917 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-75.ec2.internal" on network "default" took: 1.229804ms W0326 23:17:31.291847 3917 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-s8pvt: pod kube-system/global-pull-secret-syncer-s8pvt: no pod IPs found W0326 23:17:31.291866 3917 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-xhk27: pod openshift-network-diagnostics/network-check-target-xhk27: no pod IPs found W0326 23:17:31.291879 3917 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-ctl7v: pod openshift-multus/network-metrics-daemon-ctl7v: no pod IPs found I0326 23:17:31.297387 3917 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-1-167.ec2.internal for the network default I0326 23:17:31.298173 3917 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0326 23:17:31.298206 3917 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0326 23:17:31.298220 3917 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0326 23:17:31.298225 3917 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-167.ec2.internal in network "default" I0326 23:17:31.298280 3917 node_tracker.go:169] Node ip-10-0-1-167.ec2.internal switch + router changed, syncing services in network "default" I0326 23:17:31.298291 3917 services_controller.go:574] Full service sync requested for network=default I0326 23:17:31.298298 3917 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-111.ec2.internal in network "default" I0326 23:17:31.298307 3917 node_tracker.go:235] Node ip-10-0-2-111.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-111.ec2.internal" I0326 23:17:31.298314 3917 node_tracker.go:169] Node ip-10-0-2-111.ec2.internal switch + router changed, syncing services in network "default" I0326 23:17:31.298318 3917 services_controller.go:574] Full service sync requested for network=default I0326 23:17:31.298327 3917 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-75.ec2.internal in network "default" I0326 23:17:31.298350 3917 node_tracker.go:169] Node ip-10-0-0-75.ec2.internal switch + router changed, syncing services in network "default" I0326 23:17:31.298357 3917 services_controller.go:574] Full service sync requested for network=default I0326 23:17:31.399067 3917 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0326 23:17:31.399097 3917 services_controller.go:208] Setting up event handlers for services for network=default I0326 23:17:31.399194 3917 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0326 23:17:31.399236 3917 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0326 23:17:31.399251 3917 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0326 23:17:31.499772 3917 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0326 23:17:31.499795 3917 repair.go:60] Starting repairing loop for services I0326 23:17:31.499887 3917 repair.go:131] Deleted 0 stale service LBs I0326 23:17:31.499905 3917 repair.go:137] Deleted 0 stale Chassis Template Vars I0326 23:17:31.499921 3917 repair.go:62] Finished repairing loop for services: 127.613µs I0326 23:17:31.499943 3917 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0326 23:17:31.499953 3917 services_controller.go:254] Starting workers for network=default I0326 23:17:31.500791 3917 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-22h4b, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-22h4b: suppressed error logged: pod openshift-network-diagnostics/network-check-target-22h4b: no pod IPs found I0326 23:17:31.500788 3917 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-6cxvw of type *v1.Pod in terminal state (e.g. completed) during add event: will remove it I0326 23:17:31.500869 3917 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-5lbkg, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-5lbkg: suppressed error logged: pod openshift-multus/network-metrics-daemon-5lbkg: no pod IPs found I0326 23:17:31.500906 3917 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-ctl7v] creating logical port openshift-multus_network-metrics-daemon-ctl7v for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:31.500980 3917 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-s8pvt] creating logical port kube-system_global-pull-secret-syncer-s8pvt for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:31.501054 3917 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-xhk27] creating logical port openshift-network-diagnostics_network-check-target-xhk27 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:31.501132 3917 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-xhk27 I0326 23:17:31.501184 3917 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-ctl7v I0326 23:17:31.501192 3917 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-s8pvt W0326 23:17:31.505211 3917 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:17:31.517143 3917 pod.go:62] [openshift-multus/network-metrics-daemon-ctl7v] pod update took 16.069517ms I0326 23:17:31.517163 3917 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-ctl7v] addLogicalPort annotation time took 16.094959ms I0326 23:17:31.518117 3917 pods.go:271] [openshift-multus/network-metrics-daemon-ctl7v] addLogicalPort took 17.218109ms, libovsdb time 721.68µs I0326 23:17:31.518828 3917 pod.go:62] [kube-system/global-pull-secret-syncer-s8pvt] pod update took 17.753831ms I0326 23:17:31.518834 3917 pod.go:62] [openshift-network-diagnostics/network-check-target-xhk27] pod update took 17.730124ms I0326 23:17:31.518846 3917 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-s8pvt] addLogicalPort annotation time took 17.814839ms I0326 23:17:31.518852 3917 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-xhk27] addLogicalPort annotation time took 17.748182ms I0326 23:17:31.519689 3917 pods.go:271] [kube-system/global-pull-secret-syncer-s8pvt] addLogicalPort took 18.766559ms, libovsdb time 613.722µs I0326 23:17:31.519933 3917 pods.go:271] [openshift-network-diagnostics/network-check-target-xhk27] addLogicalPort took 18.89127ms, libovsdb time 853.999µs I0326 23:17:31.520573 3917 admin_network_policy_controller.go:227] Starting controller default-network-controller I0326 23:17:31.520593 3917 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0326 23:17:31.520606 3917 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0326 23:17:31.520611 3917 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0326 23:17:31.520831 3917 repair.go:29] Repairing admin network policies took 214.643µs I0326 23:17:31.520950 3917 repair.go:92] Repairing baseline admin network policies took 109.63µs I0326 23:17:31.521550 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0326 23:17:31.521572 3917 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0326 23:17:31.523410 3917 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0326 23:17:31.523431 3917 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0326 23:17:31.523439 3917 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0326 23:17:31.524662 3917 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0326 23:17:31.524682 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0326 23:17:31.524690 3917 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0326 23:17:31.526059 3917 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0326 23:17:31.526076 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0326 23:17:31.526084 3917 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0326 23:17:31.527541 3917 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0326 23:17:31.527559 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0326 23:17:31.527568 3917 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0326 23:17:31.528019 3917 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0326 23:17:31.528042 3917 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0326 23:17:31.528059 3917 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0326 23:17:31.530476 3917 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0326 23:17:31.530494 3917 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0326 23:17:31.530502 3917 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0326 23:17:31.531004 3917 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0326 23:17:31.531025 3917 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0326 23:17:31.531031 3917 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0326 23:17:31.532666 3917 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0326 23:17:31.532685 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0326 23:17:31.533462 3917 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0326 23:17:31.533477 3917 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0326 23:17:31.534364 3917 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0326 23:17:31.534387 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0326 23:17:31.534396 3917 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0326 23:17:31.534889 3917 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0326 23:17:31.534908 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0326 23:17:31.534917 3917 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0326 23:17:31.535372 3917 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0326 23:17:31.535390 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0326 23:17:31.535396 3917 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0326 23:17:31.535835 3917 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0326 23:17:31.535852 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0326 23:17:31.535860 3917 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0326 23:17:31.536472 3917 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0326 23:17:31.536488 3917 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0326 23:17:31.536498 3917 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0326 23:17:31.537860 3917 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0326 23:17:31.537882 3917 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0326 23:17:31.537895 3917 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0326 23:17:31.539576 3917 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0326 23:17:31.539594 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0326 23:17:31.539602 3917 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0326 23:17:31.540093 3917 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0326 23:17:31.540112 3917 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0326 23:17:31.540121 3917 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0326 23:17:31.541018 3917 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0326 23:17:31.541040 3917 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0326 23:17:31.541050 3917 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0326 23:17:31.542145 3917 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0326 23:17:31.542163 3917 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0326 23:17:31.542175 3917 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0326 23:17:31.543133 3917 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0326 23:17:31.543153 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0326 23:17:31.543161 3917 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0326 23:17:31.543614 3917 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0326 23:17:31.543632 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0326 23:17:31.543639 3917 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0326 23:17:31.544959 3917 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0326 23:17:31.544977 3917 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0326 23:17:31.544993 3917 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0326 23:17:31.546082 3917 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0326 23:17:31.546107 3917 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0326 23:17:31.546122 3917 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0326 23:17:31.547391 3917 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0326 23:17:31.547413 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0326 23:17:31.547420 3917 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0326 23:17:31.547977 3917 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0326 23:17:31.547995 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0326 23:17:31.548023 3917 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0326 23:17:31.548772 3917 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0326 23:17:31.548792 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0326 23:17:31.548799 3917 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0326 23:17:31.549131 3917 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0326 23:17:31.549148 3917 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0326 23:17:31.549156 3917 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0326 23:17:31.549788 3917 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0326 23:17:31.549808 3917 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0326 23:17:31.549817 3917 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0326 23:17:31.550920 3917 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0326 23:17:31.550938 3917 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0326 23:17:31.550948 3917 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0326 23:17:31.551486 3917 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0326 23:17:31.551910 3917 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-6cxvw of type *factory.egressIPPod in terminal state (e.g. completed) during add event: will remove it I0326 23:17:31.557452 3917 egressfirewall.go:319] Starting EgressFirewall controller I0326 23:17:31.557476 3917 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0326 23:17:31.557486 3917 controller.go:133] Adding controller egress-firewall-controller event handlers I0326 23:17:31.557509 3917 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0326 23:17:31.557532 3917 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0326 23:17:31.557504 3917 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0326 23:17:31.557541 3917 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0326 23:17:31.557905 3917 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0326 23:17:31.557921 3917 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0326 23:17:31.557930 3917 egressqos.go:193] Setting up event handlers for EgressQoS I0326 23:17:31.558027 3917 egressfirewall.go:628] Syncing node "ip-10-0-0-75.ec2.internal" for egress firewall I0326 23:17:31.558041 3917 egressfirewall.go:628] Syncing node "ip-10-0-1-167.ec2.internal" for egress firewall I0326 23:17:31.558045 3917 egressfirewall.go:628] Syncing node "ip-10-0-2-111.ec2.internal" for egress firewall I0326 23:17:31.558041 3917 egressqos.go:245] Starting EgressQoS Controller I0326 23:17:31.558058 3917 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0326 23:17:31.558069 3917 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0326 23:17:31.558074 3917 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0326 23:17:31.558079 3917 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0326 23:17:31.558084 3917 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0326 23:17:31.558088 3917 shared_informer.go:356] "Caches are synced" controller="egressqos" I0326 23:17:31.558093 3917 egressqos.go:259] Repairing EgressQoSes I0326 23:17:31.558097 3917 egressqos.go:399] Starting repairing loop for egressqos I0326 23:17:31.558183 3917 egressqos.go:401] Finished repairing loop for egressqos: 85.261µs I0326 23:17:31.558198 3917 egressservice_zone.go:132] Setting up event handlers for Egress Services I0326 23:17:31.558277 3917 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-167.ec2.internal I0326 23:17:31.558324 3917 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-167.ec2.internal : 49.621µs I0326 23:17:31.558397 3917 egressservice_zone.go:210] Starting Egress Services Controller I0326 23:17:31.558405 3917 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0326 23:17:31.558411 3917 shared_informer.go:356] "Caches are synced" controller="egressservices" I0326 23:17:31.558416 3917 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0326 23:17:31.558420 3917 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0326 23:17:31.558425 3917 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0326 23:17:31.558430 3917 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0326 23:17:31.558435 3917 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0326 23:17:31.558438 3917 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0326 23:17:31.558442 3917 egressservice_zone.go:228] Repairing Egress Services I0326 23:17:31.558801 3917 master_controller.go:88] Starting Admin Policy Based Route Controller I0326 23:17:31.558813 3917 external_controller.go:278] Starting Admin Policy Based Route Controller I0326 23:17:31.558853 3917 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-75.ec2.internal I0326 23:17:31.558870 3917 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-75.ec2.internal: 18.181µs I0326 23:17:31.558884 3917 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-167.ec2.internal I0326 23:17:31.558887 3917 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-167.ec2.internal: 4.013µs I0326 23:17:31.558892 3917 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-111.ec2.internal I0326 23:17:31.558895 3917 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-111.ec2.internal: 3.488µs I0326 23:17:31.559030 3917 default_network_controller.go:568] Completing all the Watchers took 316.630285ms I0326 23:17:31.559049 3917 default_network_controller.go:572] Starting unidling controllers I0326 23:17:31.559064 3917 unidle.go:46] Registering OVN SB ControllerEvent handler I0326 23:17:31.559074 3917 unidle.go:63] Populating Initial ContollerEvent events I0326 23:17:31.559092 3917 unidle.go:79] Setting up event handlers for services I0326 23:17:31.559188 3917 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0326 23:17:31.564913 3917 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0326 23:17:31.648492 3917 default_node_network_controller.go:518] Node connection status = connected I0326 23:17:31.665238 3917 bridgeconfig.go:537] Gateway is ready I0326 23:17:31.665262 3917 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0326 23:17:31.676899 3917 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.167. Updating OVN encap IP. I0326 23:17:31.682344 3917 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.167 - it is already configured I0326 23:17:31.682362 3917 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.167/24:{} 172.20.0.1/32:{}]. Updating annotations. I0326 23:17:31.682650 3917 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.167/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-167.ec2.internal","mac-address":"0a:ff:e3:d1:7b:89","ip-addresses":["10.0.1.167/24"],"ip-address":"10.0.1.167/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:2d25d4a6-3835-4fa6-befb-eb368b234a2a k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.167/24"}] on node ip-10-0-1-167.ec2.internal I0326 23:17:31.697638 3917 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-167.ec2.internal in network "default" I0326 23:17:31.697696 3917 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.1.167 I0326 23:17:31.697709 3917 node_tracker.go:169] Node ip-10-0-1-167.ec2.internal switch + router changed, syncing services in network "default" I0326 23:17:31.697721 3917 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-167.ec2.internal I0326 23:17:31.697730 3917 services_controller.go:574] Full service sync requested for network=default I0326 23:17:31.697760 3917 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-167.ec2.internal: 27.138µs I0326 23:17:31.697777 3917 egressfirewall.go:628] Syncing node "ip-10-0-1-167.ec2.internal" for egress firewall I0326 23:17:31.697790 3917 master.go:532] Adding or Updating local node "ip-10-0-1-167.ec2.internal" for network "default" I0326 23:17:31.697835 3917 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-167.ec2.internal W0326 23:17:31.697999 3917 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:17:31.702264 3917 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-167.ec2.internal. Updating no re-route policies I0326 23:17:31.702333 3917 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-pt69g I0326 23:17:31.702350 3917 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-pt69g I0326 23:17:31.702358 3917 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-pxkxf I0326 23:17:31.702362 3917 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-ctl7v I0326 23:17:31.702363 3917 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-167.ec2.internal I0326 23:17:31.702372 3917 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-pt69g after 0 failed attempt(s) I0326 23:17:31.702373 3917 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-pxkxf I0326 23:17:31.702374 3917 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-2r77s I0326 23:17:31.702384 3917 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-qf7wd I0326 23:17:31.702387 3917 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-2r77s I0326 23:17:31.702390 3917 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-qf7wd I0326 23:17:31.702396 3917 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-fxlkt I0326 23:17:31.702405 3917 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-2r77s after 0 failed attempt(s) I0326 23:17:31.702402 3917 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-pxkxf after 0 failed attempt(s) I0326 23:17:31.702410 3917 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-vzxhh I0326 23:17:31.702413 3917 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-fxlkt I0326 23:17:31.702381 3917 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-167.ec2.internal I0326 23:17:31.702417 3917 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-ctl7v I0326 23:17:31.702417 3917 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-gqfkt I0326 23:17:31.702422 3917 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-167.ec2.internal I0326 23:17:31.702431 3917 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-gqfkt I0326 23:17:31.702435 3917 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-167.ec2.internal I0326 23:17:31.702434 3917 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-167.ec2.internal after 0 failed attempt(s) I0326 23:17:31.702435 3917 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-fxlkt after 0 failed attempt(s) I0326 23:17:31.702449 3917 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-gqfkt after 0 failed attempt(s) I0326 23:17:31.702456 3917 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-ctl7v] creating logical port openshift-multus_network-metrics-daemon-ctl7v for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:31.702345 3917 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-s4mk6 I0326 23:17:31.702467 3917 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-s4mk6 I0326 23:17:31.702478 3917 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-s4mk6 after 0 failed attempt(s) I0326 23:17:31.702451 3917 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-167.ec2.internal after 0 failed attempt(s) I0326 23:17:31.702396 3917 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-xhk27 I0326 23:17:31.702492 3917 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-xhk27 I0326 23:17:31.702514 3917 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-xhk27] creating logical port openshift-network-diagnostics_network-check-target-xhk27 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:31.702401 3917 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-qf7wd after 0 failed attempt(s) I0326 23:17:31.702411 3917 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-s8pvt I0326 23:17:31.702593 3917 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-s8pvt I0326 23:17:31.702630 3917 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-s8pvt] creating logical port kube-system_global-pull-secret-syncer-s8pvt for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:31.702420 3917 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-vzxhh I0326 23:17:31.702691 3917 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-vzxhh after 0 failed attempt(s) I0326 23:17:31.702421 3917 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-wzd2g I0326 23:17:31.702710 3917 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-wzd2g I0326 23:17:31.702721 3917 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-wzd2g after 0 failed attempt(s) I0326 23:17:31.703219 3917 pods.go:271] [openshift-multus/network-metrics-daemon-ctl7v] addLogicalPort took 773.982µs, libovsdb time 263.14µs I0326 23:17:31.703234 3917 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-ctl7v after 0 failed attempt(s) I0326 23:17:31.703276 3917 pods.go:271] [openshift-network-diagnostics/network-check-target-xhk27] addLogicalPort took 769.168µs, libovsdb time 247.009µs I0326 23:17:31.703285 3917 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-xhk27 after 0 failed attempt(s) I0326 23:17:31.703419 3917 pods.go:271] [kube-system/global-pull-secret-syncer-s8pvt] addLogicalPort took 800.903µs, libovsdb time 276.109µs I0326 23:17:31.703428 3917 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-s8pvt after 0 failed attempt(s) I0326 23:17:31.713170 3917 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0326 23:17:31.725532 3917 iptables.go:108] Creating table: mangle chain: OUTPUT I0326 23:17:31.729238 3917 iptables.go:108] Creating table: nat chain: OUTPUT I0326 23:17:31.754682 3917 iptables.go:108] Creating table: nat chain: PREROUTING I0326 23:17:31.850656 3917 iptables.go:108] Creating table: filter chain: FORWARD I0326 23:17:31.869169 3917 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0326 23:17:31.869275 3917 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0326 23:17:31.869319 3917 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0326 23:17:31.869415 3917 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0326 23:17:31.869474 3917 gateway_init.go:325] Gateway and management port readiness took 725.691112ms I0326 23:17:31.869502 3917 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0326 23:17:31.869545 3917 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0326 23:17:31.869558 3917 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0326 23:17:31.869567 3917 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0326 23:17:31.869631 3917 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0326 23:17:31.869718 3917 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 55.228µs I0326 23:17:31.870004 3917 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0326 23:17:31.870029 3917 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 14.815µs I0326 23:17:31.870035 3917 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 31.698µs I0326 23:17:31.870041 3917 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0326 23:17:31.870049 3917 nad_controller.go:173] [node-nad-controller NAD controller]: started I0326 23:17:31.870054 3917 default_node_network_controller.go:898] Starting the default node network controller I0326 23:17:31.870059 3917 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0326 23:17:31.870142 3917 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 13.299µs I0326 23:17:31.870214 3917 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0326 23:17:31.870222 3917 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0326 23:17:31.870259 3917 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b09230 0xc0002d3ce0 0xc0002d3d50 0xc0002d3dc0 0xc0002d3e30 0xc0002d3ea0 0xc0002d3f10 0xc000b1d5e0 0xc0002dc000 0xc0002dc070 0xc0008be2d0 0xc00647afc0 0xc0002dc0e0 map[0x2f67040:0xc002595620 0x2f67580:0xc0025957a0 0x2f68000:0xc002595560 0x2f682a0:0xc0025955c0 0x2f687e0:0xc002595680 0x2f68a80:0xc0025956e0 0x2f68d20:0xc002595740 0x2f69500:0xc002595800 0x2f697a0:0xc004f06900 0x2f93d40:0xc000f1b4a0 0x2f940e0:0xc000f1b500 0x2f94bc0:0xc000c815c0 0x2f96520:0xc001bba000 0x2f97000:0xc000f1b6e0 0x2f97e80:0xc000f97bc0] 0xc000b3c5b0 0} I0326 23:17:31.870316 3917 gateway.go:151] Starting gateway service sync I0326 23:17:31.870811 3917 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0326 23:17:31.873374 3917 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0326 23:17:31.905051 3917 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0326 23:17:31.906510 3917 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0326 23:17:31.907915 3917 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0326 23:17:31.944669 3917 gateway.go:168] Gateway service sync done. Time taken: 74.338496ms I0326 23:17:31.945115 3917 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31019 I0326 23:17:31.945537 3917 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b09230 0xc0002d3ce0 0xc0002d3d50 0xc0002d3dc0 0xc0002d3e30 0xc0002d3ea0 0xc0002d3f10 0xc000b1d5e0 0xc0002dc000 0xc0002dc070 0xc0008be2d0 0xc00647afc0 0xc0002dc0e0 map[0x2f67040:0xc002595620 0x2f67580:0xc0025957a0 0x2f68000:0xc002595560 0x2f682a0:0xc0025955c0 0x2f687e0:0xc002595680 0x2f68a80:0xc0025956e0 0x2f68d20:0xc002595740 0x2f69500:0xc002595800 0x2f697a0:0xc004f06900 0x2f93d40:0xc000f1b4a0 0x2f940e0:0xc000f1b500 0x2f94bc0:0xc000c815c0 0x2f96520:0xc001bba000 0x2f97000:0xc000f1b6e0 0x2f97e80:0xc000f97bc0] 0xc000b3c5b0 0} I0326 23:17:31.946069 3917 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b09230 0xc0002d3ce0 0xc0002d3d50 0xc0002d3dc0 0xc0002d3e30 0xc0002d3ea0 0xc0002d3f10 0xc000b1d5e0 0xc0002dc000 0xc0002dc070 0xc0008be2d0 0xc00647afc0 0xc0002dc0e0 map[0x2f67040:0xc002595620 0x2f67580:0xc0025957a0 0x2f68000:0xc002595560 0x2f682a0:0xc0025955c0 0x2f687e0:0xc002595680 0x2f68a80:0xc0025956e0 0x2f68d20:0xc002595740 0x2f69500:0xc002595800 0x2f697a0:0xc004f06900 0x2f93d40:0xc000f1b4a0 0x2f940e0:0xc000f1b500 0x2f94bc0:0xc000c815c0 0x2f96520:0xc001bba000 0x2f97000:0xc000f1b6e0 0x2f97e80:0xc000f97bc0] 0xc000b3c5b0 0} I0326 23:17:31.946118 3917 gateway.go:519] Reconciling gateway with updates I0326 23:17:31.952044 3917 gateway.go:326] Spawning Conntrack Rule Check Thread I0326 23:17:31.952130 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:17:31.952140 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:17:31.952150 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:17:31.952151 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0326 23:17:31.952155 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:17:31.952163 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:17:31.952168 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0326 23:17:31.952168 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0326 23:17:31.952183 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0326 23:17:31.952186 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:17:31.952178 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:17:31.952198 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:17:31.952203 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:17:31.952205 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0326 23:17:31.952209 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:17:31.952218 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0326 23:17:31.952219 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:17:31.952224 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0326 23:17:31.952227 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:17:31.952204 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:17:31.952233 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:17:31.952243 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:17:31.952247 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:17:31.952253 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:17:31.952255 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:17:31.952259 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:17:31.952262 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0326 23:17:31.952265 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:17:31.952181 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:17:31.952283 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:17:31.952284 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:17:31.952290 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0326 23:17:31.952294 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:17:31.952264 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:17:31.952300 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:17:31.952256 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:17:31.952310 3917 default_node_network_controller.go:943] Gateway and management port readiness took 82.079475ms I0326 23:17:31.952318 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0326 23:17:31.952169 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:17:31.952330 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0326 23:17:31.952160 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0326 23:17:31.952240 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0326 23:17:31.952198 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:17:31.952286 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:17:31.952349 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0326 23:17:31.952360 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0326 23:17:31.952251 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:17:31.952370 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:17:31.952375 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:17:31.952210 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:17:31.952383 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:17:31.952360 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:17:31.952392 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:17:31.952395 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0326 23:17:31.952193 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0326 23:17:31.952406 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:17:31.952397 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:17:31.952416 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:17:31.952425 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0326 23:17:31.952195 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:17:31.952443 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:17:31.952444 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0326 23:17:31.952447 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0326 23:17:31.952449 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0326 23:17:31.952351 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:17:31.952296 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:17:31.952460 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0326 23:17:31.952425 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0326 23:17:31.952462 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0326 23:17:31.952241 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:17:31.952484 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:17:31.952494 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:17:31.952500 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:17:31.952513 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:17:31.952530 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:17:31.952538 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:17:31.952550 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:17:31.952555 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:17:31.952559 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:17:31.952517 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0326 23:17:31.952565 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:17:31.952323 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:17:31.952362 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0326 23:17:31.952541 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0326 23:17:31.952586 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0326 23:17:31.952593 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0326 23:17:31.952594 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0326 23:17:31.952604 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0326 23:17:31.952611 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:17:31.952603 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0326 23:17:31.952635 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0326 23:17:31.952648 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0326 23:17:31.952378 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:17:31.952687 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:17:31.952635 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0326 23:17:31.952693 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0326 23:17:31.952321 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0326 23:17:31.952513 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0326 23:17:31.952709 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:17:31.952527 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0326 23:17:31.952712 3917 port_claim.go:71] Svc openshift-ingress/router-default: port 30390 is already open, no action needed I0326 23:17:31.952727 3917 port_claim.go:71] Svc openshift-ingress/router-default: port 32570 is already open, no action needed I0326 23:17:31.952143 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:17:31.952739 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:17:31.952589 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0326 23:17:31.952385 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0326 23:17:31.952780 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:17:31.952223 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:17:31.952795 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:17:31.952802 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0326 23:17:31.952600 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0326 23:17:31.952830 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0326 23:17:31.952848 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:17:31.952871 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:17:31.952622 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:17:31.952433 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0326 23:17:31.952699 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:17:31.952716 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:17:31.952232 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:17:31.952728 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:17:31.953037 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:17:31.953042 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0326 23:17:31.952776 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:17:31.953051 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0326 23:17:31.953609 3917 node_ip_handler_linux.go:160] Node IP manager is running W0326 23:17:31.990889 3917 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0326 23:17:31.991217 3917 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0326 23:17:31.991356 3917 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0326 23:17:31.999207 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0326 23:17:31.999235 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0326 23:17:31.999242 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0326 23:17:32.038440 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0326 23:17:32.038465 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0326 23:17:32.058355 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0326 23:17:32.058362 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0326 23:17:32.058380 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0326 23:17:32.058391 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0326 23:17:32.058406 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0326 23:17:32.058420 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0326 23:17:33.136543 3917 default_node_network_controller.go:1364] Starting node controller node sync W0326 23:17:33.136551 3917 egressip_healthcheck.go:76] Health checking using insecure connection I0326 23:17:33.136603 3917 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0326 23:17:33.148952 3917 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 12.383203ms I0326 23:17:33.149120 3917 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-75.ec2.internal", IP: 10.0.0.75 to PMTUD blocking rules I0326 23:17:33.149132 3917 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-75.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:17:33.155136 3917 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0326 23:17:33.155404 3917 egressservice_node.go:93] Setting up event handlers for Egress Services I0326 23:17:33.155579 3917 egressservice_node.go:183] Starting Egress Services Controller I0326 23:17:33.155598 3917 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0326 23:17:33.155606 3917 shared_informer.go:356] "Caches are synced" controller="egressservices" I0326 23:17:33.155611 3917 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0326 23:17:33.155616 3917 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0326 23:17:33.155623 3917 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0326 23:17:33.155630 3917 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0326 23:17:33.155636 3917 egressservice_node.go:197] Repairing Egress Services I0326 23:17:33.166812 3917 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0326 23:17:33.166830 3917 external_controller.go:278] Starting Admin Policy Based Route Controller I0326 23:17:33.166995 3917 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0326 23:17:33.167013 3917 link_network_manager.go:117] Link manager is running I0326 23:17:33.167026 3917 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0326 23:17:33.167118 3917 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0326 23:17:33.167440 3917 vrf_manager.go:128] VRF manager is running I0326 23:17:33.167507 3917 node_controller_manager.go:437] Removing flows to drop GARP I0326 23:17:33.167520 3917 gateway.go:519] Reconciling gateway with updates I0326 23:17:33.196786 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:17:33.196802 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:17:33.196808 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:17:33.196816 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:17:33.196821 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:17:33.196823 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:17:33.196821 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0326 23:17:33.196833 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:17:33.196840 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0326 23:17:33.196841 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0326 23:17:33.196844 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:17:33.196851 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0326 23:17:33.196866 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:17:33.196864 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:17:33.196812 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:17:33.196885 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:17:33.196893 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:17:33.196901 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:17:33.196912 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0326 23:17:33.196919 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:17:33.196919 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:17:33.196926 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0326 23:17:33.196931 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:17:33.196937 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:17:33.196939 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:17:33.196944 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0326 23:17:33.196944 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:17:33.196953 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:17:33.196807 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0326 23:17:33.196958 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:17:33.196822 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:17:33.196966 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0326 23:17:33.196971 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0326 23:17:33.196973 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:17:33.196978 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0326 23:17:33.196980 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:17:33.196982 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:17:33.196986 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0326 23:17:33.196989 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0326 23:17:33.196994 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0326 23:17:33.196922 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0326 23:17:33.196997 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0326 23:17:33.196988 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:17:33.196896 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:17:33.196952 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:17:33.197012 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:17:33.197020 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0326 23:17:33.196906 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:17:33.196984 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0326 23:17:33.196968 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:17:33.197068 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:17:33.197077 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0326 23:17:33.196868 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:17:33.196939 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:17:33.197092 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:17:33.196792 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:17:33.197119 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:17:33.197127 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:17:33.197127 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:17:33.197136 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:17:33.197137 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0326 23:17:33.197143 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:17:33.197148 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:17:33.196881 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:17:33.197146 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:17:33.197163 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:17:33.196961 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:17:33.197165 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:17:33.197228 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:17:33.196824 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0326 23:17:33.196882 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:17:33.197280 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:17:33.196999 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0326 23:17:33.197291 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:17:33.197295 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:17:33.197117 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:17:33.197155 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:17:33.197303 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:17:33.197314 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0326 23:17:33.197316 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0326 23:17:33.197314 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:17:33.197328 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0326 23:17:33.197331 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0326 23:17:33.197306 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:17:33.197349 3917 port_claim.go:71] Svc openshift-ingress/router-default: port 30390 is already open, no action needed I0326 23:17:33.197358 3917 port_claim.go:71] Svc openshift-ingress/router-default: port 32570 is already open, no action needed I0326 23:17:33.197155 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:17:33.197397 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:17:33.196974 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0326 23:17:33.197414 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:17:33.197300 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0326 23:17:33.197414 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0326 23:17:33.197426 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:17:33.197433 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0326 23:17:33.197456 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:17:33.197471 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:17:33.197489 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:17:33.197435 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:17:33.197500 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:17:33.197444 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:17:33.197532 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:17:33.197545 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:17:33.197320 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:17:33.197557 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:17:33.197451 3917 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:17:33.197608 3917 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:17:33.197616 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0326 23:17:33.244100 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0326 23:17:33.244124 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0326 23:17:33.244130 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0326 23:17:33.244137 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0326 23:17:33.244148 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0326 23:17:33.244157 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0326 23:17:33.244164 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0326 23:17:33.244172 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0326 23:17:33.244180 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0326 23:17:33.244189 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0326 23:17:33.244207 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0326 23:17:33.244230 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0326 23:17:33.244240 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0326 23:17:33.244248 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0326 23:17:33.244254 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0326 23:17:33.244262 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0326 23:17:33.287288 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0326 23:17:33.287305 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0326 23:17:33.328645 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0326 23:17:33.328685 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0326 23:17:33.328696 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0326 23:17:33.328704 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0326 23:17:33.328713 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0326 23:17:33.328721 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0326 23:17:33.328735 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0326 23:17:33.328761 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0326 23:17:33.328770 3917 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0326 23:17:35.346918 3917 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-167.ec2.internal I0326 23:17:35.346953 3917 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-167.ec2.internal: 40.435µs I0326 23:17:35.371770 3917 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-846cf54457-9ld2h] creating logical port openshift-image-registry_image-registry-846cf54457-9ld2h for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:35.371926 3917 kube.go:256] Updating pod openshift-image-registry/image-registry-846cf54457-9ld2h I0326 23:17:35.392926 3917 pod.go:62] [openshift-image-registry/image-registry-846cf54457-9ld2h] pod update took 21.035178ms I0326 23:17:35.392986 3917 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-846cf54457-9ld2h] addLogicalPort annotation time took 21.101211ms I0326 23:17:35.402641 3917 pods.go:271] [openshift-image-registry/image-registry-846cf54457-9ld2h] addLogicalPort took 30.903602ms, libovsdb time 8.777334ms I0326 23:17:35.403197 3917 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-gf69z] creating logical port openshift-dns_dns-default-gf69z for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:35.403524 3917 kube.go:256] Updating pod openshift-dns/dns-default-gf69z I0326 23:17:35.404212 3917 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-8gghz] creating logical port openshift-ingress-canary_ingress-canary-8gghz for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:35.404519 3917 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-8gghz I0326 23:17:35.420436 3917 pod.go:62] [openshift-ingress-canary/ingress-canary-8gghz] pod update took 15.989143ms I0326 23:17:35.420460 3917 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-8gghz] addLogicalPort annotation time took 16.018975ms I0326 23:17:35.421823 3917 pods.go:271] [openshift-ingress-canary/ingress-canary-8gghz] addLogicalPort took 17.628507ms, libovsdb time 1.018671ms I0326 23:17:35.421956 3917 pod.go:62] [openshift-dns/dns-default-gf69z] pod update took 18.601385ms I0326 23:17:35.421972 3917 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-gf69z] addLogicalPort annotation time took 18.621382ms I0326 23:17:35.423097 3917 pods.go:271] [openshift-dns/dns-default-gf69z] addLogicalPort took 19.983853ms, libovsdb time 819.734µs I0326 23:17:35.423470 3917 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-7c6d686446-2w85r] creating logical port openshift-ingress_router-default-7c6d686446-2w85r for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:35.423568 3917 kube.go:256] Updating pod openshift-ingress/router-default-7c6d686446-2w85r I0326 23:17:35.440453 3917 pod.go:62] [openshift-ingress/router-default-7c6d686446-2w85r] pod update took 16.926409ms I0326 23:17:35.440479 3917 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-7c6d686446-2w85r] addLogicalPort annotation time took 16.956385ms I0326 23:17:35.441876 3917 pods.go:271] [openshift-ingress/router-default-7c6d686446-2w85r] addLogicalPort took 18.410715ms, libovsdb time 1.040152ms I0326 23:17:36.016079 3917 cni.go:403] [openshift-ingress-canary/ingress-canary-8gghz 959cf41e51e08ab131ffed0faae086210d8da6c9b5aa65fd33a370c464a53773 network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-8gghz 959cf41e51e08ab131ffed0faae086210d8da6c9b5aa65fd33a370c464a53773 network default NAD default] I0326 23:17:36.063200 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-8gghz, hostIfaceName: 959cf41e51e08ab, network: default, NAD default, SandboxID: "959cf41e51e08ab131ffed0faae086210d8da6c9b5aa65fd33a370c464a53773", PCI device ID: , UID: "cb13861a-e701-4bc4-b5a6-b8c0a80d432f", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0326 23:17:36.122098 3917 cni.go:424] [openshift-ingress-canary/ingress-canary-8gghz 959cf41e51e08ab131ffed0faae086210d8da6c9b5aa65fd33a370c464a53773 network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-8gghz 959cf41e51e08ab131ffed0faae086210d8da6c9b5aa65fd33a370c464a53773 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:86:24:b7:a2:1b\",\"name\":\"959cf41e51e08ab\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d7f4fd8f-57a7-4d7f-9a08-6d64a4154eef\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:17:36.285936 3917 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-111.ec2.internal in network "default" I0326 23:17:36.285954 3917 default_network_controller.go:1008] Node "ip-10-0-2-111.ec2.internal" in remote zone "ip-10-0-2-111.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:17:36.285979 3917 node_tracker.go:235] Node ip-10-0-2-111.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-111.ec2.internal" I0326 23:17:36.285993 3917 node_tracker.go:169] Node ip-10-0-2-111.ec2.internal switch + router changed, syncing services in network "default" I0326 23:17:36.286008 3917 services_controller.go:574] Full service sync requested for network=default W0326 23:17:36.286342 3917 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:17:36.380196 3917 default_network_controller.go:1008] Node "ip-10-0-2-111.ec2.internal" in remote zone "ip-10-0-2-111.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:17:36.401505 3917 default_network_controller.go:1008] Node "ip-10-0-2-111.ec2.internal" in remote zone "ip-10-0-2-111.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:17:36.401529 3917 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-111.ec2.internal in network "default" I0326 23:17:36.401579 3917 node_tracker.go:169] Node ip-10-0-2-111.ec2.internal switch + router changed, syncing services in network "default" I0326 23:17:36.401592 3917 services_controller.go:574] Full service sync requested for network=default I0326 23:17:36.402166 3917 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-111.ec2.internal for the network default I0326 23:17:36.403341 3917 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-111.ec2.internal" on network "default" took: 1.20081ms I0326 23:17:37.980113 3917 cni.go:403] [openshift-network-diagnostics/network-check-target-xhk27 3b4017de475c209bf47071f93deca0476b22a20ef8505ea6634caae2c90f8cf2 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-xhk27 3b4017de475c209bf47071f93deca0476b22a20ef8505ea6634caae2c90f8cf2 network default NAD default] I0326 23:17:37.990294 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-xhk27, hostIfaceName: 3b4017de475c209, network: default, NAD default, SandboxID: "3b4017de475c209bf47071f93deca0476b22a20ef8505ea6634caae2c90f8cf2", PCI device ID: , UID: "7cdf3bbc-8991-4378-a055-0f575a2e040d", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0326 23:17:38.040154 3917 cni.go:424] [openshift-network-diagnostics/network-check-target-xhk27 3b4017de475c209bf47071f93deca0476b22a20ef8505ea6634caae2c90f8cf2 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-xhk27 3b4017de475c209bf47071f93deca0476b22a20ef8505ea6634caae2c90f8cf2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:cc:7e:53:b1:56\",\"name\":\"3b4017de475c209\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f2a85ad4-2af3-495c-ad28-1530e0754263\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:17:38.321007 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0326 23:17:38.451023 3917 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-111.ec2.internal in network "default" I0326 23:17:38.451089 3917 node_tracker.go:169] Node ip-10-0-2-111.ec2.internal switch + router changed, syncing services in network "default" I0326 23:17:38.451100 3917 egressfirewall.go:628] Syncing node "ip-10-0-2-111.ec2.internal" for egress firewall I0326 23:17:38.451104 3917 services_controller.go:574] Full service sync requested for network=default I0326 23:17:38.451106 3917 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-111.ec2.internal I0326 23:17:38.451120 3917 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-111.ec2.internal: 20.57µs I0326 23:17:38.451147 3917 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-111.ec2.internal", IP: 10.0.2.111 to PMTUD blocking rules I0326 23:17:38.451161 3917 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-111.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0326 23:17:38.451372 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0326 23:17:38.453691 3917 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:17:38.457448 3917 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-111.ec2.internal. Updating no re-route policies I0326 23:17:38.469026 3917 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-111.ec2.internal", IP: 10.0.2.111 to PMTUD blocking rules I0326 23:17:38.469047 3917 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-111.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:17:39.066770 3917 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0326 23:17:39.067738 3917 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 945.842µs I0326 23:17:39.084406 3917 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0326 23:17:39.158165 3917 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-q78pg] creating logical port openshift-kube-storage-version-migrator_migrator-854b5fb5c9-q78pg for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:39.158261 3917 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-854b5fb5c9-q78pg I0326 23:17:39.178864 3917 pod.go:62] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-q78pg] pod update took 20.628958ms I0326 23:17:39.178884 3917 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-q78pg] addLogicalPort annotation time took 20.654856ms I0326 23:17:39.180125 3917 pods.go:271] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-q78pg] addLogicalPort took 21.977138ms, libovsdb time 859.311µs I0326 23:17:39.415211 3917 namespace.go:98] [openshift-service-ca] adding namespace I0326 23:17:39.416189 3917 namespace.go:102] [openshift-service-ca] adding namespace took 949.191µs I0326 23:17:39.432272 3917 namespace.go:142] [openshift-service-ca] updating namespace I0326 23:17:39.556507 3917 cni.go:403] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-q78pg 4a9e6fbabe8c687d4037e8646c89122016409e2f03fdf9f759a72b8ee03fa6ad network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-q78pg 4a9e6fbabe8c687d4037e8646c89122016409e2f03fdf9f759a72b8ee03fa6ad network default NAD default] I0326 23:17:39.566587 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-854b5fb5c9-q78pg, hostIfaceName: 4a9e6fbabe8c687, network: default, NAD default, SandboxID: "4a9e6fbabe8c687d4037e8646c89122016409e2f03fdf9f759a72b8ee03fa6ad", PCI device ID: , UID: "1c19000b-5c42-47ac-b754-53ff04f1002e", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0326 23:17:39.610596 3917 cni.go:424] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-q78pg 4a9e6fbabe8c687d4037e8646c89122016409e2f03fdf9f759a72b8ee03fa6ad network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-q78pg 4a9e6fbabe8c687d4037e8646c89122016409e2f03fdf9f759a72b8ee03fa6ad network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:af:86:a6:76:75\",\"name\":\"4a9e6fbabe8c687\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a7ae6df5-ad08-4964-8072-7337ac9f186c\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:17:41.084039 3917 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-f59fc589b-5d74p] creating logical port openshift-service-ca_service-ca-f59fc589b-5d74p for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:41.084183 3917 kube.go:256] Updating pod openshift-service-ca/service-ca-f59fc589b-5d74p I0326 23:17:41.096962 3917 pod.go:62] [openshift-service-ca/service-ca-f59fc589b-5d74p] pod update took 12.80757ms I0326 23:17:41.096986 3917 base_network_controller_pods.go:937] [default/openshift-service-ca/service-ca-f59fc589b-5d74p] addLogicalPort annotation time took 12.833322ms I0326 23:17:41.098214 3917 pods.go:271] [openshift-service-ca/service-ca-f59fc589b-5d74p] addLogicalPort took 14.191348ms, libovsdb time 933.192µs I0326 23:17:41.177332 3917 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-111.ec2.internal I0326 23:17:41.177436 3917 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-111.ec2.internal: 111.618µs I0326 23:17:41.480143 3917 cni.go:403] [openshift-service-ca/service-ca-f59fc589b-5d74p 268387ef6d4e98eaacab1689090666600901c58e4ccbea95c46145133426fee6 network default NAD default] ADD starting CNI request [openshift-service-ca/service-ca-f59fc589b-5d74p 268387ef6d4e98eaacab1689090666600901c58e4ccbea95c46145133426fee6 network default NAD default] I0326 23:17:41.491619 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-f59fc589b-5d74p, hostIfaceName: 268387ef6d4e98e, network: default, NAD default, SandboxID: "268387ef6d4e98eaacab1689090666600901c58e4ccbea95c46145133426fee6", PCI device ID: , UID: "3f3a3200-ec22-4acd-a282-0098fa3ca8c8", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0326 23:17:41.533702 3917 cni.go:424] [openshift-service-ca/service-ca-f59fc589b-5d74p 268387ef6d4e98eaacab1689090666600901c58e4ccbea95c46145133426fee6 network default NAD default] ADD finished CNI request [openshift-service-ca/service-ca-f59fc589b-5d74p 268387ef6d4e98eaacab1689090666600901c58e4ccbea95c46145133426fee6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:dd:52:81:74:08\",\"name\":\"268387ef6d4e98e\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fc1d6791-f829-4623-83a9-62479675daf3\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:17:41.728372 3917 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:17:43.113867 3917 cni.go:403] [kube-system/global-pull-secret-syncer-s8pvt c8d1191e7a61f612502037c695bb8d6b90754b22560e788b0b192fb9d86754cf network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-s8pvt c8d1191e7a61f612502037c695bb8d6b90754b22560e788b0b192fb9d86754cf network default NAD default] I0326 23:17:43.123117 3917 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-s8pvt, hostIfaceName: c8d1191e7a61f61, network: default, NAD default, SandboxID: "c8d1191e7a61f612502037c695bb8d6b90754b22560e788b0b192fb9d86754cf", PCI device ID: , UID: "6b378780-e020-4a30-b96d-d4ed5cc26ab5", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0326 23:17:43.166042 3917 cni.go:424] [kube-system/global-pull-secret-syncer-s8pvt c8d1191e7a61f612502037c695bb8d6b90754b22560e788b0b192fb9d86754cf network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-s8pvt c8d1191e7a61f612502037c695bb8d6b90754b22560e788b0b192fb9d86754cf network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:b9:53:e0:11:c0\",\"name\":\"c8d1191e7a61f61\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6178b494-bc71-423a-89aa-7c9191fdb6e5\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:17:43.721643 3917 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-lxd97] creating logical port openshift-insights_insights-runtime-extractor-lxd97 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:17:43.721797 3917 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-lxd97 I0326 23:17:43.740579 3917 pod.go:62] [openshift-insights/insights-runtime-extractor-lxd97] pod update took 18.811598ms I0326 23:17:43.740601 3917 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-lxd97] addLogicalPort annotation time took 18.852465ms I0326 23:17:43.741677 3917 pods.go:271] [openshift-insights/insights-runtime-extractor-lxd97] addLogicalPort took 20.048282ms, libovsdb time 762.326µs W0326 23:17:48.651972 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0326 23:17:51.392265 3917 cni.go:403] [openshift-image-registry/image-registry-846cf54457-9ld2h 34a1f82346512898f1fcd824a60e107aac1b2c74f238f53bf04cb5eed8ce87a4 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-846cf54457-9ld2h 34a1f82346512898f1fcd824a60e107aac1b2c74f238f53bf04cb5eed8ce87a4 network default NAD default] I0326 23:17:51.403088 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-846cf54457-9ld2h, hostIfaceName: 34a1f8234651289, network: default, NAD default, SandboxID: "34a1f82346512898f1fcd824a60e107aac1b2c74f238f53bf04cb5eed8ce87a4", PCI device ID: , UID: "2f5f9b67-5de1-4ecf-8bf8-a6fc79096f63", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0326 23:17:51.450522 3917 cni.go:424] [openshift-image-registry/image-registry-846cf54457-9ld2h 34a1f82346512898f1fcd824a60e107aac1b2c74f238f53bf04cb5eed8ce87a4 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-846cf54457-9ld2h 34a1f82346512898f1fcd824a60e107aac1b2c74f238f53bf04cb5eed8ce87a4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:f9:33:86:b7:62\",\"name\":\"34a1f8234651289\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2d5e8f5e-68c3-4754-ad5c-68879b3d1a97\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:17:51.660341 3917 cni.go:403] [openshift-insights/insights-runtime-extractor-lxd97 2af16b527a9d17536d1e110c5779ffc1506fb1057360949e71423ee544b7dcd8 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-lxd97 2af16b527a9d17536d1e110c5779ffc1506fb1057360949e71423ee544b7dcd8 network default NAD default] I0326 23:17:51.673377 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-lxd97, hostIfaceName: 2af16b527a9d175, network: default, NAD default, SandboxID: "2af16b527a9d17536d1e110c5779ffc1506fb1057360949e71423ee544b7dcd8", PCI device ID: , UID: "9a89b570-d4fc-47a6-a796-5e2f3f3ffae7", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0326 23:17:51.705948 3917 cni.go:403] [openshift-dns/dns-default-gf69z 80a71ab113efebf32d727e3a502c96b277f9e3800e3b2426d5e58e90de8705f2 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-gf69z 80a71ab113efebf32d727e3a502c96b277f9e3800e3b2426d5e58e90de8705f2 network default NAD default] I0326 23:17:51.718144 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-gf69z, hostIfaceName: 80a71ab113efebf, network: default, NAD default, SandboxID: "80a71ab113efebf32d727e3a502c96b277f9e3800e3b2426d5e58e90de8705f2", PCI device ID: , UID: "062e2207-bd0f-469a-a722-ad000371c6b9", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0326 23:17:51.721630 3917 cni.go:424] [openshift-insights/insights-runtime-extractor-lxd97 2af16b527a9d17536d1e110c5779ffc1506fb1057360949e71423ee544b7dcd8 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-lxd97 2af16b527a9d17536d1e110c5779ffc1506fb1057360949e71423ee544b7dcd8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:d8:07:2e:5c:1e\",\"name\":\"2af16b527a9d175\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7acea6fe-eb73-4026-849d-592371405c64\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:17:51.764796 3917 cni.go:424] [openshift-dns/dns-default-gf69z 80a71ab113efebf32d727e3a502c96b277f9e3800e3b2426d5e58e90de8705f2 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-gf69z 80a71ab113efebf32d727e3a502c96b277f9e3800e3b2426d5e58e90de8705f2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:66:a6:c9:24:ff\",\"name\":\"80a71ab113efebf\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/45fdd63f-200d-40cd-9153-56533738104b\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:18:02.418553 3917 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0326 23:18:02.418578 3917 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0326 23:18:02.418583 3917 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0326 23:18:02.426712 3917 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:18:02.426775 3917 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0326 23:18:02.496671 3917 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:18:02.525456 3917 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-7bfrw] creating logical port openshift-console_downloads-56d45c498b-7bfrw for pod on switch ip-10-0-1-167.ec2.internal I0326 23:18:02.525552 3917 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-7bfrw W0326 23:18:02.565391 3917 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:18:02.576879 3917 pod.go:62] [openshift-console/downloads-56d45c498b-7bfrw] pod update took 51.356318ms I0326 23:18:02.576911 3917 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-7bfrw] addLogicalPort annotation time took 51.39194ms I0326 23:18:02.578062 3917 pods.go:271] [openshift-console/downloads-56d45c498b-7bfrw] addLogicalPort took 52.630938ms, libovsdb time 851.636µs I0326 23:18:02.882860 3917 cni.go:403] [openshift-console/downloads-56d45c498b-7bfrw 068d003300e7faff173b6002f5ea43a8f9b33372cce74b70c48937cda61ecc48 network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-7bfrw 068d003300e7faff173b6002f5ea43a8f9b33372cce74b70c48937cda61ecc48 network default NAD default] I0326 23:18:02.891325 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-7bfrw, hostIfaceName: 068d003300e7faf, network: default, NAD default, SandboxID: "068d003300e7faff173b6002f5ea43a8f9b33372cce74b70c48937cda61ecc48", PCI device ID: , UID: "e89ecb06-7b9d-494f-8cdd-30253f41caf0", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0326 23:18:02.934275 3917 cni.go:424] [openshift-console/downloads-56d45c498b-7bfrw 068d003300e7faff173b6002f5ea43a8f9b33372cce74b70c48937cda61ecc48 network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-7bfrw 068d003300e7faff173b6002f5ea43a8f9b33372cce74b70c48937cda61ecc48 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:8f:b8:17:98:5f\",\"name\":\"068d003300e7faf\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0db1c6c4-2d05-4d89-86f8-a6f79222a6fe\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:18:03.448624 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0326 23:18:03.448644 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0326 23:18:03.454554 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:18:03.454568 3917 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:18:03.454574 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:18:03.486508 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0326 23:18:03.486525 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0326 23:18:03.491477 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:18:03.491490 3917 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:18:03.491496 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:18:04.785607 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:18:04.785629 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0326 23:18:04.785634 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0326 23:18:06.334995 3917 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7dkwq] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-7dkwq for pod on switch ip-10-0-1-167.ec2.internal I0326 23:18:06.335102 3917 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7dkwq I0326 23:18:06.351044 3917 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7dkwq] pod update took 15.975686ms I0326 23:18:06.351126 3917 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7dkwq] addLogicalPort annotation time took 16.060348ms I0326 23:18:06.352412 3917 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7dkwq] addLogicalPort took 17.43421ms, libovsdb time 897.558µs I0326 23:18:07.340513 3917 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7dkwq 550bfd30f4e9ca5a64874b9b64c4fc88e7576390bd8de83c61ca3097d677d8b6 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7dkwq 550bfd30f4e9ca5a64874b9b64c4fc88e7576390bd8de83c61ca3097d677d8b6 network default NAD default] I0326 23:18:07.349439 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-7dkwq, hostIfaceName: 550bfd30f4e9ca5, network: default, NAD default, SandboxID: "550bfd30f4e9ca5a64874b9b64c4fc88e7576390bd8de83c61ca3097d677d8b6", PCI device ID: , UID: "91f50465-e9a9-4efb-bb87-681c2c993fc3", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0326 23:18:07.399471 3917 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7dkwq 550bfd30f4e9ca5a64874b9b64c4fc88e7576390bd8de83c61ca3097d677d8b6 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-7dkwq 550bfd30f4e9ca5a64874b9b64c4fc88e7576390bd8de83c61ca3097d677d8b6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:87:4e:c0:e8:4c\",\"name\":\"550bfd30f4e9ca5\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2e2b6b4c-1267-4ee5-affb-b8f39b28a82c\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:18:07.663568 3917 cni.go:403] [openshift-ingress/router-default-7c6d686446-2w85r 4dd369835b3bff47dc9a6f372776f81e3e4003e302d375057edabf1485f9e761 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-7c6d686446-2w85r 4dd369835b3bff47dc9a6f372776f81e3e4003e302d375057edabf1485f9e761 network default NAD default] I0326 23:18:07.672836 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-7c6d686446-2w85r, hostIfaceName: 4dd369835b3bff4, network: default, NAD default, SandboxID: "4dd369835b3bff47dc9a6f372776f81e3e4003e302d375057edabf1485f9e761", PCI device ID: , UID: "4645b1ab-4447-4fae-895f-166b12320ea6", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0326 23:18:07.717893 3917 cni.go:424] [openshift-ingress/router-default-7c6d686446-2w85r 4dd369835b3bff47dc9a6f372776f81e3e4003e302d375057edabf1485f9e761 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-7c6d686446-2w85r 4dd369835b3bff47dc9a6f372776f81e3e4003e302d375057edabf1485f9e761 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:f4:81:1e:22:44\",\"name\":\"4dd369835b3bff4\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d982c6a1-6247-4e4d-805d-fd9fff14a6c3\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:18:08.835527 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0326 23:18:08.835560 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0326 23:18:09.338660 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0326 23:18:09.338694 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:18:09.515370 3917 cni.go:403] [openshift-multus/network-metrics-daemon-ctl7v 69ff171acd5988df69da68e7054142e021628aefb12e9100c895d7f18dba1343 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-ctl7v 69ff171acd5988df69da68e7054142e021628aefb12e9100c895d7f18dba1343 network default NAD default] I0326 23:18:09.524495 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-ctl7v, hostIfaceName: 69ff171acd5988d, network: default, NAD default, SandboxID: "69ff171acd5988df69da68e7054142e021628aefb12e9100c895d7f18dba1343", PCI device ID: , UID: "43cd2303-a570-4cbe-a962-ccc543fba400", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0326 23:18:09.568983 3917 cni.go:424] [openshift-multus/network-metrics-daemon-ctl7v 69ff171acd5988df69da68e7054142e021628aefb12e9100c895d7f18dba1343 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-ctl7v 69ff171acd5988df69da68e7054142e021628aefb12e9100c895d7f18dba1343 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:41:09:fc:b5:d2\",\"name\":\"69ff171acd5988d\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c252ec51-38c9-4151-adb8-6269eb19da76\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:18:09.833723 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0326 23:18:09.868338 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:18:09.868424 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:18:09.868439 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:18:09.868620 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0326 23:18:09.868631 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0326 23:18:10.008708 3917 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0326 23:18:10.047815 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0326 23:18:10.065649 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0326 23:18:10.065674 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0326 23:18:10.851237 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0326 23:18:11.361622 3917 base_network_controller_pods.go:487] [default/openshift-console/console-5687f476f-d6qdg] creating logical port openshift-console_console-5687f476f-d6qdg for pod on switch ip-10-0-1-167.ec2.internal I0326 23:18:11.361767 3917 kube.go:256] Updating pod openshift-console/console-5687f476f-d6qdg I0326 23:18:11.382009 3917 pod.go:62] [openshift-console/console-5687f476f-d6qdg] pod update took 20.299618ms I0326 23:18:11.382038 3917 base_network_controller_pods.go:937] [default/openshift-console/console-5687f476f-d6qdg] addLogicalPort annotation time took 20.331617ms I0326 23:18:11.383393 3917 pods.go:271] [openshift-console/console-5687f476f-d6qdg] addLogicalPort took 21.788969ms, libovsdb time 967.939µs W0326 23:18:12.532848 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:18:12.532873 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:18:12.788117 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:18:13.731931 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:18:17.860948 3917 cni.go:403] [openshift-console/console-5687f476f-d6qdg 53e838c8011f0350e58e05db9ebf6d079fc7f373253e82b9e3c952dcb568327f network default NAD default] ADD starting CNI request [openshift-console/console-5687f476f-d6qdg 53e838c8011f0350e58e05db9ebf6d079fc7f373253e82b9e3c952dcb568327f network default NAD default] I0326 23:18:17.869712 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5687f476f-d6qdg, hostIfaceName: 53e838c8011f035, network: default, NAD default, SandboxID: "53e838c8011f0350e58e05db9ebf6d079fc7f373253e82b9e3c952dcb568327f", PCI device ID: , UID: "e7bddf3b-f84b-402c-9ad1-1bdfc6ab7418", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0326 23:18:17.911699 3917 cni.go:424] [openshift-console/console-5687f476f-d6qdg 53e838c8011f0350e58e05db9ebf6d079fc7f373253e82b9e3c952dcb568327f network default NAD default] ADD finished CNI request [openshift-console/console-5687f476f-d6qdg 53e838c8011f0350e58e05db9ebf6d079fc7f373253e82b9e3c952dcb568327f network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:cc:f8:e9:c6:d1\",\"name\":\"53e838c8011f035\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/57a2e3c7-8aec-44ed-b799-def32f6bc752\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:18:18.877092 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0326 23:18:19.077680 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0326 23:18:19.107564 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0326 23:18:19.107592 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0326 23:18:19.515684 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0326 23:18:19.903023 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0326 23:18:20.886902 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0326 23:18:21.899026 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0326 23:18:25.042944 3917 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:18:25.043055 3917 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0326 23:18:25.065163 3917 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 22.150509ms I0326 23:18:25.065183 3917 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 22.176509ms I0326 23:18:25.066466 3917 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 23.538228ms, libovsdb time 947.255µs I0326 23:18:26.001280 3917 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-56df9666dd-m26v2] creating logical port openshift-monitoring_thanos-querier-56df9666dd-m26v2 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:18:26.001406 3917 kube.go:256] Updating pod openshift-monitoring/thanos-querier-56df9666dd-m26v2 I0326 23:18:26.022838 3917 pod.go:62] [openshift-monitoring/thanos-querier-56df9666dd-m26v2] pod update took 21.474264ms I0326 23:18:26.022870 3917 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-56df9666dd-m26v2] addLogicalPort annotation time took 21.509911ms I0326 23:18:26.024315 3917 pods.go:271] [openshift-monitoring/thanos-querier-56df9666dd-m26v2] addLogicalPort took 23.058026ms, libovsdb time 1.009442ms I0326 23:18:26.033861 3917 cni.go:403] [openshift-monitoring/alertmanager-main-1 908879e0b81a07a796f0fe390167303692b481b614d17628060cd502934c5fa4 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 908879e0b81a07a796f0fe390167303692b481b614d17628060cd502934c5fa4 network default NAD default] I0326 23:18:26.044252 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 908879e0b81a07a, network: default, NAD default, SandboxID: "908879e0b81a07a796f0fe390167303692b481b614d17628060cd502934c5fa4", PCI device ID: , UID: "bcd2c6db-3f9d-48e8-b9c2-1fffd11bd536", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0326 23:18:26.092166 3917 cni.go:424] [openshift-monitoring/alertmanager-main-1 908879e0b81a07a796f0fe390167303692b481b614d17628060cd502934c5fa4 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 908879e0b81a07a796f0fe390167303692b481b614d17628060cd502934c5fa4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:98:24:36:16:59\",\"name\":\"908879e0b81a07a\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/30940da6-0e08-454b-a1a6-5d368443deeb\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:18:26.376102 3917 cni.go:403] [openshift-monitoring/thanos-querier-56df9666dd-m26v2 ae505bbcee44b17e0ef24d38c31788340deeb725f4bd747645e97cbce1615cdb network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-56df9666dd-m26v2 ae505bbcee44b17e0ef24d38c31788340deeb725f4bd747645e97cbce1615cdb network default NAD default] I0326 23:18:26.391056 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-56df9666dd-m26v2, hostIfaceName: ae505bbcee44b17, network: default, NAD default, SandboxID: "ae505bbcee44b17e0ef24d38c31788340deeb725f4bd747645e97cbce1615cdb", PCI device ID: , UID: "34ee358e-e464-4b6c-aca0-0ab056d6758d", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0326 23:18:26.438285 3917 cni.go:424] [openshift-monitoring/thanos-querier-56df9666dd-m26v2 ae505bbcee44b17e0ef24d38c31788340deeb725f4bd747645e97cbce1615cdb network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-56df9666dd-m26v2 ae505bbcee44b17e0ef24d38c31788340deeb725f4bd747645e97cbce1615cdb network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:b7:99:51:fe:d3\",\"name\":\"ae505bbcee44b17\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fe47c05f-f169-403e-9c0b-ed563a55dee6\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:18:27.840004 3917 obj_retry.go:464] Detected object openshift-image-registry/image-registry-846cf54457-bd8vj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:27.840656 3917 obj_retry.go:464] Detected object openshift-image-registry/image-registry-846cf54457-bd8vj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:27.853735 3917 obj_retry.go:464] Detected object openshift-image-registry/image-registry-57b8c45f46-knn5j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:27.854350 3917 obj_retry.go:464] Detected object openshift-image-registry/image-registry-57b8c45f46-knn5j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:18:27.979634 3917 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:27.979662 3917 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:27.979671 3917 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:28.190090 3917 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:28.190109 3917 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:28.190115 3917 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0326 23:18:28.202053 3917 obj_retry.go:458] Detected object openshift-image-registry/image-registry-57b8c45f46-knn5j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:28.202078 3917 obj_retry.go:458] Detected object openshift-image-registry/image-registry-57b8c45f46-knn5j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:28.210692 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-57b8c45f46-knn5j I0326 23:18:28.210727 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-57b8c45f46-knn5j I0326 23:18:28.462797 3917 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-674796d4d-4h9n6] creating logical port openshift-monitoring_metrics-server-674796d4d-4h9n6 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:18:28.462906 3917 kube.go:256] Updating pod openshift-monitoring/metrics-server-674796d4d-4h9n6 I0326 23:18:28.487086 3917 pod.go:62] [openshift-monitoring/metrics-server-674796d4d-4h9n6] pod update took 24.208127ms I0326 23:18:28.487107 3917 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-674796d4d-4h9n6] addLogicalPort annotation time took 24.23472ms I0326 23:18:28.488393 3917 pods.go:271] [openshift-monitoring/metrics-server-674796d4d-4h9n6] addLogicalPort took 25.617323ms, libovsdb time 933.279µs I0326 23:18:28.584967 3917 obj_retry.go:458] Detected object openshift-image-registry/image-registry-846cf54457-bd8vj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:28.585018 3917 obj_retry.go:458] Detected object openshift-image-registry/image-registry-846cf54457-bd8vj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:28.592540 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-846cf54457-bd8vj I0326 23:18:28.592572 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-846cf54457-bd8vj I0326 23:18:28.660361 3917 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-5gskl] creating logical port openshift-monitoring_monitoring-plugin-66fd6dc9f9-5gskl for pod on switch ip-10-0-1-167.ec2.internal I0326 23:18:28.660447 3917 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-66fd6dc9f9-5gskl I0326 23:18:28.682121 3917 pod.go:62] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-5gskl] pod update took 21.700487ms I0326 23:18:28.682141 3917 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-5gskl] addLogicalPort annotation time took 21.725907ms I0326 23:18:28.683153 3917 pods.go:271] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-5gskl] addLogicalPort took 22.805666ms, libovsdb time 730.179µs I0326 23:18:28.821370 3917 cni.go:403] [openshift-monitoring/metrics-server-674796d4d-4h9n6 84a26bb40d5e7ccea3739d23db524b3aef373c7e727c4688eafe5bb9079eca93 network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-674796d4d-4h9n6 84a26bb40d5e7ccea3739d23db524b3aef373c7e727c4688eafe5bb9079eca93 network default NAD default] I0326 23:18:28.830842 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-674796d4d-4h9n6, hostIfaceName: 84a26bb40d5e7cc, network: default, NAD default, SandboxID: "84a26bb40d5e7ccea3739d23db524b3aef373c7e727c4688eafe5bb9079eca93", PCI device ID: , UID: "2d62936d-4ccc-4717-b23f-be2f121af847", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0326 23:18:28.875619 3917 cni.go:424] [openshift-monitoring/metrics-server-674796d4d-4h9n6 84a26bb40d5e7ccea3739d23db524b3aef373c7e727c4688eafe5bb9079eca93 network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-674796d4d-4h9n6 84a26bb40d5e7ccea3739d23db524b3aef373c7e727c4688eafe5bb9079eca93 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:95:b2:69:8e:f5\",\"name\":\"84a26bb40d5e7cc\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/09e74e92-dc0e-46a8-8b8d-f53f5cda3e26\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:18:29.930521 3917 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-857649cf56-nf2cl IP address from the namespace address-set, err: pod openshift-console/console-857649cf56-nf2cl: no pod IPs found I0326 23:18:30.186795 3917 cni.go:403] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-5gskl 25d1a2bcaaaa7a78b0dbcefe0539366fc2939b221550425e16575057ee4ea973 network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-5gskl 25d1a2bcaaaa7a78b0dbcefe0539366fc2939b221550425e16575057ee4ea973 network default NAD default] I0326 23:18:30.195217 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-66fd6dc9f9-5gskl, hostIfaceName: 25d1a2bcaaaa7a7, network: default, NAD default, SandboxID: "25d1a2bcaaaa7a78b0dbcefe0539366fc2939b221550425e16575057ee4ea973", PCI device ID: , UID: "2f2340de-57b7-44f9-9677-1ef0cc5a0b88", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0326 23:18:30.239327 3917 cni.go:424] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-5gskl 25d1a2bcaaaa7a78b0dbcefe0539366fc2939b221550425e16575057ee4ea973 network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-5gskl 25d1a2bcaaaa7a78b0dbcefe0539366fc2939b221550425e16575057ee4ea973 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:ee:7a:47:0f:9a\",\"name\":\"25d1a2bcaaaa7a7\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/305add4d-624b-4644-8ddb-32065dac2d58\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:18:30.361005 3917 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:18:30.361162 3917 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0326 23:18:30.408314 3917 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 47.198432ms I0326 23:18:30.408411 3917 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 47.299652ms I0326 23:18:30.409786 3917 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 48.794714ms, libovsdb time 943.977µs I0326 23:18:30.753948 3917 cni.go:403] [openshift-monitoring/prometheus-k8s-1 49405f2253b34239ea855daf16057458c9fe4f56c410d30aba641eeeae755306 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 49405f2253b34239ea855daf16057458c9fe4f56c410d30aba641eeeae755306 network default NAD default] I0326 23:18:30.763077 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 49405f2253b3423, network: default, NAD default, SandboxID: "49405f2253b34239ea855daf16057458c9fe4f56c410d30aba641eeeae755306", PCI device ID: , UID: "1f34878b-0fa8-45ef-9099-c2254943785b", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0326 23:18:30.812470 3917 cni.go:424] [openshift-monitoring/prometheus-k8s-1 49405f2253b34239ea855daf16057458c9fe4f56c410d30aba641eeeae755306 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 49405f2253b34239ea855daf16057458c9fe4f56c410d30aba641eeeae755306 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:3f:69:45:5d:7b\",\"name\":\"49405f2253b3423\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6cd959e8-acc4-4def-bd43-c299f038c2db\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:18:30.951776 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:18:31.969996 3917 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:31.970020 3917 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:31.970027 3917 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:31.970034 3917 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:32.056010 3917 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:18:32.056029 3917 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:18:32.971627 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0326 23:18:32.997485 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0326 23:18:35.208125 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:18:35.258004 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:35.258024 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:35.258032 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:35.258039 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:35.258056 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:35.258062 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:35.258069 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:35.258075 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:35.326767 3917 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:18:35.326794 3917 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0326 23:18:36.379945 3917 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390kvhwp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:36.380495 3917 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390kvhwp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:36.380529 3917 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390kvhwp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:18:36.380538 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0326 23:18:36.380551 3917 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390kvhwp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:18:36.380557 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0326 23:18:37.407670 3917 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390kvhwp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:37.407702 3917 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390kvhwp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:37.407717 3917 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390kvhwp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:37.407738 3917 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390kvhwp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:18:37.988794 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:37.988817 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:37.988825 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:18:37.988831 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier I0326 23:18:41.048127 3917 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0326 23:18:41.061253 3917 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0326 23:18:43.618276 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0326 23:18:43.640827 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0326 23:18:47.403854 3917 obj_retry.go:464] Detected object openshift-console/console-5687f476f-zm77w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:47.404385 3917 obj_retry.go:464] Detected object openshift-console/console-5687f476f-zm77w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:48.305842 3917 obj_retry.go:458] Detected object openshift-console/console-5687f476f-zm77w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:48.305866 3917 obj_retry.go:458] Detected object openshift-console/console-5687f476f-zm77w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:48.313068 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5687f476f-zm77w I0326 23:18:48.313091 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5687f476f-zm77w W0326 23:18:51.974798 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server I0326 23:18:52.780873 3917 cni.go:403] [openshift-console/console-5687f476f-d6qdg 53e838c8011f0350e58e05db9ebf6d079fc7f373253e82b9e3c952dcb568327f network default NAD default] DEL starting CNI request [openshift-console/console-5687f476f-d6qdg 53e838c8011f0350e58e05db9ebf6d079fc7f373253e82b9e3c952dcb568327f network default NAD default] I0326 23:18:52.854950 3917 cni.go:424] [openshift-console/console-5687f476f-d6qdg 53e838c8011f0350e58e05db9ebf6d079fc7f373253e82b9e3c952dcb568327f network default NAD default] DEL finished CNI request [openshift-console/console-5687f476f-d6qdg 53e838c8011f0350e58e05db9ebf6d079fc7f373253e82b9e3c952dcb568327f network default NAD default], result "{}", err I0326 23:18:52.926212 3917 obj_retry.go:464] Detected object openshift-console/console-5687f476f-d6qdg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:52.926272 3917 pods.go:176] Deleting pod: openshift-console/console-5687f476f-d6qdg I0326 23:18:52.927569 3917 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5687f476f-d6qdg, ips: 10.129.0.15 I0326 23:18:52.927597 3917 obj_retry.go:464] Detected object openshift-console/console-5687f476f-d6qdg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:52.936879 3917 base_network_controller_pods.go:487] [default/openshift-console/console-6d5d5dc7c4-2fqzf] creating logical port openshift-console_console-6d5d5dc7c4-2fqzf for pod on switch ip-10-0-1-167.ec2.internal I0326 23:18:52.937046 3917 kube.go:256] Updating pod openshift-console/console-6d5d5dc7c4-2fqzf I0326 23:18:52.953813 3917 pod.go:62] [openshift-console/console-6d5d5dc7c4-2fqzf] pod update took 16.802299ms I0326 23:18:52.953836 3917 base_network_controller_pods.go:937] [default/openshift-console/console-6d5d5dc7c4-2fqzf] addLogicalPort annotation time took 16.829536ms I0326 23:18:52.955070 3917 pods.go:271] [openshift-console/console-6d5d5dc7c4-2fqzf] addLogicalPort took 18.204702ms, libovsdb time 898.142µs I0326 23:18:53.053523 3917 obj_retry.go:458] Detected object openshift-console/console-5687f476f-d6qdg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:53.053549 3917 obj_retry.go:458] Detected object openshift-console/console-5687f476f-d6qdg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:53.056849 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5687f476f-d6qdg I0326 23:18:53.056870 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5687f476f-d6qdg I0326 23:18:53.350212 3917 cni.go:403] [openshift-console/console-6d5d5dc7c4-2fqzf 0a55a902aa9adeeb0476f030d57e796190e64179e449f22cc26e72562403c335 network default NAD default] ADD starting CNI request [openshift-console/console-6d5d5dc7c4-2fqzf 0a55a902aa9adeeb0476f030d57e796190e64179e449f22cc26e72562403c335 network default NAD default] I0326 23:18:53.359351 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-6d5d5dc7c4-2fqzf, hostIfaceName: 0a55a902aa9adee, network: default, NAD default, SandboxID: "0a55a902aa9adeeb0476f030d57e796190e64179e449f22cc26e72562403c335", PCI device ID: , UID: "efdcea5e-1f6d-40f1-bd82-abb178e807dc", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0326 23:18:53.405674 3917 cni.go:424] [openshift-console/console-6d5d5dc7c4-2fqzf 0a55a902aa9adeeb0476f030d57e796190e64179e449f22cc26e72562403c335 network default NAD default] ADD finished CNI request [openshift-console/console-6d5d5dc7c4-2fqzf 0a55a902aa9adeeb0476f030d57e796190e64179e449f22cc26e72562403c335 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:29:61:66:1c:21\",\"name\":\"0a55a902aa9adee\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/96ada6a7-33d2-4589-9a79-d32e4c52e16f\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:18:54.667873 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0326 23:18:56.026021 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:56.026046 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:56.026054 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:56.026068 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:56.026075 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:56.026081 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:56.084256 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:56.084275 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:56.084280 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:18:58.343120 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:18:58.343142 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:18:58.343158 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:18:58.343197 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:18:58.384316 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:18:58.384340 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:19:04.116934 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0326 23:19:15.391576 3917 cni.go:403] [openshift-image-registry/image-registry-846cf54457-9ld2h 34a1f82346512898f1fcd824a60e107aac1b2c74f238f53bf04cb5eed8ce87a4 network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-846cf54457-9ld2h 34a1f82346512898f1fcd824a60e107aac1b2c74f238f53bf04cb5eed8ce87a4 network default NAD default] I0326 23:19:15.491813 3917 cni.go:424] [openshift-image-registry/image-registry-846cf54457-9ld2h 34a1f82346512898f1fcd824a60e107aac1b2c74f238f53bf04cb5eed8ce87a4 network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-846cf54457-9ld2h 34a1f82346512898f1fcd824a60e107aac1b2c74f238f53bf04cb5eed8ce87a4 network default NAD default], result "{}", err I0326 23:19:15.544925 3917 obj_retry.go:464] Detected object openshift-image-registry/image-registry-846cf54457-9ld2h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:19:15.544962 3917 pods.go:176] Deleting pod: openshift-image-registry/image-registry-846cf54457-9ld2h I0326 23:19:15.546344 3917 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-846cf54457-9ld2h, ips: 10.129.0.6 I0326 23:19:15.546372 3917 obj_retry.go:464] Detected object openshift-image-registry/image-registry-846cf54457-9ld2h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:19:16.152799 3917 obj_retry.go:458] Detected object openshift-image-registry/image-registry-846cf54457-9ld2h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:19:16.152826 3917 obj_retry.go:458] Detected object openshift-image-registry/image-registry-846cf54457-9ld2h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:19:16.156123 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-846cf54457-9ld2h I0326 23:19:16.156155 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-846cf54457-9ld2h I0326 23:19:20.317794 3917 obj_retry.go:464] Detected object openshift-ingress/router-default-6d457db745-rgdsd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:19:20.318368 3917 obj_retry.go:464] Detected object openshift-ingress/router-default-6d457db745-rgdsd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:19:20.769228 3917 obj_retry.go:458] Detected object openshift-ingress/router-default-6d457db745-rgdsd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:19:20.769254 3917 obj_retry.go:458] Detected object openshift-ingress/router-default-6d457db745-rgdsd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:19:20.775477 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-6d457db745-rgdsd I0326 23:19:20.775504 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-6d457db745-rgdsd I0326 23:19:23.663838 3917 obj_retry.go:464] Detected object openshift-console/console-857649cf56-4qwzn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:19:23.664397 3917 obj_retry.go:464] Detected object openshift-console/console-857649cf56-4qwzn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:19:23.781133 3917 obj_retry.go:458] Detected object openshift-console/console-857649cf56-4qwzn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:19:23.781166 3917 obj_retry.go:458] Detected object openshift-console/console-857649cf56-4qwzn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:19:23.785538 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-857649cf56-4qwzn I0326 23:19:23.785558 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-857649cf56-4qwzn W0326 23:19:31.254469 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:19:31.254494 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0326 23:19:45.691661 3917 cni.go:403] [openshift-monitoring/alertmanager-main-1 908879e0b81a07a796f0fe390167303692b481b614d17628060cd502934c5fa4 network default NAD default] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 908879e0b81a07a796f0fe390167303692b481b614d17628060cd502934c5fa4 network default NAD default] I0326 23:19:45.767146 3917 cni.go:424] [openshift-monitoring/alertmanager-main-1 908879e0b81a07a796f0fe390167303692b481b614d17628060cd502934c5fa4 network default NAD default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 908879e0b81a07a796f0fe390167303692b481b614d17628060cd502934c5fa4 network default NAD default], result "{}", err I0326 23:19:45.829843 3917 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:19:45.829874 3917 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0326 23:19:45.831067 3917 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.129.0.16 I0326 23:19:45.831093 3917 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:19:46.286824 3917 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:19:46.286854 3917 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:19:46.292863 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0326 23:19:46.292894 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0326 23:19:46.330077 3917 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:19:46.330193 3917 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0326 23:19:46.355105 3917 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 24.953618ms I0326 23:19:46.355129 3917 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 24.982334ms I0326 23:19:46.356342 3917 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 26.281702ms, libovsdb time 873.797µs I0326 23:19:46.688979 3917 cni.go:403] [openshift-monitoring/alertmanager-main-1 56b75374b24ae63953c346fb76b30771aeb5a54b608ff31582df4e8c6f52e2fc network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 56b75374b24ae63953c346fb76b30771aeb5a54b608ff31582df4e8c6f52e2fc network default NAD default] I0326 23:19:46.697536 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 56b75374b24ae63, network: default, NAD default, SandboxID: "56b75374b24ae63953c346fb76b30771aeb5a54b608ff31582df4e8c6f52e2fc", PCI device ID: , UID: "fdf8636b-1f08-48f0-8f81-48a8d7bfaddd", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0326 23:19:46.741935 3917 cni.go:424] [openshift-monitoring/alertmanager-main-1 56b75374b24ae63953c346fb76b30771aeb5a54b608ff31582df4e8c6f52e2fc network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 56b75374b24ae63953c346fb76b30771aeb5a54b608ff31582df4e8c6f52e2fc network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:3b:3d:63:62:db\",\"name\":\"56b75374b24ae63\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e0c80f02-0f18-4f32-b967-805d0fe5b31e\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:19:50.029442 3917 cni.go:403] [openshift-monitoring/prometheus-k8s-1 49405f2253b34239ea855daf16057458c9fe4f56c410d30aba641eeeae755306 network default NAD default] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 49405f2253b34239ea855daf16057458c9fe4f56c410d30aba641eeeae755306 network default NAD default] I0326 23:19:50.101796 3917 cni.go:424] [openshift-monitoring/prometheus-k8s-1 49405f2253b34239ea855daf16057458c9fe4f56c410d30aba641eeeae755306 network default NAD default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 49405f2253b34239ea855daf16057458c9fe4f56c410d30aba641eeeae755306 network default NAD default], result "{}", err I0326 23:19:50.169325 3917 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:19:50.169356 3917 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0326 23:19:50.170491 3917 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.129.0.20 I0326 23:19:50.170522 3917 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:19:50.314659 3917 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:19:50.314683 3917 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:19:50.321806 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0326 23:19:50.321837 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0326 23:19:50.367812 3917 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:19:50.367948 3917 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0326 23:19:50.393387 3917 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 25.51151ms I0326 23:19:50.393413 3917 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 25.540678ms I0326 23:19:50.394816 3917 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 27.01933ms, libovsdb time 928.993µs I0326 23:19:50.727875 3917 cni.go:403] [openshift-monitoring/prometheus-k8s-1 9b8686093edb0cbdd809fbb66754f9e1d3648ab0782931db998e5b4adef49341 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 9b8686093edb0cbdd809fbb66754f9e1d3648ab0782931db998e5b4adef49341 network default NAD default] I0326 23:19:50.736738 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 9b8686093edb0cb, network: default, NAD default, SandboxID: "9b8686093edb0cbdd809fbb66754f9e1d3648ab0782931db998e5b4adef49341", PCI device ID: , UID: "7f5bd54d-c966-4fe3-9257-8340c83e7adb", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0326 23:19:50.779316 3917 cni.go:424] [openshift-monitoring/prometheus-k8s-1 9b8686093edb0cbdd809fbb66754f9e1d3648ab0782931db998e5b4adef49341 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 9b8686093edb0cbdd809fbb66754f9e1d3648ab0782931db998e5b4adef49341 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:90:b9:d5:53:81\",\"name\":\"9b8686093edb0cb\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cddab6b1-e175-4baa-bac3-5a593c4ab02a\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:20:05.065084 3917 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-4lsr2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:05.065630 3917 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-4lsr2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:05.963187 3917 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-4lsr2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:05.963211 3917 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-4lsr2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:20:16.748513 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:20:16.748538 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:20:16.748547 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0326 23:20:17.132688 3917 obj_retry.go:464] Detected object openshift-console/console-6d5d5dc7c4-sgnvs of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:17.133291 3917 obj_retry.go:464] Detected object openshift-console/console-6d5d5dc7c4-sgnvs of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:17.670515 3917 obj_retry.go:458] Detected object openshift-console/console-6d5d5dc7c4-sgnvs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:17.670546 3917 obj_retry.go:458] Detected object openshift-console/console-6d5d5dc7c4-sgnvs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:17.676474 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6d5d5dc7c4-sgnvs I0326 23:20:17.676505 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6d5d5dc7c4-sgnvs I0326 23:20:18.038173 3917 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:18.038789 3917 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:18.711896 3917 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:18.711920 3917 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:18.716209 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0326 23:20:18.716241 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0326 23:20:26.994657 3917 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0326 23:20:28.159405 3917 cni.go:403] [openshift-console/console-6d5d5dc7c4-2fqzf 0a55a902aa9adeeb0476f030d57e796190e64179e449f22cc26e72562403c335 network default NAD default] DEL starting CNI request [openshift-console/console-6d5d5dc7c4-2fqzf 0a55a902aa9adeeb0476f030d57e796190e64179e449f22cc26e72562403c335 network default NAD default] I0326 23:20:28.240922 3917 cni.go:424] [openshift-console/console-6d5d5dc7c4-2fqzf 0a55a902aa9adeeb0476f030d57e796190e64179e449f22cc26e72562403c335 network default NAD default] DEL finished CNI request [openshift-console/console-6d5d5dc7c4-2fqzf 0a55a902aa9adeeb0476f030d57e796190e64179e449f22cc26e72562403c335 network default NAD default], result "{}", err I0326 23:20:28.289010 3917 obj_retry.go:464] Detected object openshift-console/console-6d5d5dc7c4-2fqzf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:28.289046 3917 pods.go:176] Deleting pod: openshift-console/console-6d5d5dc7c4-2fqzf I0326 23:20:28.290443 3917 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6d5d5dc7c4-2fqzf, ips: 10.129.0.21 I0326 23:20:28.290468 3917 obj_retry.go:464] Detected object openshift-console/console-6d5d5dc7c4-2fqzf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:28.485335 3917 obj_retry.go:458] Detected object openshift-console/console-6d5d5dc7c4-2fqzf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:28.485358 3917 obj_retry.go:458] Detected object openshift-console/console-6d5d5dc7c4-2fqzf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:28.488829 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6d5d5dc7c4-2fqzf I0326 23:20:28.488854 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6d5d5dc7c4-2fqzf W0326 23:20:51.622681 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:20:51.622702 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0326 23:20:52.923338 3917 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:52.923891 3917 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:53.854999 3917 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:53.855026 3917 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:53.861213 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0326 23:20:53.861237 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0326 23:22:16.924771 3917 base_network_controller_pods.go:487] [default/openshift-console/console-6b8cc584dd-dxk6k] creating logical port openshift-console_console-6b8cc584dd-dxk6k for pod on switch ip-10-0-1-167.ec2.internal I0326 23:22:16.924920 3917 kube.go:256] Updating pod openshift-console/console-6b8cc584dd-dxk6k I0326 23:22:16.956238 3917 pod.go:62] [openshift-console/console-6b8cc584dd-dxk6k] pod update took 31.34756ms I0326 23:22:16.956264 3917 base_network_controller_pods.go:937] [default/openshift-console/console-6b8cc584dd-dxk6k] addLogicalPort annotation time took 31.379909ms I0326 23:22:16.957535 3917 pods.go:271] [openshift-console/console-6b8cc584dd-dxk6k] addLogicalPort took 32.797884ms, libovsdb time 874.356µs I0326 23:22:17.281728 3917 cni.go:403] [openshift-console/console-6b8cc584dd-dxk6k ae2a0cf8852872fd30d9c803a468db73f7215b8940ccefcd37b8a85e8e395351 network default NAD default] ADD starting CNI request [openshift-console/console-6b8cc584dd-dxk6k ae2a0cf8852872fd30d9c803a468db73f7215b8940ccefcd37b8a85e8e395351 network default NAD default] I0326 23:22:17.290773 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-6b8cc584dd-dxk6k, hostIfaceName: ae2a0cf8852872f, network: default, NAD default, SandboxID: "ae2a0cf8852872fd30d9c803a468db73f7215b8940ccefcd37b8a85e8e395351", PCI device ID: , UID: "a0b38cde-e0e2-49ab-98b0-6388dedf63f8", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0326 23:22:17.333651 3917 cni.go:424] [openshift-console/console-6b8cc584dd-dxk6k ae2a0cf8852872fd30d9c803a468db73f7215b8940ccefcd37b8a85e8e395351 network default NAD default] ADD finished CNI request [openshift-console/console-6b8cc584dd-dxk6k ae2a0cf8852872fd30d9c803a468db73f7215b8940ccefcd37b8a85e8e395351 network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:bd:be:da:93:bf\",\"name\":\"ae2a0cf8852872f\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/028aa09b-c788-4522-b3a3-3f0c0c573d31\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:22:27.948289 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0326 23:22:33.658631 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0326 23:22:42.212814 3917 obj_retry.go:464] Detected object openshift-console/console-6cf96b65cd-fjj26 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:22:42.213449 3917 obj_retry.go:464] Detected object openshift-console/console-6cf96b65cd-fjj26 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:22:42.556019 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=69 I0326 23:22:42.643053 3917 obj_retry.go:458] Detected object openshift-console/console-6cf96b65cd-fjj26 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:22:42.643071 3917 obj_retry.go:458] Detected object openshift-console/console-6cf96b65cd-fjj26 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:22:42.649464 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6cf96b65cd-fjj26 I0326 23:22:42.649488 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6cf96b65cd-fjj26 I0326 23:22:44.653569 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0326 23:22:53.265664 3917 obj_retry.go:464] Detected object openshift-console/console-6cf96b65cd-jkv58 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:22:53.266354 3917 obj_retry.go:464] Detected object openshift-console/console-6cf96b65cd-jkv58 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:22:53.573278 3917 obj_retry.go:458] Detected object openshift-console/console-6cf96b65cd-jkv58 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:22:53.573332 3917 obj_retry.go:458] Detected object openshift-console/console-6cf96b65cd-jkv58 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:22:53.578456 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6cf96b65cd-jkv58 I0326 23:22:53.578471 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6cf96b65cd-jkv58 I0326 23:23:37.557972 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=56 I0326 23:23:43.557409 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=149 W0326 23:23:45.773837 3917 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:23:52.567530 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=611 I0326 23:23:56.815426 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0326 23:23:56.815483 3917 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-gxh5d IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-gxh5d: no pod IPs found I0326 23:24:19.744535 3917 egressfirewall.go:628] Syncing node "ip-10-0-2-111.ec2.internal" for egress firewall I0326 23:24:19.744606 3917 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-111.ec2.internal I0326 23:24:19.744627 3917 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-111.ec2.internal: 27.721µs I0326 23:24:42.555849 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0326 23:24:42.665325 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0326 23:24:55.639158 3917 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0326 23:25:25.562554 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=16 I0326 23:25:28.676013 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0326 23:25:43.649112 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0326 23:25:45.643630 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0326 23:26:03.639032 3917 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0326 23:26:14.378244 3917 namespace.go:98] [tsf] adding namespace I0326 23:26:14.379238 3917 namespace.go:102] [tsf] adding namespace took 971.804µs I0326 23:26:14.397234 3917 namespace.go:142] [tsf] updating namespace I0326 23:26:14.407257 3917 namespace.go:142] [tsf] updating namespace I0326 23:26:18.255301 3917 namespace.go:98] [konflux-operator] adding namespace I0326 23:26:18.256317 3917 namespace.go:102] [konflux-operator] adding namespace took 994.461µs I0326 23:26:18.262146 3917 namespace.go:98] [konflux-ui] adding namespace I0326 23:26:18.262252 3917 namespace.go:98] [rhtpa-operator] adding namespace I0326 23:26:18.262393 3917 namespace.go:98] [rhbk-operator] adding namespace I0326 23:26:18.262937 3917 namespace.go:102] [konflux-ui] adding namespace took 770.514µs I0326 23:26:18.262959 3917 namespace.go:98] [tsf-tpa] adding namespace I0326 23:26:18.263575 3917 namespace.go:102] [rhtpa-operator] adding namespace took 1.30591ms I0326 23:26:18.263609 3917 namespace.go:98] [tsf-tas] adding namespace I0326 23:26:18.264128 3917 namespace.go:102] [tsf-tas] adding namespace took 509.991µs I0326 23:26:18.264770 3917 namespace.go:102] [rhbk-operator] adding namespace took 2.360952ms I0326 23:26:18.264793 3917 namespace.go:98] [cert-manager-operator] adding namespace I0326 23:26:18.265242 3917 namespace.go:102] [tsf-tpa] adding namespace took 2.27507ms I0326 23:26:18.265397 3917 namespace.go:98] [tsf-keycloak] adding namespace I0326 23:26:18.265874 3917 namespace.go:102] [cert-manager-operator] adding namespace took 1.073987ms I0326 23:26:18.266492 3917 namespace.go:102] [tsf-keycloak] adding namespace took 1.082312ms I0326 23:26:18.280697 3917 namespace.go:142] [konflux-operator] updating namespace I0326 23:26:18.297406 3917 namespace.go:142] [konflux-operator] updating namespace I0326 23:26:18.306768 3917 namespace.go:142] [konflux-ui] updating namespace I0326 23:26:18.321988 3917 namespace.go:142] [konflux-ui] updating namespace I0326 23:26:18.322081 3917 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:26:18.325125 3917 namespace.go:142] [rhbk-operator] updating namespace I0326 23:26:18.329208 3917 namespace.go:142] [tsf-tpa] updating namespace I0326 23:26:18.335520 3917 namespace.go:142] [tsf-tas] updating namespace I0326 23:26:18.337818 3917 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:26:18.341600 3917 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:26:18.347557 3917 namespace.go:142] [konflux-ui] updating namespace I0326 23:26:18.348937 3917 namespace.go:142] [konflux-operator] updating namespace I0326 23:26:18.382208 3917 namespace.go:142] [tsf-tpa] updating namespace I0326 23:26:18.391617 3917 namespace.go:142] [tsf-tpa] updating namespace I0326 23:26:18.414272 3917 namespace.go:142] [rhbk-operator] updating namespace I0326 23:26:18.422605 3917 namespace.go:142] [rhbk-operator] updating namespace I0326 23:26:18.442962 3917 namespace.go:142] [tsf-tas] updating namespace I0326 23:26:18.453186 3917 namespace.go:142] [tsf-tas] updating namespace I0326 23:26:18.476930 3917 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:26:18.499855 3917 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:26:18.516970 3917 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:26:18.531423 3917 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:26:18.570044 3917 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:26:18.583585 3917 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:26:19.426149 3917 namespace.go:142] [tsf-tpa] updating namespace I0326 23:26:19.430865 3917 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:26:27.792780 3917 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7md2q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:27.793384 3917 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7md2q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:27.793411 3917 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7md2q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:27.793420 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:26:27.793432 3917 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7md2q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:27.793437 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:26:28.817484 3917 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7md2q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:28.817515 3917 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7md2q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:28.817533 3917 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7md2q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:28.817549 3917 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7md2q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:29.667536 3917 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vls47 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:29.668148 3917 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vls47 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:29.668177 3917 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vls47 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:29.668189 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:26:29.668206 3917 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vls47 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:29.668211 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:26:30.693715 3917 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vls47 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:30.693739 3917 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vls47 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:30.693801 3917 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vls47 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:30.693812 3917 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vls47 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:31.511096 3917 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8khqh6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:31.511704 3917 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8khqh6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:31.511772 3917 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8khqh6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:31.511787 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:26:31.511805 3917 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8khqh6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:31.511815 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:26:31.615343 3917 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafjxmsn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:31.615872 3917 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafjxmsn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:31.615899 3917 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafjxmsn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:31.615908 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:26:31.615918 3917 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafjxmsn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:31.615922 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:26:31.635031 3917 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad79njh4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:31.635453 3917 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad79njh4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:31.635488 3917 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad79njh4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:31.635501 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:26:31.635515 3917 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad79njh4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:31.635523 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:26:32.537739 3917 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8khqh6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:32.537781 3917 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8khqh6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:32.537794 3917 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8khqh6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:32.537808 3917 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8khqh6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:32.640580 3917 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafjxmsn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:32.640600 3917 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafjxmsn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:32.640610 3917 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafjxmsn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:32.640623 3917 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafjxmsn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:32.667775 3917 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad79njh4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:32.667800 3917 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad79njh4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:32.667810 3917 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad79njh4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:32.667819 3917 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad79njh4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:33.518486 3917 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5thdx9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:33.519034 3917 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5thdx9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:33.519066 3917 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5thdx9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:33.519076 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:26:33.519093 3917 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5thdx9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:33.519100 3917 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:26:34.552505 3917 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5thdx9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:34.552529 3917 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5thdx9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:34.552543 3917 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5thdx9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:34.552556 3917 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5thdx9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:36.319683 3917 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-8468f77bc5-wnwqj] creating logical port rhbk-operator_rhbk-operator-8468f77bc5-wnwqj for pod on switch ip-10-0-1-167.ec2.internal I0326 23:26:36.319831 3917 kube.go:256] Updating pod rhbk-operator/rhbk-operator-8468f77bc5-wnwqj I0326 23:26:36.342960 3917 pod.go:62] [rhbk-operator/rhbk-operator-8468f77bc5-wnwqj] pod update took 23.158031ms I0326 23:26:36.342985 3917 base_network_controller_pods.go:937] [default/rhbk-operator/rhbk-operator-8468f77bc5-wnwqj] addLogicalPort annotation time took 23.189867ms I0326 23:26:36.344262 3917 pods.go:271] [rhbk-operator/rhbk-operator-8468f77bc5-wnwqj] addLogicalPort took 24.603616ms, libovsdb time 953.871µs I0326 23:26:36.687061 3917 cni.go:403] [rhbk-operator/rhbk-operator-8468f77bc5-wnwqj e9755ce1eb6ef425be98e2ef44877fa67c9ecd2413f4998c7005704f9f1018ff network default NAD default] ADD starting CNI request [rhbk-operator/rhbk-operator-8468f77bc5-wnwqj e9755ce1eb6ef425be98e2ef44877fa67c9ecd2413f4998c7005704f9f1018ff network default NAD default] I0326 23:26:36.695514 3917 helper_linux.go:450] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-8468f77bc5-wnwqj, hostIfaceName: e9755ce1eb6ef42, network: default, NAD default, SandboxID: "e9755ce1eb6ef425be98e2ef44877fa67c9ecd2413f4998c7005704f9f1018ff", PCI device ID: , UID: "e12d08a7-a557-461f-8db2-98eda7027960", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0326 23:26:36.738603 3917 cni.go:424] [rhbk-operator/rhbk-operator-8468f77bc5-wnwqj e9755ce1eb6ef425be98e2ef44877fa67c9ecd2413f4998c7005704f9f1018ff network default NAD default] ADD finished CNI request [rhbk-operator/rhbk-operator-8468f77bc5-wnwqj e9755ce1eb6ef425be98e2ef44877fa67c9ecd2413f4998c7005704f9f1018ff network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:80:25:ff:05:ef\",\"name\":\"e9755ce1eb6ef42\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/42a1da72-2661-4cc9-b738-27f64bda3417\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:26:45.085388 3917 namespace.go:142] [openshift-operators] updating namespace I0326 23:26:45.098554 3917 namespace.go:142] [openshift-operators] updating namespace W0326 23:26:45.514690 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:26:48.331688 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:26:48.340250 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:26:48.378907 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:26:48.388410 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:26:48.475230 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0326 23:26:53.292476 3917 namespace.go:98] [trusted-artifact-signer] adding namespace I0326 23:26:53.293368 3917 namespace.go:102] [trusted-artifact-signer] adding namespace took 867.808µs I0326 23:26:53.309349 3917 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:26:53.334392 3917 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:26:53.398281 3917 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-ljm4w] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-ljm4w for pod on switch ip-10-0-1-167.ec2.internal I0326 23:26:53.398368 3917 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-ljm4w I0326 23:26:53.421449 3917 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-ljm4w] pod update took 23.103623ms I0326 23:26:53.421467 3917 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-ljm4w] addLogicalPort annotation time took 23.126144ms I0326 23:26:53.422785 3917 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-ljm4w] addLogicalPort took 24.515474ms, libovsdb time 965.679µs W0326 23:26:53.499434 3917 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0326 23:26:53.757255 3917 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-ljm4w 08ee01960d9a6c71e39beb38f06b8a66d6d2a1dc25d4ba60d6730a19d0fb12dc network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-ljm4w 08ee01960d9a6c71e39beb38f06b8a66d6d2a1dc25d4ba60d6730a19d0fb12dc network default NAD default] I0326 23:26:53.765645 3917 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-ljm4w, hostIfaceName: 08ee01960d9a6c7, network: default, NAD default, SandboxID: "08ee01960d9a6c71e39beb38f06b8a66d6d2a1dc25d4ba60d6730a19d0fb12dc", PCI device ID: , UID: "b89f835c-6e20-471a-a4c7-e6cdcd627294", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0326 23:26:53.810047 3917 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-ljm4w 08ee01960d9a6c71e39beb38f06b8a66d6d2a1dc25d4ba60d6730a19d0fb12dc network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-ljm4w 08ee01960d9a6c71e39beb38f06b8a66d6d2a1dc25d4ba60d6730a19d0fb12dc network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:d2:f5:ce:43:87\",\"name\":\"08ee01960d9a6c7\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/49e8085f-7311-4bbe-9896-6116382deb77\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:26:55.669842 3917 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0326 23:26:57.105479 3917 namespace.go:142] [openshift-operators] updating namespace I0326 23:26:57.733646 3917 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-txwsr] creating logical port openshift-operators_tekton-operator-webhook-668b6fdd5d-txwsr for pod on switch ip-10-0-1-167.ec2.internal I0326 23:26:57.733807 3917 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-668b6fdd5d-txwsr I0326 23:26:57.837624 3917 pod.go:62] [openshift-operators/tekton-operator-webhook-668b6fdd5d-txwsr] pod update took 103.83754ms I0326 23:26:57.837673 3917 base_network_controller_pods.go:937] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-txwsr] addLogicalPort annotation time took 103.889797ms I0326 23:26:57.839340 3917 pods.go:271] [openshift-operators/tekton-operator-webhook-668b6fdd5d-txwsr] addLogicalPort took 105.711103ms, libovsdb time 1.113429ms I0326 23:26:58.181202 3917 cni.go:403] [openshift-operators/tekton-operator-webhook-668b6fdd5d-txwsr f47b52cc16125e63c7c24051fd989152d966cd5e9a608d175ec0c1e2fba86a5e network default NAD default] ADD starting CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-txwsr f47b52cc16125e63c7c24051fd989152d966cd5e9a608d175ec0c1e2fba86a5e network default NAD default] I0326 23:26:58.190193 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-668b6fdd5d-txwsr, hostIfaceName: f47b52cc16125e6, network: default, NAD default, SandboxID: "f47b52cc16125e63c7c24051fd989152d966cd5e9a608d175ec0c1e2fba86a5e", PCI device ID: , UID: "b4ca773d-1caa-44f0-8177-2bda5cca4f0b", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0326 23:26:58.236111 3917 cni.go:424] [openshift-operators/tekton-operator-webhook-668b6fdd5d-txwsr f47b52cc16125e63c7c24051fd989152d966cd5e9a608d175ec0c1e2fba86a5e network default NAD default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-txwsr f47b52cc16125e63c7c24051fd989152d966cd5e9a608d175ec0c1e2fba86a5e network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:1a:33:d5:23:f1\",\"name\":\"f47b52cc16125e6\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9790b9f6-af74-438a-9a75-0cd919a5fa9e\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:27:07.676256 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11 I0326 23:27:08.381633 3917 namespace.go:98] [cert-manager] adding namespace I0326 23:27:08.382697 3917 namespace.go:102] [cert-manager] adding namespace took 1.040025ms I0326 23:27:08.427569 3917 namespace.go:142] [cert-manager] updating namespace I0326 23:27:08.449498 3917 namespace.go:142] [cert-manager] updating namespace I0326 23:27:10.256607 3917 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-fd44f5cbb-fcqgw] creating logical port cert-manager_cert-manager-webhook-fd44f5cbb-fcqgw for pod on switch ip-10-0-1-167.ec2.internal I0326 23:27:10.256703 3917 kube.go:256] Updating pod cert-manager/cert-manager-webhook-fd44f5cbb-fcqgw I0326 23:27:10.290792 3917 pod.go:62] [cert-manager/cert-manager-webhook-fd44f5cbb-fcqgw] pod update took 34.111231ms I0326 23:27:10.290816 3917 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-webhook-fd44f5cbb-fcqgw] addLogicalPort annotation time took 34.142752ms I0326 23:27:10.292023 3917 pods.go:271] [cert-manager/cert-manager-webhook-fd44f5cbb-fcqgw] addLogicalPort took 35.432921ms, libovsdb time 832.558µs I0326 23:27:11.615624 3917 namespace.go:98] [openshift-pipelines] adding namespace I0326 23:27:11.616494 3917 namespace.go:102] [openshift-pipelines] adding namespace took 847.291µs I0326 23:27:11.646206 3917 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:27:12.091402 3917 cni.go:403] [cert-manager/cert-manager-webhook-fd44f5cbb-fcqgw 77bd36109f49870254f2cab39d83114556c4a8fed5d0367436d92ce504103f96 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-fcqgw 77bd36109f49870254f2cab39d83114556c4a8fed5d0367436d92ce504103f96 network default NAD default] I0326 23:27:12.100459 3917 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-fd44f5cbb-fcqgw, hostIfaceName: 77bd36109f49870, network: default, NAD default, SandboxID: "77bd36109f49870254f2cab39d83114556c4a8fed5d0367436d92ce504103f96", PCI device ID: , UID: "294447ce-90b9-42a3-92b9-7052b27fdc12", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0326 23:27:12.146541 3917 cni.go:424] [cert-manager/cert-manager-webhook-fd44f5cbb-fcqgw 77bd36109f49870254f2cab39d83114556c4a8fed5d0367436d92ce504103f96 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-fcqgw 77bd36109f49870254f2cab39d83114556c4a8fed5d0367436d92ce504103f96 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:ab:c0:99:3b:ae\",\"name\":\"77bd36109f49870\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4e18be16-035d-4afe-9002-4cc786187497\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:27:12.885778 3917 namespace.go:142] [cert-manager] updating namespace I0326 23:27:13.092645 3917 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:27:13.489604 3917 namespace.go:142] [konflux-operator] updating namespace I0326 23:27:13.690099 3917 namespace.go:142] [konflux-ui] updating namespace I0326 23:27:13.907135 3917 namespace.go:142] [open-cluster-management-2p9mplstndhmerb1mp1pd72ivq8vo3pp] updating namespace I0326 23:27:14.371643 3917 namespace.go:142] [rhbk-operator] updating namespace I0326 23:27:14.589982 3917 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:27:14.783695 3917 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:27:14.981013 3917 namespace.go:142] [tsf] updating namespace I0326 23:27:15.178724 3917 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:27:15.376125 3917 namespace.go:142] [tsf-tas] updating namespace I0326 23:27:15.578854 3917 namespace.go:142] [tsf-tpa] updating namespace I0326 23:27:15.707624 3917 namespace.go:142] [cert-manager] updating namespace I0326 23:27:15.717829 3917 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:27:15.725484 3917 namespace.go:142] [default] updating namespace I0326 23:27:15.736795 3917 namespace.go:142] [konflux-operator] updating namespace I0326 23:27:15.746224 3917 namespace.go:142] [konflux-ui] updating namespace I0326 23:27:15.756511 3917 namespace.go:142] [open-cluster-management-2p9mplstndhmerb1mp1pd72ivq8vo3pp] updating namespace I0326 23:27:15.764330 3917 namespace.go:142] [openshift] updating namespace I0326 23:27:15.774021 3917 namespace.go:142] [rhbk-operator] updating namespace I0326 23:27:15.783100 3917 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:27:15.792916 3917 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:27:15.800937 3917 namespace.go:142] [tsf] updating namespace I0326 23:27:15.824900 3917 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:27:15.839906 3917 namespace.go:142] [tsf-tas] updating namespace I0326 23:27:15.866364 3917 namespace.go:142] [tsf-tpa] updating namespace I0326 23:27:15.961701 3917 namespace.go:142] [cert-manager] updating namespace I0326 23:27:16.064590 3917 namespace.go:142] [cert-manager-operator] updating namespace W0326 23:27:16.076210 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0326 23:27:16.076231 3917 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0326 23:27:16.158725 3917 namespace.go:142] [default] updating namespace I0326 23:27:16.264042 3917 namespace.go:142] [konflux-operator] updating namespace I0326 23:27:16.360888 3917 namespace.go:142] [konflux-ui] updating namespace I0326 23:27:16.468888 3917 namespace.go:142] [open-cluster-management-2p9mplstndhmerb1mp1pd72ivq8vo3pp] updating namespace I0326 23:27:16.559734 3917 namespace.go:142] [openshift] updating namespace I0326 23:27:16.664070 3917 namespace.go:142] [rhbk-operator] updating namespace I0326 23:27:16.760390 3917 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:27:16.860380 3917 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:27:16.962847 3917 namespace.go:142] [tsf] updating namespace I0326 23:27:17.061735 3917 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:27:17.161363 3917 namespace.go:142] [tsf-tas] updating namespace I0326 23:27:17.262736 3917 namespace.go:142] [tsf-tpa] updating namespace I0326 23:27:17.499379 3917 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:27:20.790384 3917 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:27:20.790987 3917 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:27:20.814929 3917 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:27:20.814946 3917 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:27:20.822883 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0326 23:27:20.822907 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions W0326 23:27:22.090308 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0326 23:27:22.090327 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0326 23:27:25.961162 3917 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-j7hs4] creating logical port openshift-pipelines_tekton-pipelines-controller-84d6d58f5c-j7hs4 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:27:25.961252 3917 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-j7hs4 I0326 23:27:25.976645 3917 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-j7hs4] pod update took 15.419678ms I0326 23:27:25.976668 3917 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-j7hs4] addLogicalPort annotation time took 15.446871ms I0326 23:27:25.977939 3917 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-j7hs4] addLogicalPort took 16.79456ms, libovsdb time 888.906µs I0326 23:27:26.047914 3917 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-b79bdd59-rfhtj] creating logical port openshift-pipelines_tekton-events-controller-b79bdd59-rfhtj for pod on switch ip-10-0-1-167.ec2.internal I0326 23:27:26.048000 3917 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-b79bdd59-rfhtj I0326 23:27:26.065392 3917 pod.go:62] [openshift-pipelines/tekton-events-controller-b79bdd59-rfhtj] pod update took 17.413556ms I0326 23:27:26.065410 3917 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-events-controller-b79bdd59-rfhtj] addLogicalPort annotation time took 17.437468ms I0326 23:27:26.066549 3917 pods.go:271] [openshift-pipelines/tekton-events-controller-b79bdd59-rfhtj] addLogicalPort took 18.652248ms, libovsdb time 811.501µs I0326 23:27:26.320763 3917 cni.go:403] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-j7hs4 e106d6b5f746ccf55268a383cd5704729a159d2ad5fadce671c9ba3d504c3766 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-j7hs4 e106d6b5f746ccf55268a383cd5704729a159d2ad5fadce671c9ba3d504c3766 network default NAD default] I0326 23:27:26.329686 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-84d6d58f5c-j7hs4, hostIfaceName: e106d6b5f746ccf, network: default, NAD default, SandboxID: "e106d6b5f746ccf55268a383cd5704729a159d2ad5fadce671c9ba3d504c3766", PCI device ID: , UID: "a08087af-c4ec-47ca-9485-d9a26ba4fb13", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0326 23:27:26.374862 3917 cni.go:424] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-j7hs4 e106d6b5f746ccf55268a383cd5704729a159d2ad5fadce671c9ba3d504c3766 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-j7hs4 e106d6b5f746ccf55268a383cd5704729a159d2ad5fadce671c9ba3d504c3766 network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:01:ea:e4:54:61\",\"name\":\"e106d6b5f746ccf\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2c687527-066c-4345-ae23-85a0cf291882\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:27:26.410269 3917 cni.go:403] [openshift-pipelines/tekton-events-controller-b79bdd59-rfhtj 1c426707981573b0e608d75b492fff5ab9cb5aacdf6693a12e19da9c0d2da9ce network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-rfhtj 1c426707981573b0e608d75b492fff5ab9cb5aacdf6693a12e19da9c0d2da9ce network default NAD default] I0326 23:27:26.419777 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-b79bdd59-rfhtj, hostIfaceName: 1c426707981573b, network: default, NAD default, SandboxID: "1c426707981573b0e608d75b492fff5ab9cb5aacdf6693a12e19da9c0d2da9ce", PCI device ID: , UID: "7ba232cc-304e-455f-b401-a8e4f3c0a8a6", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0326 23:27:26.465004 3917 cni.go:424] [openshift-pipelines/tekton-events-controller-b79bdd59-rfhtj 1c426707981573b0e608d75b492fff5ab9cb5aacdf6693a12e19da9c0d2da9ce network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-rfhtj 1c426707981573b0e608d75b492fff5ab9cb5aacdf6693a12e19da9c0d2da9ce network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:60:a2:de:a7:8e\",\"name\":\"1c426707981573b\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/86ec4924-c2c5-472f-b023-2fe05b0725c4\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:27:29.313587 3917 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59ccb4bb6b-79mbg] creating logical port cert-manager_cert-manager-59ccb4bb6b-79mbg for pod on switch ip-10-0-1-167.ec2.internal I0326 23:27:29.313693 3917 kube.go:256] Updating pod cert-manager/cert-manager-59ccb4bb6b-79mbg I0326 23:27:29.328722 3917 pod.go:62] [cert-manager/cert-manager-59ccb4bb6b-79mbg] pod update took 15.062048ms I0326 23:27:29.328761 3917 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-59ccb4bb6b-79mbg] addLogicalPort annotation time took 15.092313ms I0326 23:27:29.330199 3917 pods.go:271] [cert-manager/cert-manager-59ccb4bb6b-79mbg] addLogicalPort took 16.627557ms, libovsdb time 923.329µs I0326 23:27:29.676085 3917 cni.go:403] [cert-manager/cert-manager-59ccb4bb6b-79mbg 1c6b4b08c1fa96d91713757bbba0b7c5667f3d866da6057fe5ae0935087e1c27 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-59ccb4bb6b-79mbg 1c6b4b08c1fa96d91713757bbba0b7c5667f3d866da6057fe5ae0935087e1c27 network default NAD default] I0326 23:27:29.689800 3917 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59ccb4bb6b-79mbg, hostIfaceName: 1c6b4b08c1fa96d, network: default, NAD default, SandboxID: "1c6b4b08c1fa96d91713757bbba0b7c5667f3d866da6057fe5ae0935087e1c27", PCI device ID: , UID: "0eddbcda-739a-4884-813d-8cfb7d676725", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0326 23:27:29.734769 3917 cni.go:424] [cert-manager/cert-manager-59ccb4bb6b-79mbg 1c6b4b08c1fa96d91713757bbba0b7c5667f3d866da6057fe5ae0935087e1c27 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-59ccb4bb6b-79mbg 1c6b4b08c1fa96d91713757bbba0b7c5667f3d866da6057fe5ae0935087e1c27 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:f0:1d:df:22:69\",\"name\":\"1c6b4b08c1fa96d\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/77067180-d37d-459a-9e36-cf574564b5bd\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:27:30.143642 3917 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:27:30.143659 3917 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:27:30.143665 3917 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:27:30.199945 3917 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:27:30.199964 3917 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:27:30.199969 3917 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:27:30.675839 3917 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:27:30.675862 3917 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:27:30.675867 3917 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:27:30.675872 3917 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:27:34.694777 3917 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0326 23:27:35.739210 3917 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0326 23:27:41.154285 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:27:41.154309 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:27:41.154317 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:27:41.181142 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:27:41.181162 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:27:41.181168 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0326 23:27:53.336018 3917 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-fqmgm] creating logical port openshift-pipelines_tekton-triggers-controller-68564d75ff-fqmgm for pod on switch ip-10-0-1-167.ec2.internal I0326 23:27:53.336119 3917 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-68564d75ff-fqmgm I0326 23:27:53.355207 3917 pod.go:62] [openshift-pipelines/tekton-triggers-controller-68564d75ff-fqmgm] pod update took 19.117559ms I0326 23:27:53.355230 3917 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-fqmgm] addLogicalPort annotation time took 19.146332ms I0326 23:27:53.356484 3917 pods.go:271] [openshift-pipelines/tekton-triggers-controller-68564d75ff-fqmgm] addLogicalPort took 20.480013ms, libovsdb time 883.831µs I0326 23:27:53.718502 3917 cni.go:403] [openshift-pipelines/tekton-triggers-controller-68564d75ff-fqmgm ea658b4f21739889cd2f0ae8db1e253e6a364ed07a2d2e45138a610282ad0c48 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-fqmgm ea658b4f21739889cd2f0ae8db1e253e6a364ed07a2d2e45138a610282ad0c48 network default NAD default] I0326 23:27:53.727447 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-68564d75ff-fqmgm, hostIfaceName: ea658b4f2173988, network: default, NAD default, SandboxID: "ea658b4f21739889cd2f0ae8db1e253e6a364ed07a2d2e45138a610282ad0c48", PCI device ID: , UID: "73b582c7-0067-47ca-9842-5918a4b66801", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0326 23:27:53.774429 3917 cni.go:424] [openshift-pipelines/tekton-triggers-controller-68564d75ff-fqmgm ea658b4f21739889cd2f0ae8db1e253e6a364ed07a2d2e45138a610282ad0c48 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-fqmgm ea658b4f21739889cd2f0ae8db1e253e6a364ed07a2d2e45138a610282ad0c48 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:4b:d2:de:76:fe\",\"name\":\"ea658b4f2173988\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b81e3d67-9ac8-416c-8bb9-cc5bd0058d9e\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:27:57.567805 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=34 W0326 23:27:57.891614 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0326 23:28:00.055060 3917 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:00.055601 3917 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:00.081185 3917 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:00.081210 3917 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:00.087431 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0326 23:28:00.087461 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0326 23:28:10.740764 3917 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-8wgp8] creating logical port openshift-pipelines_tekton-chains-controller-5fb999ffff-8wgp8 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:28:10.740850 3917 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-5fb999ffff-8wgp8 I0326 23:28:10.764976 3917 pod.go:62] [openshift-pipelines/tekton-chains-controller-5fb999ffff-8wgp8] pod update took 24.147205ms I0326 23:28:10.765008 3917 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-8wgp8] addLogicalPort annotation time took 24.173563ms I0326 23:28:10.766200 3917 pods.go:271] [openshift-pipelines/tekton-chains-controller-5fb999ffff-8wgp8] addLogicalPort took 25.470317ms, libovsdb time 836.111µs W0326 23:28:10.891060 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:28:10.891078 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:28:10.895458 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0326 23:28:11.103561 3917 cni.go:403] [openshift-pipelines/tekton-chains-controller-5fb999ffff-8wgp8 f43d8b62a7a5ef9a1488741a92231470064afab595ef8c64c198d74fbb3d8fc2 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-8wgp8 f43d8b62a7a5ef9a1488741a92231470064afab595ef8c64c198d74fbb3d8fc2 network default NAD default] I0326 23:28:11.112728 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-5fb999ffff-8wgp8, hostIfaceName: f43d8b62a7a5ef9, network: default, NAD default, SandboxID: "f43d8b62a7a5ef9a1488741a92231470064afab595ef8c64c198d74fbb3d8fc2", PCI device ID: , UID: "8d4251b4-118e-429f-b616-dedd8c62cfbb", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0326 23:28:11.159515 3917 cni.go:424] [openshift-pipelines/tekton-chains-controller-5fb999ffff-8wgp8 f43d8b62a7a5ef9a1488741a92231470064afab595ef8c64c198d74fbb3d8fc2 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-8wgp8 f43d8b62a7a5ef9a1488741a92231470064afab595ef8c64c198d74fbb3d8fc2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:66:14:28:12:45\",\"name\":\"f43d8b62a7a5ef9\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2729a4ed-d103-4e82-a70f-b65cc53e6352\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:28:15.334783 3917 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0326 23:28:19.662475 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0326 23:28:32.261982 3917 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:32.262574 3917 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:32.660161 3917 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-c9xlh] creating logical port openshift-pipelines_tekton-results-watcher-6f946f97cf-c9xlh for pod on switch ip-10-0-1-167.ec2.internal I0326 23:28:32.660240 3917 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-6f946f97cf-c9xlh I0326 23:28:32.675330 3917 pod.go:62] [openshift-pipelines/tekton-results-watcher-6f946f97cf-c9xlh] pod update took 15.109923ms I0326 23:28:32.675354 3917 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-c9xlh] addLogicalPort annotation time took 15.139909ms I0326 23:28:32.676552 3917 pods.go:271] [openshift-pipelines/tekton-results-watcher-6f946f97cf-c9xlh] addLogicalPort took 16.402486ms, libovsdb time 766.132µs I0326 23:28:33.022698 3917 cni.go:403] [openshift-pipelines/tekton-results-watcher-6f946f97cf-c9xlh 529e802a385ca48a06b7601e8a66c592b28968fdcc4f97500f5c8bf75e9fbcd1 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-c9xlh 529e802a385ca48a06b7601e8a66c592b28968fdcc4f97500f5c8bf75e9fbcd1 network default NAD default] I0326 23:28:33.031153 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-6f946f97cf-c9xlh, hostIfaceName: 529e802a385ca48, network: default, NAD default, SandboxID: "529e802a385ca48a06b7601e8a66c592b28968fdcc4f97500f5c8bf75e9fbcd1", PCI device ID: , UID: "2cd98aec-26d5-406c-847e-688bebbc2681", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0326 23:28:33.042054 3917 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:33.042106 3917 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:33.048734 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0326 23:28:33.048809 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0326 23:28:33.076985 3917 cni.go:424] [openshift-pipelines/tekton-results-watcher-6f946f97cf-c9xlh 529e802a385ca48a06b7601e8a66c592b28968fdcc4f97500f5c8bf75e9fbcd1 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-c9xlh 529e802a385ca48a06b7601e8a66c592b28968fdcc4f97500f5c8bf75e9fbcd1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:e4:95:3b:a5:ee\",\"name\":\"529e802a385ca48\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b6f74e57-9ccb-4c5b-a785-b1193cebf012\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:28:35.031805 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:28:35.036961 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:28:35.036978 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:28:40.108169 3917 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:28:40.108189 3917 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:28:40.108193 3917 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0326 23:28:54.322170 3917 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-dvdnf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:54.322740 3917 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-dvdnf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:55.349302 3917 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-dvdnf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:55.349336 3917 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-dvdnf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:55.356119 3917 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:55.356697 3917 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:55.403688 3917 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:55.403711 3917 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:55.416046 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0326 23:28:55.416073 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0326 23:28:55.756896 3917 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:55.757488 3917 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:56.196443 3917 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:56.196471 3917 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:56.209005 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0326 23:28:56.209033 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0326 23:28:58.238673 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:28:58.242780 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:28:58.242796 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0326 23:28:58.472026 3917 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-5789d9f8fd-f5dn4] creating logical port tsf-tas_trillian-logserver-5789d9f8fd-f5dn4 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:28:58.472177 3917 kube.go:256] Updating pod tsf-tas/trillian-logserver-5789d9f8fd-f5dn4 I0326 23:28:58.485670 3917 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-69647c4f54-4jqc4] creating logical port tsf-tas_fulcio-server-69647c4f54-4jqc4 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:28:58.485763 3917 kube.go:256] Updating pod tsf-tas/fulcio-server-69647c4f54-4jqc4 I0326 23:28:58.538707 3917 pod.go:62] [tsf-tas/trillian-logserver-5789d9f8fd-f5dn4] pod update took 66.56551ms I0326 23:28:58.538728 3917 base_network_controller_pods.go:937] [default/tsf-tas/trillian-logserver-5789d9f8fd-f5dn4] addLogicalPort annotation time took 66.594189ms I0326 23:28:58.539880 3917 pods.go:271] [tsf-tas/trillian-logserver-5789d9f8fd-f5dn4] addLogicalPort took 67.871441ms, libovsdb time 803.089µs I0326 23:28:58.559176 3917 pod.go:62] [tsf-tas/fulcio-server-69647c4f54-4jqc4] pod update took 73.445372ms I0326 23:28:58.559201 3917 base_network_controller_pods.go:937] [default/tsf-tas/fulcio-server-69647c4f54-4jqc4] addLogicalPort annotation time took 73.474263ms I0326 23:28:58.560302 3917 pods.go:271] [tsf-tas/fulcio-server-69647c4f54-4jqc4] addLogicalPort took 74.647186ms, libovsdb time 750.052µs I0326 23:28:59.064907 3917 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-vf8tp-zfwpj] creating logical port tsf-tas_rekor-createtree-job-vf8tp-zfwpj for pod on switch ip-10-0-1-167.ec2.internal I0326 23:28:59.065006 3917 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-vf8tp-zfwpj I0326 23:28:59.076894 3917 pod.go:62] [tsf-tas/rekor-createtree-job-vf8tp-zfwpj] pod update took 11.90933ms I0326 23:28:59.076927 3917 base_network_controller_pods.go:937] [default/tsf-tas/rekor-createtree-job-vf8tp-zfwpj] addLogicalPort annotation time took 11.935417ms I0326 23:28:59.078217 3917 pods.go:271] [tsf-tas/rekor-createtree-job-vf8tp-zfwpj] addLogicalPort took 13.324139ms, libovsdb time 900.144µs I0326 23:28:59.141981 3917 cni.go:403] [tsf-tas/fulcio-server-69647c4f54-4jqc4 9d7324ea6642818312d91b74891e8c6ba160c96b3c2a7934733caecefb27e7b2 network default NAD default] ADD starting CNI request [tsf-tas/fulcio-server-69647c4f54-4jqc4 9d7324ea6642818312d91b74891e8c6ba160c96b3c2a7934733caecefb27e7b2 network default NAD default] I0326 23:28:59.150377 3917 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-69647c4f54-4jqc4, hostIfaceName: 9d7324ea6642818, network: default, NAD default, SandboxID: "9d7324ea6642818312d91b74891e8c6ba160c96b3c2a7934733caecefb27e7b2", PCI device ID: , UID: "68149db6-43f8-4163-9d8c-2eb662dc753c", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0326 23:28:59.196128 3917 cni.go:424] [tsf-tas/fulcio-server-69647c4f54-4jqc4 9d7324ea6642818312d91b74891e8c6ba160c96b3c2a7934733caecefb27e7b2 network default NAD default] ADD finished CNI request [tsf-tas/fulcio-server-69647c4f54-4jqc4 9d7324ea6642818312d91b74891e8c6ba160c96b3c2a7934733caecefb27e7b2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:3f:44:9e:1b:6f\",\"name\":\"9d7324ea6642818\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3253fbe3-83c2-4487-a7cb-d6052087cb96\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:28:59.428417 3917 cni.go:403] [tsf-tas/rekor-createtree-job-vf8tp-zfwpj d564cab6d16ba463ebe4fd9b73f8a67350bf5fab2878611e0e63d13cbf207fe4 network default NAD default] ADD starting CNI request [tsf-tas/rekor-createtree-job-vf8tp-zfwpj d564cab6d16ba463ebe4fd9b73f8a67350bf5fab2878611e0e63d13cbf207fe4 network default NAD default] I0326 23:28:59.432658 3917 cni.go:403] [tsf-tas/trillian-logserver-5789d9f8fd-f5dn4 1313a857b1196b344a38c484e4482f6009cddfdd86b1b9fddc1b3c68f975c66e network default NAD default] ADD starting CNI request [tsf-tas/trillian-logserver-5789d9f8fd-f5dn4 1313a857b1196b344a38c484e4482f6009cddfdd86b1b9fddc1b3c68f975c66e network default NAD default] I0326 23:28:59.438350 3917 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-vf8tp-zfwpj, hostIfaceName: d564cab6d16ba46, network: default, NAD default, SandboxID: "d564cab6d16ba463ebe4fd9b73f8a67350bf5fab2878611e0e63d13cbf207fe4", PCI device ID: , UID: "4ee97706-ec63-4143-b76a-2049e4fe6be8", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0326 23:28:59.443091 3917 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-5789d9f8fd-f5dn4, hostIfaceName: 1313a857b1196b3, network: default, NAD default, SandboxID: "1313a857b1196b344a38c484e4482f6009cddfdd86b1b9fddc1b3c68f975c66e", PCI device ID: , UID: "1b095463-cbf3-40d2-b304-07d9b6195503", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0326 23:28:59.485162 3917 cni.go:424] [tsf-tas/rekor-createtree-job-vf8tp-zfwpj d564cab6d16ba463ebe4fd9b73f8a67350bf5fab2878611e0e63d13cbf207fe4 network default NAD default] ADD finished CNI request [tsf-tas/rekor-createtree-job-vf8tp-zfwpj d564cab6d16ba463ebe4fd9b73f8a67350bf5fab2878611e0e63d13cbf207fe4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:c0:80:45:cb:5b\",\"name\":\"d564cab6d16ba46\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6ae58e47-335a-4980-b26c-f352d01e19db\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:28:59.504842 3917 cni.go:424] [tsf-tas/trillian-logserver-5789d9f8fd-f5dn4 1313a857b1196b344a38c484e4482f6009cddfdd86b1b9fddc1b3c68f975c66e network default NAD default] ADD finished CNI request [tsf-tas/trillian-logserver-5789d9f8fd-f5dn4 1313a857b1196b344a38c484e4482f6009cddfdd86b1b9fddc1b3c68f975c66e network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:17:4d:6c:c2:8b\",\"name\":\"1313a857b1196b3\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4ec7b99d-db44-487d-8e7b-4aed6045d5df\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:29:06.624099 3917 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0326 23:29:11.335172 3917 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0326 23:29:11.657338 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 W0326 23:29:14.373054 3917 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0326 23:29:15.496403 3917 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-f5s69-2692j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:29:15.496940 3917 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-f5s69-2692j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:29:16.377416 3917 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-f5s69-2692j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:29:16.377435 3917 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-f5s69-2692j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:29:24.085001 3917 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hv9gx] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-hv9gx for pod on switch ip-10-0-1-167.ec2.internal I0326 23:29:24.085088 3917 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hv9gx I0326 23:29:24.102206 3917 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hv9gx] pod update took 17.138405ms I0326 23:29:24.102226 3917 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hv9gx] addLogicalPort annotation time took 17.162712ms I0326 23:29:24.103385 3917 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hv9gx] addLogicalPort took 18.402521ms, libovsdb time 811.487µs I0326 23:29:24.443176 3917 cni.go:403] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hv9gx 254ce0dedfa230a1fce4d65bc1c910a3335c2f128405763a336052267e09c436 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hv9gx 254ce0dedfa230a1fce4d65bc1c910a3335c2f128405763a336052267e09c436 network default NAD default] I0326 23:29:24.455673 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-hv9gx, hostIfaceName: 254ce0dedfa230a, network: default, NAD default, SandboxID: "254ce0dedfa230a1fce4d65bc1c910a3335c2f128405763a336052267e09c436", PCI device ID: , UID: "cf3900eb-b53e-4aaa-bef2-98005e904aa8", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0326 23:29:24.503335 3917 cni.go:424] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hv9gx 254ce0dedfa230a1fce4d65bc1c910a3335c2f128405763a336052267e09c436 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hv9gx 254ce0dedfa230a1fce4d65bc1c910a3335c2f128405763a336052267e09c436 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:14:ec:e5:96:63\",\"name\":\"254ce0dedfa230a\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7aa13200-a5d8-40ce-9c0d-1cf9fc372b20\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:29:26.717894 3917 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0326 23:29:27.441457 3917 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0326 23:29:27.441480 3917 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0326 23:29:37.741604 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0326 23:29:39.745669 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0326 23:29:45.704633 3917 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:29:45.705193 3917 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:29:45.726528 3917 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:29:45.726549 3917 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:29:45.732083 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0326 23:29:45.732107 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0326 23:29:54.760117 3917 obj_retry.go:464] Detected object tsf/patch-tekton-config-hwrns of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:29:54.760664 3917 obj_retry.go:464] Detected object tsf/patch-tekton-config-hwrns of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:29:54.857337 3917 cni.go:403] [tsf-tas/rekor-createtree-job-vf8tp-zfwpj d564cab6d16ba463ebe4fd9b73f8a67350bf5fab2878611e0e63d13cbf207fe4 network default NAD default] DEL starting CNI request [tsf-tas/rekor-createtree-job-vf8tp-zfwpj d564cab6d16ba463ebe4fd9b73f8a67350bf5fab2878611e0e63d13cbf207fe4 network default NAD default] I0326 23:29:54.929981 3917 cni.go:424] [tsf-tas/rekor-createtree-job-vf8tp-zfwpj d564cab6d16ba463ebe4fd9b73f8a67350bf5fab2878611e0e63d13cbf207fe4 network default NAD default] DEL finished CNI request [tsf-tas/rekor-createtree-job-vf8tp-zfwpj d564cab6d16ba463ebe4fd9b73f8a67350bf5fab2878611e0e63d13cbf207fe4 network default NAD default], result "{}", err I0326 23:29:54.978456 3917 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-vf8tp-zfwpj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:29:54.978501 3917 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-vf8tp-zfwpj I0326 23:29:54.979981 3917 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-vf8tp-zfwpj, ips: 10.129.0.37 I0326 23:29:54.980008 3917 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-vf8tp-zfwpj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:29:55.781364 3917 obj_retry.go:458] Detected object tsf/patch-tekton-config-hwrns of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:29:55.781394 3917 obj_retry.go:458] Detected object tsf/patch-tekton-config-hwrns of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:29:55.874514 3917 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-vf8tp-zfwpj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:29:55.874539 3917 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-vf8tp-zfwpj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:29:55.886638 3917 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-xp8cm] creating logical port tsf_tsf-tekton-configuration-xp8cm for pod on switch ip-10-0-1-167.ec2.internal I0326 23:29:55.886767 3917 kube.go:256] Updating pod tsf/tsf-tekton-configuration-xp8cm I0326 23:29:55.897987 3917 pod.go:62] [tsf/tsf-tekton-configuration-xp8cm] pod update took 11.264486ms I0326 23:29:55.898013 3917 base_network_controller_pods.go:937] [default/tsf/tsf-tekton-configuration-xp8cm] addLogicalPort annotation time took 11.289775ms I0326 23:29:55.899406 3917 pods.go:271] [tsf/tsf-tekton-configuration-xp8cm] addLogicalPort took 12.779837ms, libovsdb time 1.024828ms I0326 23:29:56.244202 3917 cni.go:403] [tsf/tsf-tekton-configuration-xp8cm 3a292f4fac93916550b3743dcc6d27e4fdce9d9e23c1e36559fe76133b6daa86 network default NAD default] ADD starting CNI request [tsf/tsf-tekton-configuration-xp8cm 3a292f4fac93916550b3743dcc6d27e4fdce9d9e23c1e36559fe76133b6daa86 network default NAD default] I0326 23:29:56.252341 3917 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-xp8cm, hostIfaceName: 3a292f4fac93916, network: default, NAD default, SandboxID: "3a292f4fac93916550b3743dcc6d27e4fdce9d9e23c1e36559fe76133b6daa86", PCI device ID: , UID: "93af8865-a17d-4fc5-adb0-cad1870a64a2", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0326 23:29:56.299666 3917 cni.go:424] [tsf/tsf-tekton-configuration-xp8cm 3a292f4fac93916550b3743dcc6d27e4fdce9d9e23c1e36559fe76133b6daa86 network default NAD default] ADD finished CNI request [tsf/tsf-tekton-configuration-xp8cm 3a292f4fac93916550b3743dcc6d27e4fdce9d9e23c1e36559fe76133b6daa86 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:f6:7d:57:1e:10\",\"name\":\"3a292f4fac93916\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d47574eb-7f99-4fc9-9629-0016307e74be\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:29:59.791623 3917 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-x4w9m-fgdtp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:29:59.792218 3917 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-x4w9m-fgdtp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:00.210180 3917 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-bdbb8865f-whbp8] creating logical port tsf-tas_rekor-search-ui-bdbb8865f-whbp8 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:30:00.210291 3917 kube.go:256] Updating pod tsf-tas/rekor-search-ui-bdbb8865f-whbp8 I0326 23:30:00.223529 3917 pod.go:62] [tsf-tas/rekor-search-ui-bdbb8865f-whbp8] pod update took 13.258663ms I0326 23:30:00.223548 3917 base_network_controller_pods.go:937] [default/tsf-tas/rekor-search-ui-bdbb8865f-whbp8] addLogicalPort annotation time took 13.281897ms I0326 23:30:00.224717 3917 pods.go:271] [tsf-tas/rekor-search-ui-bdbb8865f-whbp8] addLogicalPort took 14.549932ms, libovsdb time 799.951µs I0326 23:30:00.572387 3917 cni.go:403] [tsf-tas/rekor-search-ui-bdbb8865f-whbp8 bde4c2d8d4759f3125da9b2d96937a5c431c8e9bc437b4b75f7178427d297f88 network default NAD default] ADD starting CNI request [tsf-tas/rekor-search-ui-bdbb8865f-whbp8 bde4c2d8d4759f3125da9b2d96937a5c431c8e9bc437b4b75f7178427d297f88 network default NAD default] I0326 23:30:00.580707 3917 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-bdbb8865f-whbp8, hostIfaceName: bde4c2d8d4759f3, network: default, NAD default, SandboxID: "bde4c2d8d4759f3125da9b2d96937a5c431c8e9bc437b4b75f7178427d297f88", PCI device ID: , UID: "3dd30f22-3897-43b5-b889-f2cbd531b30c", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0326 23:30:00.627729 3917 cni.go:424] [tsf-tas/rekor-search-ui-bdbb8865f-whbp8 bde4c2d8d4759f3125da9b2d96937a5c431c8e9bc437b4b75f7178427d297f88 network default NAD default] ADD finished CNI request [tsf-tas/rekor-search-ui-bdbb8865f-whbp8 bde4c2d8d4759f3125da9b2d96937a5c431c8e9bc437b4b75f7178427d297f88 network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:b5:6f:0b:5c:9f\",\"name\":\"bde4c2d8d4759f3\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/70e6c018-28e3-4b03-b1ce-95093b9bbee1\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:30:00.662835 3917 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-x4w9m-fgdtp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:00.662856 3917 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-x4w9m-fgdtp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:00.893320 3917 cni.go:403] [tsf/tsf-tekton-configuration-xp8cm 3a292f4fac93916550b3743dcc6d27e4fdce9d9e23c1e36559fe76133b6daa86 network default NAD default] DEL starting CNI request [tsf/tsf-tekton-configuration-xp8cm 3a292f4fac93916550b3743dcc6d27e4fdce9d9e23c1e36559fe76133b6daa86 network default NAD default] I0326 23:30:00.966918 3917 cni.go:424] [tsf/tsf-tekton-configuration-xp8cm 3a292f4fac93916550b3743dcc6d27e4fdce9d9e23c1e36559fe76133b6daa86 network default NAD default] DEL finished CNI request [tsf/tsf-tekton-configuration-xp8cm 3a292f4fac93916550b3743dcc6d27e4fdce9d9e23c1e36559fe76133b6daa86 network default NAD default], result "{}", err I0326 23:30:01.043713 3917 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-xp8cm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:01.043763 3917 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-xp8cm I0326 23:30:01.045054 3917 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-xp8cm, ips: 10.129.0.39 I0326 23:30:01.045081 3917 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-xp8cm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:02.064030 3917 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-xp8cm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:02.064064 3917 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-xp8cm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:02.113234 3917 obj_retry.go:458] Detected object tsf/patch-tekton-config-hwrns of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:02.113268 3917 obj_retry.go:458] Detected object tsf/patch-tekton-config-hwrns of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:02.119422 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-hwrns I0326 23:30:02.119449 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-hwrns I0326 23:30:02.246628 3917 base_network_controller_pods.go:487] [default/tsf/test-tsf-pipelines] creating logical port tsf_test-tsf-pipelines for pod on switch ip-10-0-1-167.ec2.internal I0326 23:30:02.246730 3917 kube.go:256] Updating pod tsf/test-tsf-pipelines I0326 23:30:02.260045 3917 pod.go:62] [tsf/test-tsf-pipelines] pod update took 13.332859ms I0326 23:30:02.260128 3917 base_network_controller_pods.go:937] [default/tsf/test-tsf-pipelines] addLogicalPort annotation time took 13.418342ms I0326 23:30:02.261606 3917 pods.go:271] [tsf/test-tsf-pipelines] addLogicalPort took 14.996504ms, libovsdb time 941.854µs I0326 23:30:03.544225 3917 cni.go:403] [tsf/test-tsf-pipelines 53d666e3c74a9ac738f28dd34709d17660e37be51e9049038c1d5a476993e672 network default NAD default] ADD starting CNI request [tsf/test-tsf-pipelines 53d666e3c74a9ac738f28dd34709d17660e37be51e9049038c1d5a476993e672 network default NAD default] I0326 23:30:03.553669 3917 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-pipelines, hostIfaceName: 53d666e3c74a9ac, network: default, NAD default, SandboxID: "53d666e3c74a9ac738f28dd34709d17660e37be51e9049038c1d5a476993e672", PCI device ID: , UID: "f2e891b8-8c0f-47ce-beeb-26e143c02f6e", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0326 23:30:03.600920 3917 cni.go:424] [tsf/test-tsf-pipelines 53d666e3c74a9ac738f28dd34709d17660e37be51e9049038c1d5a476993e672 network default NAD default] ADD finished CNI request [tsf/test-tsf-pipelines 53d666e3c74a9ac738f28dd34709d17660e37be51e9049038c1d5a476993e672 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:8a:41:46:c3:91\",\"name\":\"53d666e3c74a9ac\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bcd9780c-b115-4ee2-bbdb-318a717462a5\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:30:03.905407 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0326 23:30:04.697035 3917 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zkbtd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:04.697560 3917 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zkbtd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:05.579914 3917 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zkbtd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:05.579944 3917 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zkbtd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:30:05.684232 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0326 23:30:05.702314 3917 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0326 23:30:09.374110 3917 namespace.go:142] [openshift-pipelines] updating namespace W0326 23:30:10.725459 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0326 23:30:13.965678 3917 cni.go:403] [tsf/test-tsf-pipelines 53d666e3c74a9ac738f28dd34709d17660e37be51e9049038c1d5a476993e672 network default NAD default] DEL starting CNI request [tsf/test-tsf-pipelines 53d666e3c74a9ac738f28dd34709d17660e37be51e9049038c1d5a476993e672 network default NAD default] I0326 23:30:14.056486 3917 cni.go:424] [tsf/test-tsf-pipelines 53d666e3c74a9ac738f28dd34709d17660e37be51e9049038c1d5a476993e672 network default NAD default] DEL finished CNI request [tsf/test-tsf-pipelines 53d666e3c74a9ac738f28dd34709d17660e37be51e9049038c1d5a476993e672 network default NAD default], result "{}", err I0326 23:30:14.107612 3917 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:14.107649 3917 pods.go:176] Deleting pod: tsf/test-tsf-pipelines I0326 23:30:14.108829 3917 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-pipelines, ips: 10.129.0.41 I0326 23:30:14.108857 3917 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:14.126182 3917 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:14.126206 3917 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:14.130511 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0326 23:30:14.130543 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines W0326 23:30:15.878201 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0326 23:30:18.921002 3917 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:18.921503 3917 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:18.942467 3917 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:18.942485 3917 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:18.947842 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0326 23:30:18.947867 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0326 23:30:24.824612 3917 obj_retry.go:464] Detected object tsf-tpa/create-db-zq5hr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:24.825206 3917 obj_retry.go:464] Detected object tsf-tpa/create-db-zq5hr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:25.846489 3917 obj_retry.go:458] Detected object tsf-tpa/create-db-zq5hr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:25.846513 3917 obj_retry.go:458] Detected object tsf-tpa/create-db-zq5hr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:26.049808 3917 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-kttm8-wpxpc] creating logical port tsf-tas_tuf-repository-init-kttm8-wpxpc for pod on switch ip-10-0-1-167.ec2.internal I0326 23:30:26.049904 3917 kube.go:256] Updating pod tsf-tas/tuf-repository-init-kttm8-wpxpc I0326 23:30:26.062509 3917 pod.go:62] [tsf-tas/tuf-repository-init-kttm8-wpxpc] pod update took 12.626093ms I0326 23:30:26.062534 3917 base_network_controller_pods.go:937] [default/tsf-tas/tuf-repository-init-kttm8-wpxpc] addLogicalPort annotation time took 12.654124ms I0326 23:30:26.063769 3917 pods.go:271] [tsf-tas/tuf-repository-init-kttm8-wpxpc] addLogicalPort took 13.959881ms, libovsdb time 774.471µs I0326 23:30:28.562039 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=71 I0326 23:30:28.839830 3917 obj_retry.go:464] Detected object tsf-tpa/migrate-db-ftp5f of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:28.840331 3917 obj_retry.go:464] Detected object tsf-tpa/migrate-db-ftp5f of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:28.970970 3917 obj_retry.go:458] Detected object tsf-tpa/migrate-db-ftp5f of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:28.970996 3917 obj_retry.go:458] Detected object tsf-tpa/migrate-db-ftp5f of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:31.851885 3917 obj_retry.go:464] Detected object tsf-tpa/create-importers-zmgm9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:31.852419 3917 obj_retry.go:464] Detected object tsf-tpa/create-importers-zmgm9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:32.129893 3917 cni.go:403] [tsf-tas/tuf-repository-init-kttm8-wpxpc 89872b53cab8eb8ff468e3f5cfb7b9de67b9501d12e0633233d8c587b09ff62b network default NAD default] ADD starting CNI request [tsf-tas/tuf-repository-init-kttm8-wpxpc 89872b53cab8eb8ff468e3f5cfb7b9de67b9501d12e0633233d8c587b09ff62b network default NAD default] I0326 23:30:32.138376 3917 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-kttm8-wpxpc, hostIfaceName: 89872b53cab8eb8, network: default, NAD default, SandboxID: "89872b53cab8eb8ff468e3f5cfb7b9de67b9501d12e0633233d8c587b09ff62b", PCI device ID: , UID: "4cb1d88f-33e4-4f48-a83a-4265e280afd4", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0326 23:30:32.186914 3917 cni.go:424] [tsf-tas/tuf-repository-init-kttm8-wpxpc 89872b53cab8eb8ff468e3f5cfb7b9de67b9501d12e0633233d8c587b09ff62b network default NAD default] ADD finished CNI request [tsf-tas/tuf-repository-init-kttm8-wpxpc 89872b53cab8eb8ff468e3f5cfb7b9de67b9501d12e0633233d8c587b09ff62b network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:ee:41:cd:59:f4\",\"name\":\"89872b53cab8eb8\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9bad0c91-0dbb-4300-85ef-907d0ce2e4c0\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:30:32.874126 3917 obj_retry.go:458] Detected object tsf-tpa/create-importers-zmgm9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:32.874148 3917 obj_retry.go:458] Detected object tsf-tpa/create-importers-zmgm9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:36.088916 3917 cni.go:403] [tsf-tas/tuf-repository-init-kttm8-wpxpc 89872b53cab8eb8ff468e3f5cfb7b9de67b9501d12e0633233d8c587b09ff62b network default NAD default] DEL starting CNI request [tsf-tas/tuf-repository-init-kttm8-wpxpc 89872b53cab8eb8ff468e3f5cfb7b9de67b9501d12e0633233d8c587b09ff62b network default NAD default] I0326 23:30:36.173972 3917 cni.go:424] [tsf-tas/tuf-repository-init-kttm8-wpxpc 89872b53cab8eb8ff468e3f5cfb7b9de67b9501d12e0633233d8c587b09ff62b network default NAD default] DEL finished CNI request [tsf-tas/tuf-repository-init-kttm8-wpxpc 89872b53cab8eb8ff468e3f5cfb7b9de67b9501d12e0633233d8c587b09ff62b network default NAD default], result "{}", err I0326 23:30:36.224994 3917 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-kttm8-wpxpc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:36.225030 3917 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-kttm8-wpxpc I0326 23:30:36.226439 3917 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-kttm8-wpxpc, ips: 10.129.0.42 I0326 23:30:36.226470 3917 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-kttm8-wpxpc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:37.246921 3917 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-kttm8-wpxpc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:37.246942 3917 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-kttm8-wpxpc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:38.161139 3917 base_network_controller_pods.go:487] [default/tsf-tas/tuf-bc5768c88-gsbzt] creating logical port tsf-tas_tuf-bc5768c88-gsbzt for pod on switch ip-10-0-1-167.ec2.internal I0326 23:30:38.161236 3917 kube.go:256] Updating pod tsf-tas/tuf-bc5768c88-gsbzt I0326 23:30:38.175306 3917 pod.go:62] [tsf-tas/tuf-bc5768c88-gsbzt] pod update took 14.09576ms I0326 23:30:38.175327 3917 base_network_controller_pods.go:937] [default/tsf-tas/tuf-bc5768c88-gsbzt] addLogicalPort annotation time took 14.122146ms I0326 23:30:38.176634 3917 pods.go:271] [tsf-tas/tuf-bc5768c88-gsbzt] addLogicalPort took 15.513816ms, libovsdb time 879.573µs I0326 23:30:42.283006 3917 cni.go:403] [tsf-tas/tuf-bc5768c88-gsbzt 92c5c7cbe73a4532077653b4d3b505e47217af92464eb67a73d3488f2499b137 network default NAD default] ADD starting CNI request [tsf-tas/tuf-bc5768c88-gsbzt 92c5c7cbe73a4532077653b4d3b505e47217af92464eb67a73d3488f2499b137 network default NAD default] I0326 23:30:42.291632 3917 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-bc5768c88-gsbzt, hostIfaceName: 92c5c7cbe73a453, network: default, NAD default, SandboxID: "92c5c7cbe73a4532077653b4d3b505e47217af92464eb67a73d3488f2499b137", PCI device ID: , UID: "760cc31f-064a-4126-9da8-c03654cee975", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0326 23:30:42.338342 3917 cni.go:424] [tsf-tas/tuf-bc5768c88-gsbzt 92c5c7cbe73a4532077653b4d3b505e47217af92464eb67a73d3488f2499b137 network default NAD default] ADD finished CNI request [tsf-tas/tuf-bc5768c88-gsbzt 92c5c7cbe73a4532077653b4d3b505e47217af92464eb67a73d3488f2499b137 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:b5:a6:98:ab:0e\",\"name\":\"92c5c7cbe73a453\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cc2ea211-c380-4b2f-8957-7dd4aaed1012\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:30:42.962363 3917 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0326 23:30:44.123264 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0326 23:30:48.559665 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 W0326 23:30:55.137471 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf I0326 23:31:02.570883 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=568 I0326 23:31:11.270124 3917 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:11.270705 3917 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:11.322431 3917 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:11.322451 3917 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:11.331553 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0326 23:31:11.331580 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0326 23:31:12.107299 3917 namespace.go:98] [image-controller] adding namespace I0326 23:31:12.107379 3917 namespace.go:98] [build-service] adding namespace I0326 23:31:12.108144 3917 namespace.go:102] [image-controller] adding namespace took 825.669µs I0326 23:31:12.108636 3917 namespace.go:102] [build-service] adding namespace took 1.239965ms I0326 23:31:12.109519 3917 namespace.go:98] [integration-service] adding namespace I0326 23:31:12.110153 3917 namespace.go:102] [integration-service] adding namespace took 618.945µs I0326 23:31:12.140264 3917 namespace.go:142] [image-controller] updating namespace I0326 23:31:12.162038 3917 namespace.go:142] [image-controller] updating namespace I0326 23:31:12.175228 3917 namespace.go:142] [build-service] updating namespace I0326 23:31:12.203066 3917 namespace.go:142] [build-service] updating namespace I0326 23:31:12.239146 3917 namespace.go:142] [integration-service] updating namespace I0326 23:31:12.259857 3917 namespace.go:142] [integration-service] updating namespace I0326 23:31:12.320363 3917 namespace.go:142] [build-service] updating namespace I0326 23:31:12.334624 3917 namespace.go:142] [integration-service] updating namespace I0326 23:31:12.364631 3917 namespace.go:98] [release-service] adding namespace I0326 23:31:12.365363 3917 namespace.go:102] [release-service] adding namespace took 718.425µs I0326 23:31:12.391389 3917 namespace.go:142] [release-service] updating namespace I0326 23:31:12.399787 3917 namespace.go:142] [konflux-ui] updating namespace I0326 23:31:12.432311 3917 namespace.go:142] [release-service] updating namespace I0326 23:31:12.474415 3917 base_network_controller_pods.go:487] [default/konflux-ui/tsf-konflux-public-key-gmmj8] creating logical port konflux-ui_tsf-konflux-public-key-gmmj8 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:31:12.474522 3917 kube.go:256] Updating pod konflux-ui/tsf-konflux-public-key-gmmj8 I0326 23:31:12.514451 3917 pod.go:62] [konflux-ui/tsf-konflux-public-key-gmmj8] pod update took 39.951189ms I0326 23:31:12.514473 3917 base_network_controller_pods.go:937] [default/konflux-ui/tsf-konflux-public-key-gmmj8] addLogicalPort annotation time took 39.977769ms I0326 23:31:12.515709 3917 pods.go:271] [konflux-ui/tsf-konflux-public-key-gmmj8] addLogicalPort took 41.304459ms, libovsdb time 775.321µs I0326 23:31:12.541974 3917 namespace.go:98] [konflux-info] adding namespace I0326 23:31:12.542691 3917 namespace.go:102] [konflux-info] adding namespace took 698.929µs I0326 23:31:12.691584 3917 namespace.go:98] [namespace-lister] adding namespace I0326 23:31:12.692244 3917 namespace.go:102] [namespace-lister] adding namespace took 645.145µs I0326 23:31:12.745520 3917 namespace.go:142] [konflux-info] updating namespace I0326 23:31:12.835695 3917 cni.go:403] [konflux-ui/tsf-konflux-public-key-gmmj8 d863868ba03f5c69f0ac1dededddc4ca6b86a422d03ea543eda9b455f24f60bd network default NAD default] ADD starting CNI request [konflux-ui/tsf-konflux-public-key-gmmj8 d863868ba03f5c69f0ac1dededddc4ca6b86a422d03ea543eda9b455f24f60bd network default NAD default] I0326 23:31:12.841702 3917 namespace.go:142] [konflux-info] updating namespace I0326 23:31:12.844492 3917 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: tsf-konflux-public-key-gmmj8, hostIfaceName: d863868ba03f5c6, network: default, NAD default, SandboxID: "d863868ba03f5c69f0ac1dededddc4ca6b86a422d03ea543eda9b455f24f60bd", PCI device ID: , UID: "fb0543d0-e1af-4377-87f4-354341e24492", MAC: 0a:58:0a:81:00:2c, IPs: [10.129.0.44/23] I0326 23:31:12.877557 3917 namespace.go:142] [build-service] updating namespace I0326 23:31:12.890012 3917 namespace.go:142] [namespace-lister] updating namespace I0326 23:31:12.891601 3917 cni.go:424] [konflux-ui/tsf-konflux-public-key-gmmj8 d863868ba03f5c69f0ac1dededddc4ca6b86a422d03ea543eda9b455f24f60bd network default NAD default] ADD finished CNI request [konflux-ui/tsf-konflux-public-key-gmmj8 d863868ba03f5c69f0ac1dededddc4ca6b86a422d03ea543eda9b455f24f60bd network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:90:5e:96:c7:65\",\"name\":\"d863868ba03f5c6\"},{\"mac\":\"0a:58:0a:81:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d253add2-f67f-41f7-8233-d79f91fdb570\"}],\"ips\":[{\"address\":\"10.129.0.44/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:31:12.924131 3917 namespace.go:142] [namespace-lister] updating namespace I0326 23:31:12.988360 3917 namespace.go:98] [enterprise-contract-service] adding namespace I0326 23:31:12.989184 3917 namespace.go:102] [enterprise-contract-service] adding namespace took 806.483µs I0326 23:31:13.035078 3917 namespace.go:142] [image-controller] updating namespace I0326 23:31:13.102648 3917 namespace.go:142] [image-controller] updating namespace I0326 23:31:13.182455 3917 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:31:13.297605 3917 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:31:13.459445 3917 namespace.go:142] [integration-service] updating namespace I0326 23:31:13.622171 3917 namespace.go:98] [default-tenant] adding namespace I0326 23:31:13.623097 3917 namespace.go:102] [default-tenant] adding namespace took 898.912µs I0326 23:31:13.840452 3917 namespace.go:98] [segment-bridge] adding namespace I0326 23:31:13.841242 3917 namespace.go:102] [segment-bridge] adding namespace took 765.955µs I0326 23:31:13.909974 3917 namespace.go:142] [default-tenant] updating namespace I0326 23:31:14.032413 3917 namespace.go:142] [default-tenant] updating namespace I0326 23:31:14.032500 3917 namespace.go:142] [segment-bridge] updating namespace I0326 23:31:14.043451 3917 namespace.go:142] [release-service] updating namespace I0326 23:31:14.156426 3917 namespace.go:142] [segment-bridge] updating namespace I0326 23:31:14.162534 3917 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0326 23:31:14.163181 3917 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0326 23:31:14.163198 3917 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0326 23:31:14.166017 3917 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0326 23:31:14.268886 3917 namespace.go:142] [build-service] updating namespace I0326 23:31:14.303364 3917 namespace.go:142] [image-controller] updating namespace I0326 23:31:14.320204 3917 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0326 23:31:14.320229 3917 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0326 23:31:14.321064 3917 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0326 23:31:14.329345 3917 namespace.go:142] [integration-service] updating namespace I0326 23:31:14.342848 3917 namespace.go:142] [release-service] updating namespace I0326 23:31:14.453949 3917 namespace.go:142] [build-service] updating namespace I0326 23:31:14.669284 3917 namespace.go:142] [image-controller] updating namespace I0326 23:31:14.776529 3917 namespace.go:142] [integration-service] updating namespace I0326 23:31:14.902354 3917 namespace.go:142] [release-service] updating namespace I0326 23:31:15.191888 3917 base_network_controller_pods.go:487] [default/konflux-ui/proxy-d66fd9698-ts62h] creating logical port konflux-ui_proxy-d66fd9698-ts62h for pod on switch ip-10-0-1-167.ec2.internal I0326 23:31:15.192008 3917 kube.go:256] Updating pod konflux-ui/proxy-d66fd9698-ts62h I0326 23:31:15.213961 3917 pod.go:62] [konflux-ui/proxy-d66fd9698-ts62h] pod update took 21.993531ms I0326 23:31:15.213986 3917 base_network_controller_pods.go:937] [default/konflux-ui/proxy-d66fd9698-ts62h] addLogicalPort annotation time took 22.023038ms I0326 23:31:15.215197 3917 pods.go:271] [konflux-ui/proxy-d66fd9698-ts62h] addLogicalPort took 23.325454ms, libovsdb time 838.71µs I0326 23:31:15.297296 3917 cni.go:403] [konflux-ui/tsf-konflux-public-key-gmmj8 d863868ba03f5c69f0ac1dededddc4ca6b86a422d03ea543eda9b455f24f60bd network default NAD default] DEL starting CNI request [konflux-ui/tsf-konflux-public-key-gmmj8 d863868ba03f5c69f0ac1dededddc4ca6b86a422d03ea543eda9b455f24f60bd network default NAD default] I0326 23:31:15.393626 3917 cni.go:424] [konflux-ui/tsf-konflux-public-key-gmmj8 d863868ba03f5c69f0ac1dededddc4ca6b86a422d03ea543eda9b455f24f60bd network default NAD default] DEL finished CNI request [konflux-ui/tsf-konflux-public-key-gmmj8 d863868ba03f5c69f0ac1dededddc4ca6b86a422d03ea543eda9b455f24f60bd network default NAD default], result "{}", err I0326 23:31:15.451247 3917 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-gmmj8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:15.451280 3917 pods.go:176] Deleting pod: konflux-ui/tsf-konflux-public-key-gmmj8 I0326 23:31:15.452830 3917 pods.go:236] Attempting to release IPs for pod: konflux-ui/tsf-konflux-public-key-gmmj8, ips: 10.129.0.44 I0326 23:31:15.452858 3917 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-gmmj8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:15.478333 3917 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7df75dd4df-lf7d7] creating logical port build-service_build-service-controller-manager-7df75dd4df-lf7d7 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:31:15.478462 3917 kube.go:256] Updating pod build-service/build-service-controller-manager-7df75dd4df-lf7d7 I0326 23:31:15.498993 3917 pod.go:62] [build-service/build-service-controller-manager-7df75dd4df-lf7d7] pod update took 20.558086ms I0326 23:31:15.499017 3917 base_network_controller_pods.go:937] [default/build-service/build-service-controller-manager-7df75dd4df-lf7d7] addLogicalPort annotation time took 20.587404ms I0326 23:31:15.500340 3917 pods.go:271] [build-service/build-service-controller-manager-7df75dd4df-lf7d7] addLogicalPort took 22.02492ms, libovsdb time 894.024µs I0326 23:31:15.565386 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=139 I0326 23:31:15.901510 3917 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-854f775895-t84r4] creating logical port release-service_release-service-controller-manager-854f775895-t84r4 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:31:15.901631 3917 kube.go:256] Updating pod release-service/release-service-controller-manager-854f775895-t84r4 I0326 23:31:15.917815 3917 pod.go:62] [release-service/release-service-controller-manager-854f775895-t84r4] pod update took 16.205764ms I0326 23:31:15.917843 3917 base_network_controller_pods.go:937] [default/release-service/release-service-controller-manager-854f775895-t84r4] addLogicalPort annotation time took 16.237195ms I0326 23:31:15.919114 3917 pods.go:271] [release-service/release-service-controller-manager-854f775895-t84r4] addLogicalPort took 17.622217ms, libovsdb time 859.025µs I0326 23:31:16.066160 3917 namespace.go:142] [default-tenant] updating namespace I0326 23:31:16.151737 3917 cni.go:403] [build-service/build-service-controller-manager-7df75dd4df-lf7d7 dfee821681f5a90efe37b43d7518d8736049049240254090bb129b6d3c7b60f0 network default NAD default] ADD starting CNI request [build-service/build-service-controller-manager-7df75dd4df-lf7d7 dfee821681f5a90efe37b43d7518d8736049049240254090bb129b6d3c7b60f0 network default NAD default] I0326 23:31:16.160197 3917 helper_linux.go:450] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7df75dd4df-lf7d7, hostIfaceName: dfee821681f5a90, network: default, NAD default, SandboxID: "dfee821681f5a90efe37b43d7518d8736049049240254090bb129b6d3c7b60f0", PCI device ID: , UID: "9382449f-a4e7-4136-98a4-b6f8e667abdc", MAC: 0a:58:0a:81:00:2e, IPs: [10.129.0.46/23] I0326 23:31:16.206567 3917 cni.go:424] [build-service/build-service-controller-manager-7df75dd4df-lf7d7 dfee821681f5a90efe37b43d7518d8736049049240254090bb129b6d3c7b60f0 network default NAD default] ADD finished CNI request [build-service/build-service-controller-manager-7df75dd4df-lf7d7 dfee821681f5a90efe37b43d7518d8736049049240254090bb129b6d3c7b60f0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:51:ba:e9:0b:e4\",\"name\":\"dfee821681f5a90\"},{\"mac\":\"0a:58:0a:81:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/abdbdfcf-5d4f-49de-a413-d2a1c74eecea\"}],\"ips\":[{\"address\":\"10.129.0.46/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:31:16.283508 3917 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:31:16.476884 3917 namespace.go:142] [konflux-info] updating namespace I0326 23:31:16.500046 3917 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-gmmj8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:16.500069 3917 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-gmmj8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:16.671212 3917 namespace.go:142] [namespace-lister] updating namespace I0326 23:31:16.721581 3917 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-1-167.ec2.internal I0326 23:31:16.721670 3917 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0326 23:31:16.737778 3917 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 16.138547ms I0326 23:31:16.737801 3917 base_network_controller_pods.go:937] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 16.164804ms I0326 23:31:16.739159 3917 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 17.596209ms, libovsdb time 946.723µs I0326 23:31:16.820993 3917 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-8487c7c664-w76lc] creating logical port integration-service_integration-service-controller-manager-8487c7c664-w76lc for pod on switch ip-10-0-1-167.ec2.internal I0326 23:31:16.821079 3917 kube.go:256] Updating pod integration-service/integration-service-controller-manager-8487c7c664-w76lc I0326 23:31:16.836113 3917 pod.go:62] [integration-service/integration-service-controller-manager-8487c7c664-w76lc] pod update took 15.058826ms I0326 23:31:16.836133 3917 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-8487c7c664-w76lc] addLogicalPort annotation time took 15.084193ms I0326 23:31:16.837438 3917 pods.go:271] [integration-service/integration-service-controller-manager-8487c7c664-w76lc] addLogicalPort took 16.462019ms, libovsdb time 914.056µs I0326 23:31:16.886166 3917 namespace.go:142] [segment-bridge] updating namespace W0326 23:31:16.988866 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0326 23:31:16.998636 3917 namespace.go:142] [default-tenant] updating namespace I0326 23:31:17.017115 3917 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:31:17.033958 3917 namespace.go:142] [konflux-info] updating namespace I0326 23:31:17.047995 3917 namespace.go:142] [namespace-lister] updating namespace I0326 23:31:17.061355 3917 namespace.go:142] [segment-bridge] updating namespace I0326 23:31:17.089429 3917 cni.go:403] [konflux-ui/test-tsf-konflux 2ea151041eb558cd09bd15688a0c0429e2bd3560fccfb808bed39e6bf1908d29 network default NAD default] ADD starting CNI request [konflux-ui/test-tsf-konflux 2ea151041eb558cd09bd15688a0c0429e2bd3560fccfb808bed39e6bf1908d29 network default NAD default] I0326 23:31:17.099711 3917 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: 2ea151041eb558c, network: default, NAD default, SandboxID: "2ea151041eb558cd09bd15688a0c0429e2bd3560fccfb808bed39e6bf1908d29", PCI device ID: , UID: "e02b9def-ca30-4584-b3f1-9376e133944f", MAC: 0a:58:0a:81:00:30, IPs: [10.129.0.48/23] I0326 23:31:17.105279 3917 namespace.go:142] [default-tenant] updating namespace I0326 23:31:17.154023 3917 cni.go:424] [konflux-ui/test-tsf-konflux 2ea151041eb558cd09bd15688a0c0429e2bd3560fccfb808bed39e6bf1908d29 network default NAD default] ADD finished CNI request [konflux-ui/test-tsf-konflux 2ea151041eb558cd09bd15688a0c0429e2bd3560fccfb808bed39e6bf1908d29 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:2f:4d:bb:61:15\",\"name\":\"2ea151041eb558c\"},{\"mac\":\"0a:58:0a:81:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/03eae44b-3ce6-4581-b0e0-c1d582b1446a\"}],\"ips\":[{\"address\":\"10.129.0.48/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:31:17.193632 3917 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:31:17.282041 3917 namespace.go:142] [konflux-info] updating namespace I0326 23:31:17.382042 3917 namespace.go:142] [namespace-lister] updating namespace I0326 23:31:17.482813 3917 namespace.go:142] [segment-bridge] updating namespace W0326 23:31:18.963915 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0326 23:31:19.338633 3917 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0326 23:31:25.662324 3917 cni.go:403] [integration-service/integration-service-controller-manager-8487c7c664-w76lc 8e0928a51e8007a791f011c3aa4263dfa91b8ec968436afdc881de33014000b0 network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-8487c7c664-w76lc 8e0928a51e8007a791f011c3aa4263dfa91b8ec968436afdc881de33014000b0 network default NAD default] I0326 23:31:25.662930 3917 cni.go:403] [konflux-ui/proxy-d66fd9698-ts62h 42af42a35eb004c4cb47c6c9445cb2b89210af57ecf4fe524e2e1501385dc1c0 network default NAD default] ADD starting CNI request [konflux-ui/proxy-d66fd9698-ts62h 42af42a35eb004c4cb47c6c9445cb2b89210af57ecf4fe524e2e1501385dc1c0 network default NAD default] I0326 23:31:25.666206 3917 cni.go:403] [release-service/release-service-controller-manager-854f775895-t84r4 5305e61345d9ff3ea1d313757d497706129def23fb541b2843ba2635b37bd183 network default NAD default] ADD starting CNI request [release-service/release-service-controller-manager-854f775895-t84r4 5305e61345d9ff3ea1d313757d497706129def23fb541b2843ba2635b37bd183 network default NAD default] I0326 23:31:25.674265 3917 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-d66fd9698-ts62h, hostIfaceName: 42af42a35eb004c, network: default, NAD default, SandboxID: "42af42a35eb004c4cb47c6c9445cb2b89210af57ecf4fe524e2e1501385dc1c0", PCI device ID: , UID: "660893f0-b477-4cc9-bad5-74cc3f21eb9c", MAC: 0a:58:0a:81:00:2d, IPs: [10.129.0.45/23] I0326 23:31:25.674433 3917 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-8487c7c664-w76lc, hostIfaceName: 8e0928a51e8007a, network: default, NAD default, SandboxID: "8e0928a51e8007a791f011c3aa4263dfa91b8ec968436afdc881de33014000b0", PCI device ID: , UID: "6a26b4be-d0b7-4e37-99f9-230e550c7e29", MAC: 0a:58:0a:81:00:31, IPs: [10.129.0.49/23] I0326 23:31:25.689430 3917 helper_linux.go:450] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-854f775895-t84r4, hostIfaceName: 5305e61345d9ff3, network: default, NAD default, SandboxID: "5305e61345d9ff3ea1d313757d497706129def23fb541b2843ba2635b37bd183", PCI device ID: , UID: "a8f50d7a-17e9-4404-9cbb-ceb6b60db3ce", MAC: 0a:58:0a:81:00:2f, IPs: [10.129.0.47/23] I0326 23:31:25.732037 3917 cni.go:424] [integration-service/integration-service-controller-manager-8487c7c664-w76lc 8e0928a51e8007a791f011c3aa4263dfa91b8ec968436afdc881de33014000b0 network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-8487c7c664-w76lc 8e0928a51e8007a791f011c3aa4263dfa91b8ec968436afdc881de33014000b0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:24:5f:d3:9c:37\",\"name\":\"8e0928a51e8007a\"},{\"mac\":\"0a:58:0a:81:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/107ac3a0-a12b-4096-8670-6a84dbece0e3\"}],\"ips\":[{\"address\":\"10.129.0.49/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:31:25.752483 3917 cni.go:424] [konflux-ui/proxy-d66fd9698-ts62h 42af42a35eb004c4cb47c6c9445cb2b89210af57ecf4fe524e2e1501385dc1c0 network default NAD default] ADD finished CNI request [konflux-ui/proxy-d66fd9698-ts62h 42af42a35eb004c4cb47c6c9445cb2b89210af57ecf4fe524e2e1501385dc1c0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:7d:b1:a5:b8:f0\",\"name\":\"42af42a35eb004c\"},{\"mac\":\"0a:58:0a:81:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/87ac5266-432c-4814-9855-70e8107b342f\"}],\"ips\":[{\"address\":\"10.129.0.45/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:31:25.772232 3917 cni.go:424] [release-service/release-service-controller-manager-854f775895-t84r4 5305e61345d9ff3ea1d313757d497706129def23fb541b2843ba2635b37bd183 network default NAD default] ADD finished CNI request [release-service/release-service-controller-manager-854f775895-t84r4 5305e61345d9ff3ea1d313757d497706129def23fb541b2843ba2635b37bd183 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:5e:86:86:0d:4e\",\"name\":\"5305e61345d9ff3\"},{\"mac\":\"0a:58:0a:81:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ba704cf4-a128-46a8-8095-ce1bbd01c563\"}],\"ips\":[{\"address\":\"10.129.0.47/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:31:26.017107 3917 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0326 23:31:30.046854 3917 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0326 23:31:30.046882 3917 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0326 23:31:30.089998 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:31:30.092069 3917 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:31:30.331077 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0326 23:31:32.408862 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0326 23:31:32.409837 3917 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0326 23:31:32.442579 3917 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0326 23:31:32.442596 3917 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0326 23:31:32.465109 3917 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:31:32.466082 3917 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:31:41.075112 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:31:41.075136 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:31:41.076112 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:31:41.076133 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:31:43.422600 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:31:43.424544 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:31:43.453711 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0326 23:31:43.453860 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service I0326 23:31:44.340263 3917 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-gnjzz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:44.340832 3917 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-gnjzz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:45.134837 3917 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-gnjzz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:45.134865 3917 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-gnjzz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:45.137972 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-gnjzz I0326 23:31:45.137994 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-gnjzz I0326 23:31:55.652431 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 W0326 23:32:08.111147 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0326 23:32:08.111169 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0326 23:32:08.111185 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0326 23:32:08.111192 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0326 23:32:08.462301 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0326 23:32:08.462325 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0326 23:32:13.668853 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0326 23:32:27.561698 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=171 I0326 23:32:30.647129 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0326 23:32:36.679919 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0326 23:32:38.213635 3917 cni.go:403] [konflux-ui/proxy-d66fd9698-ts62h 42af42a35eb004c4cb47c6c9445cb2b89210af57ecf4fe524e2e1501385dc1c0 network default NAD default] DEL starting CNI request [konflux-ui/proxy-d66fd9698-ts62h 42af42a35eb004c4cb47c6c9445cb2b89210af57ecf4fe524e2e1501385dc1c0 network default NAD default] I0326 23:32:38.304022 3917 cni.go:424] [konflux-ui/proxy-d66fd9698-ts62h 42af42a35eb004c4cb47c6c9445cb2b89210af57ecf4fe524e2e1501385dc1c0 network default NAD default] DEL finished CNI request [konflux-ui/proxy-d66fd9698-ts62h 42af42a35eb004c4cb47c6c9445cb2b89210af57ecf4fe524e2e1501385dc1c0 network default NAD default], result "{}", err I0326 23:32:38.366896 3917 obj_retry.go:464] Detected object konflux-ui/proxy-d66fd9698-ts62h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:38.366930 3917 pods.go:176] Deleting pod: konflux-ui/proxy-d66fd9698-ts62h I0326 23:32:38.368257 3917 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-d66fd9698-ts62h, ips: 10.129.0.45 I0326 23:32:38.368288 3917 obj_retry.go:464] Detected object konflux-ui/proxy-d66fd9698-ts62h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:38.774042 3917 obj_retry.go:458] Detected object konflux-ui/proxy-d66fd9698-ts62h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:38.774070 3917 obj_retry.go:458] Detected object konflux-ui/proxy-d66fd9698-ts62h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:38.780373 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-d66fd9698-ts62h I0326 23:32:38.780406 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-d66fd9698-ts62h I0326 23:32:47.831964 3917 cni.go:403] [konflux-ui/test-tsf-konflux 2ea151041eb558cd09bd15688a0c0429e2bd3560fccfb808bed39e6bf1908d29 network default NAD default] DEL starting CNI request [konflux-ui/test-tsf-konflux 2ea151041eb558cd09bd15688a0c0429e2bd3560fccfb808bed39e6bf1908d29 network default NAD default] I0326 23:32:47.924135 3917 cni.go:424] [konflux-ui/test-tsf-konflux 2ea151041eb558cd09bd15688a0c0429e2bd3560fccfb808bed39e6bf1908d29 network default NAD default] DEL finished CNI request [konflux-ui/test-tsf-konflux 2ea151041eb558cd09bd15688a0c0429e2bd3560fccfb808bed39e6bf1908d29 network default NAD default], result "{}", err I0326 23:32:47.982320 3917 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:47.982354 3917 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0326 23:32:47.983827 3917 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.129.0.48 I0326 23:32:47.983860 3917 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:48.006741 3917 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:48.006779 3917 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:48.014954 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0326 23:32:48.014982 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0326 23:33:13.755081 3917 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-t7vzh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:13.755704 3917 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-t7vzh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:14.615605 3917 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-t7vzh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:14.615630 3917 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-t7vzh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:14.621576 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-t7vzh I0326 23:33:14.621605 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-t7vzh I0326 23:33:26.955172 3917 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-k4mhh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:26.955763 3917 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-k4mhh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:27.683855 3917 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-k4mhh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:27.683878 3917 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-k4mhh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:27.691116 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-k4mhh I0326 23:33:27.691139 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-k4mhh I0326 23:33:31.673474 3917 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0326 23:33:52.666213 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0326 23:34:32.642603 3917 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0326 23:36:01.573315 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=52 I0326 23:36:01.642974 3917 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=11 I0326 23:36:03.679549 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0326 23:36:16.563623 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0326 23:37:42.660984 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0326 23:37:50.574876 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=128 I0326 23:38:53.568953 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=23 I0326 23:39:07.683900 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0326 23:39:18.565496 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=55 I0326 23:39:38.564878 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=17 I0326 23:40:03.878833 3917 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-8c747 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:03.879453 3917 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-8c747 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:04.765051 3917 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-8c747 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:04.765099 3917 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-8c747 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:21.676525 3917 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0326 23:40:25.049469 3917 obj_retry.go:458] Detected object tsf-tpa/create-db-zq5hr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:25.049498 3917 obj_retry.go:458] Detected object tsf-tpa/create-db-zq5hr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:25.055400 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-zq5hr I0326 23:40:25.055426 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-zq5hr I0326 23:40:28.028637 3917 obj_retry.go:458] Detected object tsf-tpa/migrate-db-ftp5f of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:28.028667 3917 obj_retry.go:458] Detected object tsf-tpa/migrate-db-ftp5f of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:28.033989 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-ftp5f I0326 23:40:28.034014 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-ftp5f I0326 23:40:32.039979 3917 obj_retry.go:458] Detected object tsf-tpa/create-importers-zmgm9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:32.040024 3917 obj_retry.go:458] Detected object tsf-tpa/create-importers-zmgm9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:32.044840 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-zmgm9 I0326 23:40:32.044870 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-zmgm9 I0326 23:40:50.672521 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0326 23:40:57.655245 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0326 23:41:08.576943 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=6 I0326 23:41:16.645962 3917 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0326 23:41:22.566723 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=5 I0326 23:41:41.683236 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0326 23:42:14.650645 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0326 23:43:27.669353 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=11 I0326 23:43:51.646975 3917 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0326 23:44:52.569365 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=29 I0326 23:45:15.687693 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0326 23:45:48.568067 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=7 I0326 23:46:03.578881 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=24 I0326 23:46:22.074852 3917 namespace.go:98] [default-tenant-managed] adding namespace I0326 23:46:22.075839 3917 namespace.go:102] [default-tenant-managed] adding namespace took 962.877µs I0326 23:46:22.124412 3917 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:46:22.142371 3917 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:46:23.118836 3917 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:46:23.250401 3917 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:46:23.351796 3917 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:46:54.679544 3917 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0326 23:47:08.966217 3917 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-588z9] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-588z9 for pod on switch ip-10-0-1-167.ec2.internal I0326 23:47:08.966348 3917 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-588z9 I0326 23:47:09.002453 3917 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-588z9] pod update took 36.137201ms I0326 23:47:09.002486 3917 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-588z9] addLogicalPort annotation time took 36.176274ms I0326 23:47:09.003924 3917 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-588z9] addLogicalPort took 37.723073ms, libovsdb time 974.523µs I0326 23:47:09.330493 3917 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-588z9 216c48f3c42b951189498969aceb4d0f33aace25ef04a170edfc874f4a4bd1bc network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-588z9 216c48f3c42b951189498969aceb4d0f33aace25ef04a170edfc874f4a4bd1bc network default NAD default] I0326 23:47:09.339937 3917 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-588z9, hostIfaceName: 216c48f3c42b951, network: default, NAD default, SandboxID: "216c48f3c42b951189498969aceb4d0f33aace25ef04a170edfc874f4a4bd1bc", PCI device ID: , UID: "ebd27c4a-8e02-412a-9454-158cd2fededa", MAC: 0a:58:0a:81:00:32, IPs: [10.129.0.50/23] I0326 23:47:09.394076 3917 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-588z9 216c48f3c42b951189498969aceb4d0f33aace25ef04a170edfc874f4a4bd1bc network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-588z9 216c48f3c42b951189498969aceb4d0f33aace25ef04a170edfc874f4a4bd1bc network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:60:72:d0:0d:ae\",\"name\":\"216c48f3c42b951\"},{\"mac\":\"0a:58:0a:81:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cab54068-b8d2-4c83-8777-38f01109a6ec\"}],\"ips\":[{\"address\":\"10.129.0.50/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:47:11.686269 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0326 23:47:13.750865 3917 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod] creating logical port default-tenant_tsf-comp-tzwe-on-push-b5mcb-init-pod for pod on switch ip-10-0-1-167.ec2.internal I0326 23:47:13.751031 3917 kube.go:256] Updating pod default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod I0326 23:47:13.768770 3917 pod.go:62] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod] pod update took 17.746184ms I0326 23:47:13.768803 3917 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod] addLogicalPort annotation time took 17.80402ms I0326 23:47:13.770229 3917 pods.go:271] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod] addLogicalPort took 19.388326ms, libovsdb time 958.656µs I0326 23:47:14.118076 3917 cni.go:403] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod 255d2565b0f7e6e51d479fca9c903ba933619e72f98056c82ee4627dd3ccb371 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod 255d2565b0f7e6e51d479fca9c903ba933619e72f98056c82ee4627dd3ccb371 network default NAD default] I0326 23:47:14.127026 3917 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-tzwe-on-push-b5mcb-init-pod, hostIfaceName: 255d2565b0f7e6e, network: default, NAD default, SandboxID: "255d2565b0f7e6e51d479fca9c903ba933619e72f98056c82ee4627dd3ccb371", PCI device ID: , UID: "f01710e5-33ee-4c83-9b12-14b6936dd3e3", MAC: 0a:58:0a:81:00:33, IPs: [10.129.0.51/23] I0326 23:47:14.180632 3917 cni.go:424] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod 255d2565b0f7e6e51d479fca9c903ba933619e72f98056c82ee4627dd3ccb371 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod 255d2565b0f7e6e51d479fca9c903ba933619e72f98056c82ee4627dd3ccb371 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:1b:b0:5b:3b:eb\",\"name\":\"255d2565b0f7e6e\"},{\"mac\":\"0a:58:0a:81:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/990372e3-fef1-48c8-98ad-8952705b1833\"}],\"ips\":[{\"address\":\"10.129.0.51/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:47:20.664860 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 W0326 23:47:20.990811 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:47:20.990831 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:47:20.990843 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:47:20.990848 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:47:20.990864 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:47:20.990868 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:47:20.990874 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:47:20.990878 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:47:22.554145 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:47:22.554168 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:47:22.554177 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:47:22.554184 3917 util.go:959] Failed to build node endpoints for node ip-10-0-1-167.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0326 23:47:30.668763 3917 cni.go:403] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod 255d2565b0f7e6e51d479fca9c903ba933619e72f98056c82ee4627dd3ccb371 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod 255d2565b0f7e6e51d479fca9c903ba933619e72f98056c82ee4627dd3ccb371 network default NAD default] I0326 23:47:30.758041 3917 cni.go:424] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod 255d2565b0f7e6e51d479fca9c903ba933619e72f98056c82ee4627dd3ccb371 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod 255d2565b0f7e6e51d479fca9c903ba933619e72f98056c82ee4627dd3ccb371 network default NAD default], result "{}", err I0326 23:47:30.910713 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:30.910775 3917 pods.go:176] Deleting pod: default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod I0326 23:47:30.912162 3917 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod, ips: 10.129.0.51 I0326 23:47:30.912196 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:34.006237 3917 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod] creating logical port default-tenant_tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod for pod on switch ip-10-0-1-167.ec2.internal I0326 23:47:34.006397 3917 kube.go:256] Updating pod default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod I0326 23:47:34.031717 3917 pod.go:62] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod] pod update took 25.350051ms I0326 23:47:34.031767 3917 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod] addLogicalPort annotation time took 25.385198ms I0326 23:47:34.033707 3917 pods.go:271] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod] addLogicalPort took 27.482663ms, libovsdb time 1.342786ms I0326 23:47:34.402319 3917 cni.go:403] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod dfb99420bac7449bdde214e0e7afccd8e2ddb1732ede44c79cdff37b0fea32c3 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod dfb99420bac7449bdde214e0e7afccd8e2ddb1732ede44c79cdff37b0fea32c3 network default NAD default] I0326 23:47:34.411622 3917 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod, hostIfaceName: dfb99420bac7449, network: default, NAD default, SandboxID: "dfb99420bac7449bdde214e0e7afccd8e2ddb1732ede44c79cdff37b0fea32c3", PCI device ID: , UID: "0a6c6fef-874f-4f29-873f-ee889ac90f84", MAC: 0a:58:0a:81:00:34, IPs: [10.129.0.52/23] I0326 23:47:34.461435 3917 cni.go:424] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod dfb99420bac7449bdde214e0e7afccd8e2ddb1732ede44c79cdff37b0fea32c3 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod dfb99420bac7449bdde214e0e7afccd8e2ddb1732ede44c79cdff37b0fea32c3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:53:5e:3e:93:a6\",\"name\":\"dfb99420bac7449\"},{\"mac\":\"0a:58:0a:81:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4906e31f-4294-47e0-b1b7-d00312aea830\"}],\"ips\":[{\"address\":\"10.129.0.52/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:47:42.686131 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=5 I0326 23:47:49.660202 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0326 23:48:00.881376 3917 cni.go:403] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod dfb99420bac7449bdde214e0e7afccd8e2ddb1732ede44c79cdff37b0fea32c3 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod dfb99420bac7449bdde214e0e7afccd8e2ddb1732ede44c79cdff37b0fea32c3 network default NAD default] I0326 23:48:00.985781 3917 cni.go:424] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod dfb99420bac7449bdde214e0e7afccd8e2ddb1732ede44c79cdff37b0fea32c3 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod dfb99420bac7449bdde214e0e7afccd8e2ddb1732ede44c79cdff37b0fea32c3 network default NAD default], result "{}", err I0326 23:48:01.064190 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:01.064218 3917 pods.go:176] Deleting pod: default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod I0326 23:48:01.065605 3917 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod, ips: 10.129.0.52 I0326 23:48:01.065644 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:07.570404 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0326 23:48:14.572598 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=17 I0326 23:48:19.568227 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:19.568800 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:19.581914 3917 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod] creating logical port default-tenant_tsf-comp-tzwe-on-push-b5mcb-build-container-pod for pod on switch ip-10-0-1-167.ec2.internal I0326 23:48:19.582054 3917 kube.go:256] Updating pod default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod I0326 23:48:19.628663 3917 pod.go:62] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod] pod update took 46.647185ms I0326 23:48:19.628697 3917 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod] addLogicalPort annotation time took 46.687566ms I0326 23:48:19.630529 3917 pods.go:271] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod] addLogicalPort took 48.637435ms, libovsdb time 1.219364ms I0326 23:48:19.965928 3917 cni.go:403] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod 2371c3513fa1195dc70fad5e45a1493735d2ec055d949822d1849530cb830b35 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod 2371c3513fa1195dc70fad5e45a1493735d2ec055d949822d1849530cb830b35 network default NAD default] I0326 23:48:19.975843 3917 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-tzwe-on-push-b5mcb-build-container-pod, hostIfaceName: 2371c3513fa1195, network: default, NAD default, SandboxID: "2371c3513fa1195dc70fad5e45a1493735d2ec055d949822d1849530cb830b35", PCI device ID: , UID: "c21201dd-a2ad-42f1-9c2c-cb13c47bae2b", MAC: 0a:58:0a:81:00:35, IPs: [10.129.0.53/23] I0326 23:48:20.235491 3917 cni.go:424] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod 2371c3513fa1195dc70fad5e45a1493735d2ec055d949822d1849530cb830b35 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod 2371c3513fa1195dc70fad5e45a1493735d2ec055d949822d1849530cb830b35 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:d9:a6:5d:cd:ed\",\"name\":\"2371c3513fa1195\"},{\"mac\":\"0a:58:0a:81:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/96586f81-6adf-4a10-ac24-8a704758af5b\"}],\"ips\":[{\"address\":\"10.129.0.53/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:49:06.649798 3917 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0326 23:49:28.580498 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=9 I0326 23:49:35.651084 3917 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0326 23:50:04.110726 3917 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-dh8hp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:04.111374 3917 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-dh8hp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:04.972711 3917 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-dh8hp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:04.972757 3917 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-dh8hp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:05.009514 3917 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-4lsr2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:05.009536 3917 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-4lsr2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:05.013968 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-4lsr2 I0326 23:50:05.013997 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-4lsr2 I0326 23:50:07.658461 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0326 23:50:22.672524 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0326 23:50:36.886567 3917 cni.go:403] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod 2371c3513fa1195dc70fad5e45a1493735d2ec055d949822d1849530cb830b35 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod 2371c3513fa1195dc70fad5e45a1493735d2ec055d949822d1849530cb830b35 network default NAD default] I0326 23:50:36.966721 3917 cni.go:424] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod 2371c3513fa1195dc70fad5e45a1493735d2ec055d949822d1849530cb830b35 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod 2371c3513fa1195dc70fad5e45a1493735d2ec055d949822d1849530cb830b35 network default NAD default], result "{}", err I0326 23:50:37.077987 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:37.078054 3917 pods.go:176] Deleting pod: default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod I0326 23:50:37.088942 3917 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod, ips: 10.129.0.53 I0326 23:50:37.088996 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:44.572680 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=37 I0326 23:51:16.968167 3917 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod] creating logical port default-tenant_tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod for pod on switch ip-10-0-1-167.ec2.internal I0326 23:51:16.968315 3917 kube.go:256] Updating pod default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod I0326 23:51:16.987796 3917 pod.go:62] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod] pod update took 19.509657ms I0326 23:51:16.987826 3917 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod] addLogicalPort annotation time took 19.549581ms I0326 23:51:16.989291 3917 pods.go:271] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod] addLogicalPort took 21.142681ms, libovsdb time 1.037096ms I0326 23:51:17.032308 3917 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod] creating logical port default-tenant_tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod for pod on switch ip-10-0-1-167.ec2.internal I0326 23:51:17.032505 3917 kube.go:256] Updating pod default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod I0326 23:51:17.060094 3917 pod.go:62] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod] pod update took 27.654945ms I0326 23:51:17.060129 3917 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod] addLogicalPort annotation time took 27.692865ms I0326 23:51:17.061515 3917 pods.go:271] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod] addLogicalPort took 29.223519ms, libovsdb time 950.535µs I0326 23:51:17.335020 3917 cni.go:403] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod f4d49d21516ea21d18fb1566c8b84888e96848ed302d5788a03688eda86c3687 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod f4d49d21516ea21d18fb1566c8b84888e96848ed302d5788a03688eda86c3687 network default NAD default] I0326 23:51:17.345044 3917 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod, hostIfaceName: f4d49d21516ea21, network: default, NAD default, SandboxID: "f4d49d21516ea21d18fb1566c8b84888e96848ed302d5788a03688eda86c3687", PCI device ID: , UID: "55cbc5d2-69f0-4617-a660-45effafd90d8", MAC: 0a:58:0a:81:00:36, IPs: [10.129.0.54/23] I0326 23:51:17.402094 3917 cni.go:424] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod f4d49d21516ea21d18fb1566c8b84888e96848ed302d5788a03688eda86c3687 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod f4d49d21516ea21d18fb1566c8b84888e96848ed302d5788a03688eda86c3687 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:65:bb:b0:27:87\",\"name\":\"f4d49d21516ea21\"},{\"mac\":\"0a:58:0a:81:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/83bdda5a-a13b-4f83-986b-a705518d6466\"}],\"ips\":[{\"address\":\"10.129.0.54/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:51:17.404649 3917 cni.go:403] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod fa9e548069803f34a9fd316e6f617de372e93151eef55acdb81dbe9fa3dd692e network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod fa9e548069803f34a9fd316e6f617de372e93151eef55acdb81dbe9fa3dd692e network default NAD default] I0326 23:51:17.414379 3917 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod, hostIfaceName: fa9e548069803f3, network: default, NAD default, SandboxID: "fa9e548069803f34a9fd316e6f617de372e93151eef55acdb81dbe9fa3dd692e", PCI device ID: , UID: "650562a5-5c67-4fce-81b8-208ea4a2ef29", MAC: 0a:58:0a:81:00:37, IPs: [10.129.0.55/23] I0326 23:51:17.469777 3917 cni.go:424] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod fa9e548069803f34a9fd316e6f617de372e93151eef55acdb81dbe9fa3dd692e network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod fa9e548069803f34a9fd316e6f617de372e93151eef55acdb81dbe9fa3dd692e network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:fe:0a:2e:22:09\",\"name\":\"fa9e548069803f3\"},{\"mac\":\"0a:58:0a:81:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3e4a1e07-ccfc-46a3-81e7-de3ff99e748e\"}],\"ips\":[{\"address\":\"10.129.0.55/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:51:17.585439 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:51:17.586071 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:51:17.887469 3917 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod] creating logical port default-tenant_tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod for pod on switch ip-10-0-1-167.ec2.internal I0326 23:51:17.887636 3917 kube.go:256] Updating pod default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod I0326 23:51:17.909688 3917 pod.go:62] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod] pod update took 22.083208ms I0326 23:51:17.909721 3917 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod] addLogicalPort annotation time took 22.119437ms I0326 23:51:17.911137 3917 pods.go:271] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod] addLogicalPort took 23.686942ms, libovsdb time 918.104µs I0326 23:51:18.253528 3917 cni.go:403] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod cb2332e51f2b562a938ade530ac9b1702680230b9d7a45f4d68a246fc7b8d606 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod cb2332e51f2b562a938ade530ac9b1702680230b9d7a45f4d68a246fc7b8d606 network default NAD default] I0326 23:51:18.263051 3917 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod, hostIfaceName: cb2332e51f2b562, network: default, NAD default, SandboxID: "cb2332e51f2b562a938ade530ac9b1702680230b9d7a45f4d68a246fc7b8d606", PCI device ID: , UID: "943243e7-0fd2-4dd2-abc4-a2dd0660e0a2", MAC: 0a:58:0a:81:00:38, IPs: [10.129.0.56/23] I0326 23:51:18.317212 3917 cni.go:424] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod cb2332e51f2b562a938ade530ac9b1702680230b9d7a45f4d68a246fc7b8d606 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod cb2332e51f2b562a938ade530ac9b1702680230b9d7a45f4d68a246fc7b8d606 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:b1:39:5e:3d:b2\",\"name\":\"cb2332e51f2b562\"},{\"mac\":\"0a:58:0a:81:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/15ddc4ae-9b31-4b8f-9530-a8d7ad21de5d\"}],\"ips\":[{\"address\":\"10.129.0.56/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:51:24.571894 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=12 I0326 23:52:17.367581 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-coverity-availability-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:17.368238 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-coverity-availability-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:17.399463 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-apply-tags-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:17.400112 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-apply-tags-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:17.632848 3917 cni.go:403] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod f4d49d21516ea21d18fb1566c8b84888e96848ed302d5788a03688eda86c3687 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod f4d49d21516ea21d18fb1566c8b84888e96848ed302d5788a03688eda86c3687 network default NAD default] I0326 23:52:17.742713 3917 cni.go:424] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod f4d49d21516ea21d18fb1566c8b84888e96848ed302d5788a03688eda86c3687 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod f4d49d21516ea21d18fb1566c8b84888e96848ed302d5788a03688eda86c3687 network default NAD default], result "{}", err I0326 23:52:17.875359 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:17.875400 3917 pods.go:176] Deleting pod: default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod I0326 23:52:17.876911 3917 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod, ips: 10.129.0.54 I0326 23:52:17.876960 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:18.258263 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-sast-snyk-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:18.258987 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-sast-snyk-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:31.672949 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:31.673637 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:31.737795 3917 cni.go:403] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod cb2332e51f2b562a938ade530ac9b1702680230b9d7a45f4d68a246fc7b8d606 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod cb2332e51f2b562a938ade530ac9b1702680230b9d7a45f4d68a246fc7b8d606 network default NAD default] I0326 23:52:31.841937 3917 cni.go:424] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod cb2332e51f2b562a938ade530ac9b1702680230b9d7a45f4d68a246fc7b8d606 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod cb2332e51f2b562a938ade530ac9b1702680230b9d7a45f4d68a246fc7b8d606 network default NAD default], result "{}", err I0326 23:52:31.954731 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:31.954791 3917 pods.go:176] Deleting pod: default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod I0326 23:52:31.956550 3917 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod, ips: 10.129.0.56 I0326 23:52:31.956584 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:32.493527 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-push-dockerfile-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:32.494232 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-push-dockerfile-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:32.528675 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:32.529373 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:05.614609 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:05.615303 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:08.749184 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-ecosystem-cert-preflight-checks-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:08.749797 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-ecosystem-cert-preflight-checks-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:18.683437 3917 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0326 23:53:44.582895 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=231 I0326 23:53:58.691595 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0326 23:54:03.663906 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0326 23:54:36.576629 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=8 I0326 23:54:39.599458 3917 cni.go:403] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod fa9e548069803f34a9fd316e6f617de372e93151eef55acdb81dbe9fa3dd692e network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod fa9e548069803f34a9fd316e6f617de372e93151eef55acdb81dbe9fa3dd692e network default NAD default] I0326 23:54:39.702330 3917 cni.go:424] [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod fa9e548069803f34a9fd316e6f617de372e93151eef55acdb81dbe9fa3dd692e network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod fa9e548069803f34a9fd316e6f617de372e93151eef55acdb81dbe9fa3dd692e network default NAD default], result "{}", err I0326 23:54:39.786067 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:39.786113 3917 pods.go:176] Deleting pod: default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod I0326 23:54:39.787504 3917 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod, ips: 10.129.0.55 I0326 23:54:39.787535 3917 obj_retry.go:464] Detected object default-tenant/tsf-comp-tzwe-on-push-b5mcb-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:43.007127 3917 base_network_controller_pods.go:487] [default/default-tenant/my-integration-test-trnc-j8fqm-verify-pod] creating logical port default-tenant_my-integration-test-trnc-j8fqm-verify-pod for pod on switch ip-10-0-1-167.ec2.internal I0326 23:54:43.007307 3917 kube.go:256] Updating pod default-tenant/my-integration-test-trnc-j8fqm-verify-pod I0326 23:54:43.042871 3917 pod.go:62] [default-tenant/my-integration-test-trnc-j8fqm-verify-pod] pod update took 35.604092ms I0326 23:54:43.042896 3917 base_network_controller_pods.go:937] [default/default-tenant/my-integration-test-trnc-j8fqm-verify-pod] addLogicalPort annotation time took 35.635764ms I0326 23:54:43.044455 3917 pods.go:271] [default-tenant/my-integration-test-trnc-j8fqm-verify-pod] addLogicalPort took 37.34451ms, libovsdb time 1.131862ms I0326 23:54:43.379468 3917 cni.go:403] [default-tenant/my-integration-test-trnc-j8fqm-verify-pod 11b9c0eb7c50e83ea16572f4b3d6b43ee5b4c4321332217f43fe1a02a15c2783 network default NAD default] ADD starting CNI request [default-tenant/my-integration-test-trnc-j8fqm-verify-pod 11b9c0eb7c50e83ea16572f4b3d6b43ee5b4c4321332217f43fe1a02a15c2783 network default NAD default] I0326 23:54:43.388460 3917 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: my-integration-test-trnc-j8fqm-verify-pod, hostIfaceName: 11b9c0eb7c50e83, network: default, NAD default, SandboxID: "11b9c0eb7c50e83ea16572f4b3d6b43ee5b4c4321332217f43fe1a02a15c2783", PCI device ID: , UID: "43126644-f1d3-4397-b423-dfc05a5dea6e", MAC: 0a:58:0a:81:00:39, IPs: [10.129.0.57/23] I0326 23:54:43.648046 3917 cni.go:424] [default-tenant/my-integration-test-trnc-j8fqm-verify-pod 11b9c0eb7c50e83ea16572f4b3d6b43ee5b4c4321332217f43fe1a02a15c2783 network default NAD default] ADD finished CNI request [default-tenant/my-integration-test-trnc-j8fqm-verify-pod 11b9c0eb7c50e83ea16572f4b3d6b43ee5b4c4321332217f43fe1a02a15c2783 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:e6:e1:19:40:ae\",\"name\":\"11b9c0eb7c50e83\"},{\"mac\":\"0a:58:0a:81:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5d0d1091-618e-4e02-b6ed-fa76c1610371\"}],\"ips\":[{\"address\":\"10.129.0.57/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:54:52.654449 3917 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0326 23:54:54.861046 3917 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-5pvnj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:54.861678 3917 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-5pvnj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:54.885453 3917 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-pftct of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:54.886116 3917 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-pftct of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:55.157567 3917 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-pftct of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:54:55.157615 3917 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-pftct of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:54:55.163825 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-pftct I0326 23:54:55.163863 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-pftct I0326 23:54:55.573998 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0326 23:54:55.762758 3917 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-5pvnj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:54:55.762790 3917 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-5pvnj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:54:55.769092 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-5pvnj I0326 23:54:55.769119 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-5pvnj I0326 23:55:04.797421 3917 cni.go:403] [default-tenant/my-integration-test-trnc-j8fqm-verify-pod 11b9c0eb7c50e83ea16572f4b3d6b43ee5b4c4321332217f43fe1a02a15c2783 network default NAD default] DEL starting CNI request [default-tenant/my-integration-test-trnc-j8fqm-verify-pod 11b9c0eb7c50e83ea16572f4b3d6b43ee5b4c4321332217f43fe1a02a15c2783 network default NAD default] I0326 23:55:04.897536 3917 cni.go:424] [default-tenant/my-integration-test-trnc-j8fqm-verify-pod 11b9c0eb7c50e83ea16572f4b3d6b43ee5b4c4321332217f43fe1a02a15c2783 network default NAD default] DEL finished CNI request [default-tenant/my-integration-test-trnc-j8fqm-verify-pod 11b9c0eb7c50e83ea16572f4b3d6b43ee5b4c4321332217f43fe1a02a15c2783 network default NAD default], result "{}", err I0326 23:55:05.006547 3917 obj_retry.go:464] Detected object default-tenant/my-integration-test-trnc-j8fqm-verify-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:55:05.006585 3917 pods.go:176] Deleting pod: default-tenant/my-integration-test-trnc-j8fqm-verify-pod I0326 23:55:05.008058 3917 pods.go:236] Attempting to release IPs for pod: default-tenant/my-integration-test-trnc-j8fqm-verify-pod, ips: 10.129.0.57 I0326 23:55:05.008089 3917 obj_retry.go:464] Detected object default-tenant/my-integration-test-trnc-j8fqm-verify-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:55:34.749252 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-verify-access-to-resources-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:55:34.749931 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-verify-access-to-resources-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:55:35.814042 3917 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-n7dtm-collect-data-pod] creating logical port default-tenant-managed_managed-n7dtm-collect-data-pod for pod on switch ip-10-0-1-167.ec2.internal I0326 23:55:35.814233 3917 kube.go:256] Updating pod default-tenant-managed/managed-n7dtm-collect-data-pod I0326 23:55:35.835317 3917 pod.go:62] [default-tenant-managed/managed-n7dtm-collect-data-pod] pod update took 21.130552ms I0326 23:55:35.835352 3917 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-n7dtm-collect-data-pod] addLogicalPort annotation time took 21.173773ms I0326 23:55:35.836827 3917 pods.go:271] [default-tenant-managed/managed-n7dtm-collect-data-pod] addLogicalPort took 22.806032ms, libovsdb time 976.051µs I0326 23:55:36.180688 3917 cni.go:403] [default-tenant-managed/managed-n7dtm-collect-data-pod 674babb5f651eb9dfef3fefb11d013a1f8c57590c1692ca6ac8b81e8f1327963 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-n7dtm-collect-data-pod 674babb5f651eb9dfef3fefb11d013a1f8c57590c1692ca6ac8b81e8f1327963 network default NAD default] I0326 23:55:36.190159 3917 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-n7dtm-collect-data-pod, hostIfaceName: 674babb5f651eb9, network: default, NAD default, SandboxID: "674babb5f651eb9dfef3fefb11d013a1f8c57590c1692ca6ac8b81e8f1327963", PCI device ID: , UID: "79c04316-8070-47b2-8365-612b6fe45851", MAC: 0a:58:0a:81:00:3a, IPs: [10.129.0.58/23] I0326 23:55:36.241553 3917 cni.go:424] [default-tenant-managed/managed-n7dtm-collect-data-pod 674babb5f651eb9dfef3fefb11d013a1f8c57590c1692ca6ac8b81e8f1327963 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-n7dtm-collect-data-pod 674babb5f651eb9dfef3fefb11d013a1f8c57590c1692ca6ac8b81e8f1327963 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:20:89:c0:fd:f8\",\"name\":\"674babb5f651eb9\"},{\"mac\":\"0a:58:0a:81:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6c410e1f-a035-4bd6-ad89-62a27abe2ce4\"}],\"ips\":[{\"address\":\"10.129.0.58/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:56:09.264520 3917 cni.go:403] [default-tenant-managed/managed-n7dtm-collect-data-pod 674babb5f651eb9dfef3fefb11d013a1f8c57590c1692ca6ac8b81e8f1327963 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-n7dtm-collect-data-pod 674babb5f651eb9dfef3fefb11d013a1f8c57590c1692ca6ac8b81e8f1327963 network default NAD default] I0326 23:56:09.361883 3917 cni.go:424] [default-tenant-managed/managed-n7dtm-collect-data-pod 674babb5f651eb9dfef3fefb11d013a1f8c57590c1692ca6ac8b81e8f1327963 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-n7dtm-collect-data-pod 674babb5f651eb9dfef3fefb11d013a1f8c57590c1692ca6ac8b81e8f1327963 network default NAD default], result "{}", err I0326 23:56:09.449065 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-collect-data-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:09.449125 3917 pods.go:176] Deleting pod: default-tenant-managed/managed-n7dtm-collect-data-pod I0326 23:56:09.450618 3917 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-n7dtm-collect-data-pod, ips: 10.129.0.58 I0326 23:56:09.450653 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-collect-data-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:12.996835 3917 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-n7dtm-reduce-snapshot-pod] creating logical port default-tenant-managed_managed-n7dtm-reduce-snapshot-pod for pod on switch ip-10-0-1-167.ec2.internal I0326 23:56:12.996985 3917 kube.go:256] Updating pod default-tenant-managed/managed-n7dtm-reduce-snapshot-pod I0326 23:56:13.021816 3917 pod.go:62] [default-tenant-managed/managed-n7dtm-reduce-snapshot-pod] pod update took 24.874428ms I0326 23:56:13.021846 3917 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-n7dtm-reduce-snapshot-pod] addLogicalPort annotation time took 24.910672ms I0326 23:56:13.023345 3917 pods.go:271] [default-tenant-managed/managed-n7dtm-reduce-snapshot-pod] addLogicalPort took 26.530058ms, libovsdb time 939.283µs I0326 23:56:13.377632 3917 cni.go:403] [default-tenant-managed/managed-n7dtm-reduce-snapshot-pod 5fa0dc7503fd7cad6757416eb4853e94f1f0fef74ea145b89c5b87271a8f8ba5 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-n7dtm-reduce-snapshot-pod 5fa0dc7503fd7cad6757416eb4853e94f1f0fef74ea145b89c5b87271a8f8ba5 network default NAD default] I0326 23:56:13.386418 3917 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-n7dtm-reduce-snapshot-pod, hostIfaceName: 5fa0dc7503fd7ca, network: default, NAD default, SandboxID: "5fa0dc7503fd7cad6757416eb4853e94f1f0fef74ea145b89c5b87271a8f8ba5", PCI device ID: , UID: "538cbd87-b789-4ded-98fb-e621eed5cfbe", MAC: 0a:58:0a:81:00:3b, IPs: [10.129.0.59/23] I0326 23:56:13.645212 3917 cni.go:424] [default-tenant-managed/managed-n7dtm-reduce-snapshot-pod 5fa0dc7503fd7cad6757416eb4853e94f1f0fef74ea145b89c5b87271a8f8ba5 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-n7dtm-reduce-snapshot-pod 5fa0dc7503fd7cad6757416eb4853e94f1f0fef74ea145b89c5b87271a8f8ba5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:0e:4f:ac:0d:87\",\"name\":\"5fa0dc7503fd7ca\"},{\"mac\":\"0a:58:0a:81:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b5d0b40f-203a-4ea3-b287-4c0d04353dd8\"}],\"ips\":[{\"address\":\"10.129.0.59/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:56:20.043191 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-collect-registry-token-secret-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:20.043792 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-collect-registry-token-secret-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:20.073573 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-collect-task-params-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:20.074200 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-collect-task-params-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:31.690068 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0326 23:56:33.122823 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-check-data-keys-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:33.123394 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-check-data-keys-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:36.694620 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=11 I0326 23:56:40.508798 3917 cni.go:403] [default-tenant-managed/managed-n7dtm-reduce-snapshot-pod 5fa0dc7503fd7cad6757416eb4853e94f1f0fef74ea145b89c5b87271a8f8ba5 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-n7dtm-reduce-snapshot-pod 5fa0dc7503fd7cad6757416eb4853e94f1f0fef74ea145b89c5b87271a8f8ba5 network default NAD default] I0326 23:56:40.597493 3917 cni.go:424] [default-tenant-managed/managed-n7dtm-reduce-snapshot-pod 5fa0dc7503fd7cad6757416eb4853e94f1f0fef74ea145b89c5b87271a8f8ba5 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-n7dtm-reduce-snapshot-pod 5fa0dc7503fd7cad6757416eb4853e94f1f0fef74ea145b89c5b87271a8f8ba5 network default NAD default], result "{}", err I0326 23:56:40.676244 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-reduce-snapshot-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:40.676279 3917 pods.go:176] Deleting pod: default-tenant-managed/managed-n7dtm-reduce-snapshot-pod I0326 23:56:40.677701 3917 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-n7dtm-reduce-snapshot-pod, ips: 10.129.0.59 I0326 23:56:40.677730 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-reduce-snapshot-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:42.515659 3917 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-n7dtm-apply-mapping-pod] creating logical port default-tenant-managed_managed-n7dtm-apply-mapping-pod for pod on switch ip-10-0-1-167.ec2.internal I0326 23:56:42.515847 3917 kube.go:256] Updating pod default-tenant-managed/managed-n7dtm-apply-mapping-pod I0326 23:56:42.549873 3917 pod.go:62] [default-tenant-managed/managed-n7dtm-apply-mapping-pod] pod update took 34.081244ms I0326 23:56:42.549906 3917 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-n7dtm-apply-mapping-pod] addLogicalPort annotation time took 34.119206ms I0326 23:56:42.551534 3917 pods.go:271] [default-tenant-managed/managed-n7dtm-apply-mapping-pod] addLogicalPort took 35.890431ms, libovsdb time 1.060026ms I0326 23:56:42.885222 3917 cni.go:403] [default-tenant-managed/managed-n7dtm-apply-mapping-pod a2d9d1210f334bc2e900e14de6d88b5b501ef62a8bfdc7b1012b933a42d1f36a network default NAD default] ADD starting CNI request [default-tenant-managed/managed-n7dtm-apply-mapping-pod a2d9d1210f334bc2e900e14de6d88b5b501ef62a8bfdc7b1012b933a42d1f36a network default NAD default] I0326 23:56:42.894575 3917 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-n7dtm-apply-mapping-pod, hostIfaceName: a2d9d1210f334bc, network: default, NAD default, SandboxID: "a2d9d1210f334bc2e900e14de6d88b5b501ef62a8bfdc7b1012b933a42d1f36a", PCI device ID: , UID: "b56213f6-c99a-4658-ac88-c8b321551af4", MAC: 0a:58:0a:81:00:3c, IPs: [10.129.0.60/23] I0326 23:56:42.943070 3917 cni.go:424] [default-tenant-managed/managed-n7dtm-apply-mapping-pod a2d9d1210f334bc2e900e14de6d88b5b501ef62a8bfdc7b1012b933a42d1f36a network default NAD default] ADD finished CNI request [default-tenant-managed/managed-n7dtm-apply-mapping-pod a2d9d1210f334bc2e900e14de6d88b5b501ef62a8bfdc7b1012b933a42d1f36a network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:6c:dc:8e:6e:8b\",\"name\":\"a2d9d1210f334bc\"},{\"mac\":\"0a:58:0a:81:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e820f1b5-e98c-4a50-83f2-8f402c1b4e65\"}],\"ips\":[{\"address\":\"10.129.0.60/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:56:45.676191 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0326 23:57:02.653390 3917 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0326 23:57:05.720068 3917 cni.go:403] [default-tenant-managed/managed-n7dtm-apply-mapping-pod a2d9d1210f334bc2e900e14de6d88b5b501ef62a8bfdc7b1012b933a42d1f36a network default NAD default] DEL starting CNI request [default-tenant-managed/managed-n7dtm-apply-mapping-pod a2d9d1210f334bc2e900e14de6d88b5b501ef62a8bfdc7b1012b933a42d1f36a network default NAD default] I0326 23:57:05.814549 3917 cni.go:424] [default-tenant-managed/managed-n7dtm-apply-mapping-pod a2d9d1210f334bc2e900e14de6d88b5b501ef62a8bfdc7b1012b933a42d1f36a network default NAD default] DEL finished CNI request [default-tenant-managed/managed-n7dtm-apply-mapping-pod a2d9d1210f334bc2e900e14de6d88b5b501ef62a8bfdc7b1012b933a42d1f36a network default NAD default], result "{}", err I0326 23:57:05.892508 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-apply-mapping-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:05.892551 3917 pods.go:176] Deleting pod: default-tenant-managed/managed-n7dtm-apply-mapping-pod I0326 23:57:05.894239 3917 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-n7dtm-apply-mapping-pod, ips: 10.129.0.60 I0326 23:57:05.894277 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-apply-mapping-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:07.100657 3917 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-n7dtm-filter-already-released-images-pod] creating logical port default-tenant-managed_managed-n7dtm-filter-already-released-images-pod for pod on switch ip-10-0-1-167.ec2.internal I0326 23:57:07.100810 3917 kube.go:256] Updating pod default-tenant-managed/managed-n7dtm-filter-already-released-images-pod I0326 23:57:07.123446 3917 pod.go:62] [default-tenant-managed/managed-n7dtm-filter-already-released-images-pod] pod update took 22.67223ms I0326 23:57:07.123479 3917 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-n7dtm-filter-already-released-images-pod] addLogicalPort annotation time took 22.710417ms I0326 23:57:07.125313 3917 pods.go:271] [default-tenant-managed/managed-n7dtm-filter-already-released-images-pod] addLogicalPort took 24.677714ms, libovsdb time 1.066394ms I0326 23:57:07.473110 3917 cni.go:403] [default-tenant-managed/managed-n7dtm-filter-already-released-images-pod 173ff5d88c99fc486537f8061ad46d8b24e4340baa13c9bb8b97f3bdfd7a9d96 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-n7dtm-filter-already-released-images-pod 173ff5d88c99fc486537f8061ad46d8b24e4340baa13c9bb8b97f3bdfd7a9d96 network default NAD default] I0326 23:57:07.482237 3917 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-n7dtm-filter-already-released-images-pod, hostIfaceName: 173ff5d88c99fc4, network: default, NAD default, SandboxID: "173ff5d88c99fc486537f8061ad46d8b24e4340baa13c9bb8b97f3bdfd7a9d96", PCI device ID: , UID: "7477931a-021e-44fe-aef9-164692c4c24f", MAC: 0a:58:0a:81:00:3d, IPs: [10.129.0.61/23] I0326 23:57:07.535387 3917 cni.go:424] [default-tenant-managed/managed-n7dtm-filter-already-released-images-pod 173ff5d88c99fc486537f8061ad46d8b24e4340baa13c9bb8b97f3bdfd7a9d96 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-n7dtm-filter-already-released-images-pod 173ff5d88c99fc486537f8061ad46d8b24e4340baa13c9bb8b97f3bdfd7a9d96 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:1d:4f:af:d9:40\",\"name\":\"173ff5d88c99fc4\"},{\"mac\":\"0a:58:0a:81:00:3d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cd67d9b0-2503-45fb-bce9-815037d2cbcd\"}],\"ips\":[{\"address\":\"10.129.0.61/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:57:10.668254 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0326 23:57:26.661510 3917 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0326 23:57:26.896000 3917 cni.go:403] [default-tenant-managed/managed-n7dtm-filter-already-released-images-pod 173ff5d88c99fc486537f8061ad46d8b24e4340baa13c9bb8b97f3bdfd7a9d96 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-n7dtm-filter-already-released-images-pod 173ff5d88c99fc486537f8061ad46d8b24e4340baa13c9bb8b97f3bdfd7a9d96 network default NAD default] I0326 23:57:26.988506 3917 cni.go:424] [default-tenant-managed/managed-n7dtm-filter-already-released-images-pod 173ff5d88c99fc486537f8061ad46d8b24e4340baa13c9bb8b97f3bdfd7a9d96 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-n7dtm-filter-already-released-images-pod 173ff5d88c99fc486537f8061ad46d8b24e4340baa13c9bb8b97f3bdfd7a9d96 network default NAD default], result "{}", err I0326 23:57:27.061173 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-filter-already-released-images-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:27.061211 3917 pods.go:176] Deleting pod: default-tenant-managed/managed-n7dtm-filter-already-released-images-pod I0326 23:57:27.062986 3917 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-n7dtm-filter-already-released-images-pod, ips: 10.129.0.61 I0326 23:57:27.063025 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-filter-already-released-images-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:59.920244 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-verify-conforma-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:59.920866 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-verify-conforma-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:01.657563 3917 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-n7dtm-push-snapshot-pod] creating logical port default-tenant-managed_managed-n7dtm-push-snapshot-pod for pod on switch ip-10-0-1-167.ec2.internal I0326 23:58:01.657701 3917 kube.go:256] Updating pod default-tenant-managed/managed-n7dtm-push-snapshot-pod I0326 23:58:01.683865 3917 pod.go:62] [default-tenant-managed/managed-n7dtm-push-snapshot-pod] pod update took 26.201069ms I0326 23:58:01.683890 3917 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-n7dtm-push-snapshot-pod] addLogicalPort annotation time took 26.230896ms I0326 23:58:01.685253 3917 pods.go:271] [default-tenant-managed/managed-n7dtm-push-snapshot-pod] addLogicalPort took 27.712183ms, libovsdb time 975.272µs I0326 23:58:02.028227 3917 cni.go:403] [default-tenant-managed/managed-n7dtm-push-snapshot-pod 3365eb0a8340e312effd175cf0762caa35fa991135e04a4253689311d8f04d14 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-n7dtm-push-snapshot-pod 3365eb0a8340e312effd175cf0762caa35fa991135e04a4253689311d8f04d14 network default NAD default] I0326 23:58:02.036910 3917 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-n7dtm-push-snapshot-pod, hostIfaceName: 3365eb0a8340e31, network: default, NAD default, SandboxID: "3365eb0a8340e312effd175cf0762caa35fa991135e04a4253689311d8f04d14", PCI device ID: , UID: "4161680b-1eeb-43c3-a305-bde7b0c4a27d", MAC: 0a:58:0a:81:00:3e, IPs: [10.129.0.62/23] I0326 23:58:02.090457 3917 cni.go:424] [default-tenant-managed/managed-n7dtm-push-snapshot-pod 3365eb0a8340e312effd175cf0762caa35fa991135e04a4253689311d8f04d14 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-n7dtm-push-snapshot-pod 3365eb0a8340e312effd175cf0762caa35fa991135e04a4253689311d8f04d14 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:8b:9f:37:9f:5f\",\"name\":\"3365eb0a8340e31\"},{\"mac\":\"0a:58:0a:81:00:3e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/76c8c42d-0deb-4498-aa34-6eaeaddc27c4\"}],\"ips\":[{\"address\":\"10.129.0.62/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:58:26.358973 3917 cni.go:403] [default-tenant-managed/managed-n7dtm-push-snapshot-pod 3365eb0a8340e312effd175cf0762caa35fa991135e04a4253689311d8f04d14 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-n7dtm-push-snapshot-pod 3365eb0a8340e312effd175cf0762caa35fa991135e04a4253689311d8f04d14 network default NAD default] I0326 23:58:26.448180 3917 cni.go:424] [default-tenant-managed/managed-n7dtm-push-snapshot-pod 3365eb0a8340e312effd175cf0762caa35fa991135e04a4253689311d8f04d14 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-n7dtm-push-snapshot-pod 3365eb0a8340e312effd175cf0762caa35fa991135e04a4253689311d8f04d14 network default NAD default], result "{}", err I0326 23:58:26.523175 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-push-snapshot-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:26.523213 3917 pods.go:176] Deleting pod: default-tenant-managed/managed-n7dtm-push-snapshot-pod I0326 23:58:26.524542 3917 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-n7dtm-push-snapshot-pod, ips: 10.129.0.62 I0326 23:58:26.524570 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-push-snapshot-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:33.401922 3917 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-87af7b4a69-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:33.402538 3917 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-87af7b4a69-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:33.576322 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=43 I0326 23:58:33.969701 3917 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-87af7b4a69-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:58:33.969733 3917 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-87af7b4a69-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:58:33.975278 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod default-tenant-managed/affinity-assistant-87af7b4a69-0 I0326 23:58:33.975306 3917 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod default-tenant-managed/affinity-assistant-87af7b4a69-0 I0326 23:58:34.123843 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-update-cr-status-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:34.124406 3917 obj_retry.go:464] Detected object default-tenant-managed/managed-n7dtm-update-cr-status-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:38.583960 3917 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=10