+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-126.ec2.internal ++ [[ -n ip-10-0-1-126.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-126.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0420 11:38:08.168687703 - disable conntrack on geneve port' I0420 11:38:08.168687703 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0420 11:38:08.230734135 - starting ovnkube-node' I0420 11:38:08.230734135 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-126.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-126.ec2.internal --init-node ip-10-0-1-126.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-126.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0420 11:38:08.281510 3689 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0420 11:38:08.281563 3689 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-2de2d7076e.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0420 11:38:08.283383 3689 kube.go:431] Waiting for certificate I0420 11:38:08.283419 3689 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0420 11:38:08.283447 3689 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0420 11:38:08.284117 3689 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0420 11:38:08.314697 3689 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0420 11:38:08.314714 3689 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0420 11:38:08.317279 3689 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0420 11:38:08.319958 3689 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-rp5jn" I0420 11:38:08.328466 3689 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-rp5jn" I0420 11:38:08.328548 3689 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0420 11:38:09.283943 3689 kube.go:438] Certificate found I0420 11:38:09.284465 3689 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0420 11:38:09.285783 3689 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0420 11:38:09.334035 3689 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-21 11:33:08 +0000 UTC" deadline="2026-04-21 05:00:59.057937739 +0000 UTC" I0420 11:38:09.334061 3689 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="17h22m49.723884163s" I0420 11:38:09.353912 3689 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0420 11:38:09.370440 3689 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0420 11:38:09.370521 3689 factory.go:561] Starting watch factory I0420 11:38:09.370634 3689 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.370654 3689 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.370638 3689 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.370685 3689 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.370830 3689 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.370853 3689 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.370939 3689 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0420 11:38:09.371017 3689 server.go:101] MetricServer registers OVS metrics I0420 11:38:09.371567 3689 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.371597 3689 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.371738 3689 server.go:105] MetricServer registers OVN DB metrics I0420 11:38:09.372110 3689 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.372128 3689 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.372406 3689 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.372572 3689 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.376847 3689 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0420 11:38:09.376873 3689 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0420 11:38:09.377754 3689 controller_manager.go:370] Starting the ovnkube controller I0420 11:38:09.377767 3689 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-126.ec2.internal I0420 11:38:09.377810 3689 controller_manager.go:395] NBDB zone sync took: 34.886µs I0420 11:38:09.377821 3689 factory.go:561] Starting watch factory I0420 11:38:09.391210 3689 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0420 11:38:09.391242 3689 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0420 11:38:09.392542 3689 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.392881 3689 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.395217 3689 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0420 11:38:09.395295 3689 server.go:109] MetricServer registers OVN Controller metrics I0420 11:38:09.396281 3689 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.396333 3689 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.396998 3689 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.402305 3689 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0420 11:38:09.403278 3689 server.go:113] MetricServer registers OVN Northd metrics I0420 11:38:09.407548 3689 metrics.go:519] OVN Metrics Server starts to run ... I0420 11:38:09.476431 3689 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0420 11:38:09.476455 3689 factory.go:1977] *v1.Pod informer cache synced successfully I0420 11:38:09.476460 3689 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0420 11:38:09.476464 3689 factory.go:1977] *v1.Namespace informer cache synced successfully I0420 11:38:09.476469 3689 factory.go:1977] *v1.Node informer cache synced successfully I0420 11:38:09.476473 3689 factory.go:1977] *v1.Service informer cache synced successfully I0420 11:38:09.476999 3689 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0420 11:38:09.477022 3689 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0420 11:38:09.477138 3689 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0420 11:38:09.477157 3689 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0420 11:38:09.479612 3689 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0420 11:38:09.479629 3689 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0420 11:38:09.481597 3689 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0420 11:38:09.481610 3689 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0420 11:38:09.481669 3689 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.481688 3689 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.483287 3689 factory.go:1977] *v1.Pod informer cache synced successfully I0420 11:38:09.483298 3689 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0420 11:38:09.483301 3689 factory.go:1977] *v1.Namespace informer cache synced successfully I0420 11:38:09.483305 3689 factory.go:1977] *v1.Node informer cache synced successfully I0420 11:38:09.483308 3689 factory.go:1977] *v1.Service informer cache synced successfully I0420 11:38:09.483311 3689 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0420 11:38:09.483532 3689 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.487168 3689 factory.go:1977] *v1.EgressIP informer cache synced successfully I0420 11:38:09.487226 3689 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.487238 3689 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.488539 3689 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0420 11:38:09.488548 3689 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0420 11:38:09.489223 3689 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.493027 3689 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0420 11:38:09.493084 3689 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.493094 3689 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.493680 3689 factory.go:1977] *v1.EgressIP informer cache synced successfully I0420 11:38:09.495002 3689 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.498619 3689 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0420 11:38:09.498674 3689 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.498690 3689 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.498707 3689 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0420 11:38:09.500581 3689 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.504227 3689 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0420 11:38:09.504240 3689 factory.go:1977] *v1.EgressService informer cache synced successfully I0420 11:38:09.504292 3689 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.504312 3689 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.506483 3689 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.510081 3689 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0420 11:38:09.510175 3689 factory.go:1977] *v1.EgressService informer cache synced successfully I0420 11:38:09.510270 3689 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0420 11:38:09.510287 3689 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0420 11:38:09.512135 3689 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0420 11:38:09.515766 3689 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0420 11:38:09.515832 3689 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.515844 3689 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.515872 3689 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.515891 3689 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.515947 3689 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0420 11:38:09.517805 3689 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.518008 3689 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0420 11:38:09.521729 3689 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0420 11:38:09.521804 3689 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0420 11:38:09.521815 3689 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0420 11:38:09.521825 3689 factory.go:680] Watch Factory start up complete, took: 151.307911ms I0420 11:38:09.521892 3689 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0420 11:38:09.527415 3689 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0420 11:38:09.527425 3689 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0420 11:38:09.527432 3689 factory.go:680] Watch Factory start up complete, took: 149.610776ms I0420 11:38:09.527438 3689 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-1-126.ec2.internal" zone I0420 11:38:09.567346 3689 default_node_network_controller.go:710] Initializing the default node network controller I0420 11:38:09.641178 3689 udn_isolation.go:101] Starting UDN host isolation manager I0420 11:38:09.648256 3689 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0420 11:38:09.709528 3689 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0420 11:38:09.709611 3689 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0420 11:38:09.709625 3689 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0420 11:38:09.740232 3689 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0420 11:38:09.740291 3689 default_node_network_controller.go:809] Node ip-10-0-1-126.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0420 11:38:09.745598 3689 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.126"] k8s.ovn.org/zone-name:ip-10-0-1-126.ec2.internal] on node ip-10-0-1-126.ec2.internal I0420 11:38:09.768859 3689 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0420 11:38:09.775166 3689 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0420 11:38:09.775284 3689 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0420 11:38:09.775426 3689 gateway_init.go:263] Preparing Gateway I0420 11:38:09.775437 3689 gateway_shared_intf.go:1687] Creating new gateway I0420 11:38:09.777694 3689 controller_manager.go:429] Waiting for node in zone sync took: 250.245843ms I0420 11:38:09.796580 3689 controller_manager.go:611] Switching to transit router for layer2 networks I0420 11:38:09.796593 3689 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-126.ec2.internal I0420 11:38:09.816287 3689 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0420 11:38:09.823109 3689 controller_manager.go:290] SCTP support detected in OVN I0420 11:38:09.824494 3689 services_controller.go:69] Creating services controller for network=default I0420 11:38:09.824578 3689 udn_enabled_svc.go:80] Waiting for service informer to sync I0420 11:38:09.824633 3689 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0420 11:38:09.824670 3689 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0420 11:38:09.824683 3689 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0420 11:38:09.824692 3689 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0420 11:38:09.824916 3689 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0420 11:38:09.824974 3689 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0420 11:38:09.825041 3689 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 83.233µs I0420 11:38:09.825064 3689 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 80.748µs I0420 11:38:09.825070 3689 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 96.378µs I0420 11:38:09.825078 3689 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0420 11:38:09.825087 3689 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0420 11:38:09.825098 3689 default_network_controller.go:325] Starting the default network controller I0420 11:38:09.825165 3689 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 14.893µs I0420 11:38:09.825195 3689 udn_enabled_svc.go:95] Performing full resync I0420 11:38:09.825604 3689 udn_enabled_svc.go:99] Waiting for handler to sync I0420 11:38:09.825617 3689 udn_enabled_svc.go:104] Starting worker I0420 11:38:09.835171 3689 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0420 11:38:09.835273 3689 repair.go:33] Syncing exgw routes took 86.909µs I0420 11:38:09.835301 3689 default_network_controller.go:414] Starting all the Watchers... I0420 11:38:09.835439 3689 namespace.go:98] [default] adding namespace I0420 11:38:09.835445 3689 namespace.go:98] [openshift-logging] adding namespace I0420 11:38:09.835440 3689 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0420 11:38:09.835460 3689 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0420 11:38:09.835467 3689 namespace.go:98] [openshift-config-managed] adding namespace I0420 11:38:09.835473 3689 namespace.go:98] [openshift-backplane-cee] adding namespace I0420 11:38:09.835478 3689 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0420 11:38:09.835485 3689 namespace.go:98] [openshift-service-ca-operator] adding namespace I0420 11:38:09.835492 3689 namespace.go:98] [openshift-node] adding namespace I0420 11:38:09.835499 3689 namespace.go:98] [openshift-backplane] adding namespace I0420 11:38:09.835503 3689 namespace.go:98] [openshift-machine-api] adding namespace I0420 11:38:09.835509 3689 namespace.go:98] [kube-system] adding namespace I0420 11:38:09.835513 3689 namespace.go:98] [openshift-multus] adding namespace I0420 11:38:09.835516 3689 namespace.go:98] [open-cluster-management-2pprst5nusibgq03117kjfn2t3hbbnqb] adding namespace I0420 11:38:09.835534 3689 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0420 11:38:09.836201 3689 namespace.go:102] [default] adding namespace took 748.487µs I0420 11:38:09.836221 3689 namespace.go:98] [openshift-apiserver-operator] adding namespace I0420 11:38:09.836945 3689 namespace.go:102] [openshift-logging] adding namespace took 1.486328ms I0420 11:38:09.837013 3689 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0420 11:38:09.837697 3689 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 670.877µs I0420 11:38:09.837717 3689 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0420 11:38:09.838522 3689 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 3.061378ms I0420 11:38:09.838539 3689 namespace.go:98] [openshift-customer-monitoring] adding namespace I0420 11:38:09.839446 3689 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 3.974182ms I0420 11:38:09.839484 3689 namespace.go:98] [openshift-operators] adding namespace I0420 11:38:09.840200 3689 namespace.go:102] [openshift-config-managed] adding namespace took 4.721485ms I0420 11:38:09.840220 3689 namespace.go:98] [openshift-apiserver] adding namespace I0420 11:38:09.840732 3689 namespace.go:102] [openshift-backplane-cee] adding namespace took 5.251293ms I0420 11:38:09.840748 3689 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0420 11:38:09.841369 3689 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 5.88317ms I0420 11:38:09.841385 3689 namespace.go:98] [openshift-network-diagnostics] adding namespace I0420 11:38:09.841872 3689 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.377396ms I0420 11:38:09.841891 3689 namespace.go:98] [openshift-image-registry] adding namespace I0420 11:38:09.842443 3689 namespace.go:102] [openshift-node] adding namespace took 6.944048ms I0420 11:38:09.842459 3689 namespace.go:98] [openshift-dns-operator] adding namespace I0420 11:38:09.843013 3689 namespace.go:102] [openshift-backplane] adding namespace took 7.50748ms I0420 11:38:09.843032 3689 namespace.go:98] [openshift-etcd] adding namespace I0420 11:38:09.843890 3689 namespace.go:102] [openshift-machine-api] adding namespace took 8.381241ms I0420 11:38:09.843917 3689 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0420 11:38:09.844404 3689 namespace.go:102] [kube-system] adding namespace took 8.889882ms I0420 11:38:09.844421 3689 namespace.go:98] [openshift-cluster-version] adding namespace W0420 11:38:09.844429 3689 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-58qj7: pod openshift-multus/network-metrics-daemon-58qj7: no pod IPs found W0420 11:38:09.844457 3689 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-sgztn: pod openshift-multus/network-metrics-daemon-sgztn: no pod IPs found I0420 11:38:09.844990 3689 namespace.go:102] [openshift-multus] adding namespace took 9.471867ms I0420 11:38:09.845007 3689 namespace.go:98] [openshift-must-gather-operator] adding namespace I0420 11:38:09.845589 3689 namespace.go:102] [open-cluster-management-2pprst5nusibgq03117kjfn2t3hbbnqb] adding namespace took 10.066017ms I0420 11:38:09.845606 3689 namespace.go:98] [openshift-authentication-operator] adding namespace I0420 11:38:09.846109 3689 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 10.556385ms I0420 11:38:09.846127 3689 namespace.go:98] [openshift-kube-apiserver] adding namespace I0420 11:38:09.846688 3689 namespace.go:102] [openshift-apiserver-operator] adding namespace took 10.460126ms I0420 11:38:09.846707 3689 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0420 11:38:09.847273 3689 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 9.55058ms I0420 11:38:09.847286 3689 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0420 11:38:09.847888 3689 namespace.go:102] [openshift-customer-monitoring] adding namespace took 9.340875ms I0420 11:38:09.847920 3689 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0420 11:38:09.848387 3689 namespace.go:102] [openshift-operators] adding namespace took 8.895161ms I0420 11:38:09.848399 3689 namespace.go:98] [openshift-ingress] adding namespace I0420 11:38:09.848940 3689 namespace.go:102] [openshift-apiserver] adding namespace took 8.714314ms I0420 11:38:09.848952 3689 namespace.go:98] [openshift-ingress-operator] adding namespace W0420 11:38:09.849496 3689 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-6sjft: pod openshift-network-diagnostics/network-check-target-6sjft: no pod IPs found W0420 11:38:09.849508 3689 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-96r95: pod openshift-network-diagnostics/network-check-target-96r95: no pod IPs found I0420 11:38:09.849531 3689 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 8.778853ms I0420 11:38:09.849540 3689 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0420 11:38:09.850162 3689 namespace.go:102] [openshift-network-diagnostics] adding namespace took 8.770281ms I0420 11:38:09.850180 3689 namespace.go:98] [openshift-controller-manager] adding namespace I0420 11:38:09.850678 3689 namespace.go:102] [openshift-image-registry] adding namespace took 8.777671ms I0420 11:38:09.850697 3689 namespace.go:98] [openshift-network-node-identity] adding namespace I0420 11:38:09.851144 3689 namespace.go:102] [openshift-dns-operator] adding namespace took 8.679392ms I0420 11:38:09.851156 3689 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0420 11:38:09.851555 3689 namespace.go:102] [openshift-etcd] adding namespace took 8.517483ms I0420 11:38:09.851564 3689 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0420 11:38:09.851919 3689 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.99809ms I0420 11:38:09.851928 3689 namespace.go:98] [openshift-backplane-csm] adding namespace I0420 11:38:09.852393 3689 namespace.go:102] [openshift-cluster-version] adding namespace took 7.966627ms I0420 11:38:09.852408 3689 namespace.go:98] [openshift-host-network] adding namespace I0420 11:38:09.852763 3689 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.74954ms I0420 11:38:09.852776 3689 namespace.go:98] [openshift-insights] adding namespace I0420 11:38:09.853568 3689 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.956461ms I0420 11:38:09.853580 3689 namespace.go:98] [openshift-backplane-tam] adding namespace I0420 11:38:09.854000 3689 namespace.go:102] [openshift-kube-apiserver] adding namespace took 7.867534ms I0420 11:38:09.854012 3689 namespace.go:98] [openshift] adding namespace I0420 11:38:09.854377 3689 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.664355ms I0420 11:38:09.854388 3689 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0420 11:38:09.854758 3689 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.456968ms I0420 11:38:09.854770 3689 namespace.go:98] [openshift-config] adding namespace I0420 11:38:09.855182 3689 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 7.255261ms I0420 11:38:09.855196 3689 namespace.go:98] [openshift-console-user-settings] adding namespace I0420 11:38:09.855584 3689 namespace.go:102] [openshift-ingress] adding namespace took 7.1805ms I0420 11:38:09.855597 3689 namespace.go:98] [kube-node-lease] adding namespace I0420 11:38:09.855980 3689 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.023925ms I0420 11:38:09.855990 3689 namespace.go:98] [dedicated-admin] adding namespace I0420 11:38:09.856345 3689 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.800071ms I0420 11:38:09.856357 3689 namespace.go:98] [openshift-network-operator] adding namespace I0420 11:38:09.856692 3689 namespace.go:102] [openshift-controller-manager] adding namespace took 6.5073ms I0420 11:38:09.856702 3689 namespace.go:98] [openshift-operators-redhat] adding namespace I0420 11:38:09.857109 3689 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.40353ms I0420 11:38:09.857118 3689 namespace.go:98] [openshift-console-operator] adding namespace I0420 11:38:09.857482 3689 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.320545ms I0420 11:38:09.857494 3689 namespace.go:98] [openshift-network-console] adding namespace I0420 11:38:09.857840 3689 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.272049ms I0420 11:38:09.857851 3689 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0420 11:38:09.858236 3689 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.304246ms I0420 11:38:09.858245 3689 namespace.go:98] [openshift-kube-scheduler] adding namespace I0420 11:38:09.858649 3689 namespace.go:102] [openshift-host-network] adding namespace took 6.235673ms I0420 11:38:09.858662 3689 namespace.go:98] [openshift-authentication] adding namespace I0420 11:38:09.859024 3689 namespace.go:102] [openshift-insights] adding namespace took 6.243881ms I0420 11:38:09.859035 3689 namespace.go:98] [openshift-infra] adding namespace I0420 11:38:09.859398 3689 namespace.go:102] [openshift-backplane-tam] adding namespace took 5.812522ms I0420 11:38:09.859409 3689 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0420 11:38:09.859784 3689 namespace.go:102] [openshift] adding namespace took 5.767177ms I0420 11:38:09.859797 3689 namespace.go:98] [openshift-marketplace] adding namespace I0420 11:38:09.860165 3689 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 5.773274ms I0420 11:38:09.860176 3689 namespace.go:98] [openshift-console] adding namespace I0420 11:38:09.860528 3689 namespace.go:102] [openshift-config] adding namespace took 5.753608ms I0420 11:38:09.860539 3689 namespace.go:98] [openshift-backplane-mobb] adding namespace I0420 11:38:09.860933 3689 namespace.go:102] [openshift-console-user-settings] adding namespace took 5.695582ms I0420 11:38:09.860946 3689 namespace.go:98] [kube-public] adding namespace I0420 11:38:09.861305 3689 namespace.go:102] [kube-node-lease] adding namespace took 5.704162ms I0420 11:38:09.861316 3689 namespace.go:98] [openshift-machine-config-operator] adding namespace I0420 11:38:09.861676 3689 namespace.go:102] [dedicated-admin] adding namespace took 5.681491ms I0420 11:38:09.861687 3689 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0420 11:38:09.862114 3689 namespace.go:102] [openshift-network-operator] adding namespace took 5.752272ms I0420 11:38:09.862125 3689 namespace.go:98] [openshift-route-controller-manager] adding namespace I0420 11:38:09.862530 3689 namespace.go:102] [openshift-operators-redhat] adding namespace took 5.824096ms I0420 11:38:09.862542 3689 namespace.go:98] [openshift-dns] adding namespace I0420 11:38:09.862920 3689 namespace.go:102] [openshift-console-operator] adding namespace took 5.798857ms I0420 11:38:09.862930 3689 namespace.go:98] [openshift-monitoring] adding namespace I0420 11:38:09.863384 3689 namespace.go:102] [openshift-network-console] adding namespace took 5.884877ms I0420 11:38:09.863396 3689 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0420 11:38:09.863852 3689 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 5.995211ms I0420 11:38:09.863865 3689 namespace.go:98] [openshift-config-operator] adding namespace I0420 11:38:09.864245 3689 namespace.go:102] [openshift-kube-scheduler] adding namespace took 5.995656ms I0420 11:38:09.864257 3689 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0420 11:38:09.864641 3689 namespace.go:102] [openshift-authentication] adding namespace took 5.973427ms I0420 11:38:09.864656 3689 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0420 11:38:09.865038 3689 namespace.go:102] [openshift-infra] adding namespace took 5.998984ms I0420 11:38:09.865051 3689 namespace.go:98] [openshift-backplane-srep] adding namespace I0420 11:38:09.865408 3689 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 5.995152ms I0420 11:38:09.865418 3689 namespace.go:98] [openshift-backplane-cse] adding namespace I0420 11:38:09.865783 3689 namespace.go:102] [openshift-marketplace] adding namespace took 5.982223ms I0420 11:38:09.865794 3689 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0420 11:38:09.866179 3689 namespace.go:102] [openshift-console] adding namespace took 5.998877ms I0420 11:38:09.866558 3689 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.015519ms I0420 11:38:09.866926 3689 namespace.go:102] [kube-public] adding namespace took 5.974895ms I0420 11:38:09.867514 3689 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.193158ms I0420 11:38:09.867982 3689 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.291737ms I0420 11:38:09.868497 3689 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.367599ms I0420 11:38:09.868987 3689 namespace.go:102] [openshift-dns] adding namespace took 6.436987ms I0420 11:38:09.869401 3689 namespace.go:102] [openshift-monitoring] adding namespace took 6.46636ms I0420 11:38:09.870021 3689 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.618757ms I0420 11:38:09.870570 3689 namespace.go:102] [openshift-config-operator] adding namespace took 6.6999ms I0420 11:38:09.870584 3689 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0420 11:38:09.870988 3689 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.72753ms I0420 11:38:09.870999 3689 namespace.go:98] [openshift-package-operator] adding namespace I0420 11:38:09.871494 3689 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.832292ms I0420 11:38:09.872110 3689 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.052938ms I0420 11:38:09.872125 3689 namespace.go:98] [openshift-ingress-canary] adding namespace I0420 11:38:09.872527 3689 namespace.go:102] [openshift-backplane-cse] adding namespace took 7.102965ms I0420 11:38:09.872935 3689 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.12653ms I0420 11:38:09.873456 3689 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 2.866644ms I0420 11:38:09.873885 3689 namespace.go:102] [openshift-package-operator] adding namespace took 2.881147ms I0420 11:38:09.874510 3689 namespace.go:102] [openshift-ingress-canary] adding namespace took 2.379229ms W0420 11:38:09.874601 3689 master.go:458] Unable to parse local node ip-10-0-1-126.ec2.internal chassis-id annotation. Chassis may be removed during sync W0420 11:38:09.874611 3689 master.go:471] Unable to parse remote node ip-10-0-2-190.ec2.internal chassis-id annotation. Chassis may be removed during sync I0420 11:38:09.874615 3689 master.go:481] Removing stale chassis with ID/Name: a444c116-e33b-448a-baa5-3fbe85655511, hostname: ip-10-0-1-126.ec2.internal I0420 11:38:09.875087 3689 master.go:548] Adding or Updating local node "ip-10-0-1-126.ec2.internal" for network "default" I0420 11:38:09.875179 3689 obj_retry.go:578] Failed to create *v1.Node ip-10-0-2-190.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-190.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-190.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-190.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-190.ec2.internal I0420 11:38:09.878343 3689 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-126.ec2.internal for the network default I0420 11:38:09.878656 3689 zone_ic_handler.go:195] Time taken to create transit switch: 300.273µs I0420 11:38:09.879334 3689 default_network_controller.go:823] Node add failed for ip-10-0-1-126.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-126.ec2.internal, error getting gateway config for node ip-10-0-1-126.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-126.ec2.internal", failed to update chassis to local for local node ip-10-0-1-126.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-126.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-126.ec2.internal] E0420 11:38:09.879384 3689 obj_retry.go:575] Failed to create *v1.Node ip-10-0-1-126.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-126.ec2.internal, error getting gateway config for node ip-10-0-1-126.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-126.ec2.internal", failed to update chassis to local for local node ip-10-0-1-126.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-126.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-126.ec2.internal] I0420 11:38:09.879426 3689 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0420 11:38:09.879447 3689 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0420 11:38:09.879461 3689 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0420 11:38:09.879494 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-126.ec2.internal in network "default" I0420 11:38:09.879507 3689 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-1-126.ec2.internal", UID:"d1411c76-ed7b-4b7e-8263-391105c9c0b0", APIVersion:"v1", ResourceVersion:"8073", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-126.ec2.internal, error getting gateway config for node ip-10-0-1-126.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-126.ec2.internal", failed to update chassis to local for local node ip-10-0-1-126.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-126.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-126.ec2.internal] I0420 11:38:09.879528 3689 node_tracker.go:235] Node ip-10-0-1-126.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-126.ec2.internal" I0420 11:38:09.879541 3689 node_tracker.go:169] Node ip-10-0-1-126.ec2.internal switch + router changed, syncing services in network "default" I0420 11:38:09.879556 3689 services_controller.go:574] Full service sync requested for network=default I0420 11:38:09.879569 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-190.ec2.internal in network "default" I0420 11:38:09.879582 3689 node_tracker.go:235] Node ip-10-0-2-190.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-190.ec2.internal" I0420 11:38:09.879592 3689 node_tracker.go:169] Node ip-10-0-2-190.ec2.internal switch + router changed, syncing services in network "default" I0420 11:38:09.879601 3689 services_controller.go:574] Full service sync requested for network=default I0420 11:38:09.897212 3689 iptables.go:108] Creating table: filter chain: FORWARD I0420 11:38:09.903540 3689 iptables.go:108] Creating table: filter chain: OUTPUT I0420 11:38:09.909630 3689 gateway_shared_intf.go:1816] Gateway Creation Complete I0420 11:38:09.909644 3689 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-126.ec2.internal","mac-address":"0a:ff:c0:b3:8d:ff","ip-addresses":["10.0.1.126/24"],"ip-address":"10.0.1.126/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:a444c116-e33b-448a-baa5-3fbe85655511 k8s.ovn.org/node-encap-ips:["10.0.1.126"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.126/24"} k8s.ovn.org/zone-name:ip-10-0-1-126.ec2.internal] on node ip-10-0-1-126.ec2.internal I0420 11:38:09.926585 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-126.ec2.internal in network "default" I0420 11:38:09.926640 3689 master.go:548] Adding or Updating local node "ip-10-0-1-126.ec2.internal" for network "default" I0420 11:38:09.926656 3689 node_tracker.go:169] Node ip-10-0-1-126.ec2.internal switch + router changed, syncing services in network "default" I0420 11:38:09.926669 3689 services_controller.go:574] Full service sync requested for network=default I0420 11:38:09.930469 3689 default_node_network_controller.go:519] Node connection status = connected W0420 11:38:09.931802 3689 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-58qj7: pod openshift-multus/network-metrics-daemon-58qj7: no pod IPs found W0420 11:38:09.931818 3689 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-6sjft: pod openshift-network-diagnostics/network-check-target-6sjft: no pod IPs found I0420 11:38:09.932824 3689 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-126.ec2.internal for the network default I0420 11:38:09.932995 3689 zone_ic_handler.go:195] Time taken to create transit switch: 158.685µs I0420 11:38:09.980006 3689 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0420 11:38:09.980025 3689 services_controller.go:208] Setting up event handlers for services for network=default I0420 11:38:09.980077 3689 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0420 11:38:09.980112 3689 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0420 11:38:09.980121 3689 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0420 11:38:10.080633 3689 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0420 11:38:10.080650 3689 repair.go:60] Starting repairing loop for services I0420 11:38:10.080744 3689 repair.go:131] Deleted 0 stale service LBs I0420 11:38:10.080762 3689 repair.go:137] Deleted 0 stale Chassis Template Vars I0420 11:38:10.080786 3689 repair.go:62] Finished repairing loop for services: 138.364µs I0420 11:38:10.080816 3689 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0420 11:38:10.080826 3689 services_controller.go:254] Starting workers for network=default I0420 11:38:10.081278 3689 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-6sjft] creating logical port openshift-network-diagnostics_network-check-target-6sjft for pod on switch ip-10-0-1-126.ec2.internal I0420 11:38:10.081356 3689 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-58qj7] creating logical port openshift-multus_network-metrics-daemon-58qj7 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:38:10.081368 3689 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-sgztn, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-sgztn: suppressed error logged: pod openshift-multus/network-metrics-daemon-sgztn: no pod IPs found I0420 11:38:10.081437 3689 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-6sjft I0420 11:38:10.081445 3689 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-58qj7 I0420 11:38:10.102888 3689 pod.go:62] [openshift-network-diagnostics/network-check-target-6sjft] pod update took 21.527915ms I0420 11:38:10.102919 3689 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-6sjft] addLogicalPort annotation time took 21.563863ms I0420 11:38:10.103682 3689 pods.go:271] [openshift-network-diagnostics/network-check-target-6sjft] addLogicalPort took 22.412819ms, libovsdb time 575.169µs I0420 11:38:10.103719 3689 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-96r95, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-96r95: suppressed error logged: pod openshift-network-diagnostics/network-check-target-96r95: no pod IPs found I0420 11:38:10.108594 3689 pod.go:62] [openshift-multus/network-metrics-daemon-58qj7] pod update took 27.175886ms I0420 11:38:10.108616 3689 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-58qj7] addLogicalPort annotation time took 27.197907ms I0420 11:38:10.109358 3689 pods.go:271] [openshift-multus/network-metrics-daemon-58qj7] addLogicalPort took 28.011839ms, libovsdb time 570.172µs I0420 11:38:10.109748 3689 admin_network_policy_controller.go:227] Starting controller default-network-controller I0420 11:38:10.109771 3689 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0420 11:38:10.109782 3689 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0420 11:38:10.109786 3689 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0420 11:38:10.109989 3689 repair.go:29] Repairing admin network policies took 195.915µs I0420 11:38:10.110092 3689 repair.go:92] Repairing baseline admin network policies took 91.415µs I0420 11:38:10.110600 3689 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0420 11:38:10.110617 3689 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0420 11:38:10.112812 3689 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0420 11:38:10.112832 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0420 11:38:10.112841 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0420 11:38:10.114592 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0420 11:38:10.114612 3689 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0420 11:38:10.114619 3689 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0420 11:38:10.116085 3689 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0420 11:38:10.116102 3689 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0420 11:38:10.116123 3689 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0420 11:38:10.117245 3689 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0420 11:38:10.117264 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0420 11:38:10.117272 3689 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0420 11:38:10.119091 3689 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0420 11:38:10.119109 3689 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0420 11:38:10.119117 3689 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0420 11:38:10.120703 3689 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0420 11:38:10.120720 3689 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0420 11:38:10.120740 3689 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0420 11:38:10.122190 3689 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0420 11:38:10.122212 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0420 11:38:10.122220 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0420 11:38:10.122721 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0420 11:38:10.122742 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0420 11:38:10.122751 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0420 11:38:10.124352 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0420 11:38:10.124373 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0420 11:38:10.124380 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0420 11:38:10.124866 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0420 11:38:10.124884 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0420 11:38:10.124891 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0420 11:38:10.126278 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0420 11:38:10.126299 3689 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0420 11:38:10.126307 3689 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0420 11:38:10.126838 3689 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0420 11:38:10.126855 3689 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0420 11:38:10.126863 3689 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0420 11:38:10.127823 3689 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0420 11:38:10.127841 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0420 11:38:10.127848 3689 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0420 11:38:10.128388 3689 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0420 11:38:10.128405 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0420 11:38:10.128874 3689 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0420 11:38:10.128887 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0420 11:38:10.129778 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0420 11:38:10.129795 3689 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0420 11:38:10.129803 3689 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0420 11:38:10.131037 3689 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0420 11:38:10.131054 3689 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0420 11:38:10.131063 3689 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0420 11:38:10.132268 3689 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0420 11:38:10.132290 3689 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0420 11:38:10.132303 3689 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0420 11:38:10.133423 3689 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0420 11:38:10.133440 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0420 11:38:10.133460 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0420 11:38:10.134269 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0420 11:38:10.134300 3689 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0420 11:38:10.134308 3689 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0420 11:38:10.134644 3689 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0420 11:38:10.134660 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0420 11:38:10.134666 3689 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0420 11:38:10.134959 3689 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0420 11:38:10.134975 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0420 11:38:10.134982 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0420 11:38:10.135577 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0420 11:38:10.135599 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0420 11:38:10.135609 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0420 11:38:10.136201 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0420 11:38:10.136218 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0420 11:38:10.136226 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0420 11:38:10.136758 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0420 11:38:10.136777 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0420 11:38:10.136784 3689 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0420 11:38:10.137357 3689 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0420 11:38:10.137374 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0420 11:38:10.137381 3689 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0420 11:38:10.137882 3689 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0420 11:38:10.137911 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0420 11:38:10.137919 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0420 11:38:10.138211 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0420 11:38:10.138228 3689 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0420 11:38:10.138237 3689 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0420 11:38:10.139812 3689 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0420 11:38:10.139830 3689 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0420 11:38:10.139843 3689 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0420 11:38:10.141389 3689 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0420 11:38:10.145716 3689 egressfirewall.go:374] Starting EgressFirewall controller I0420 11:38:10.145749 3689 controller.go:133] Adding controller egress-firewall-controller event handlers I0420 11:38:10.145768 3689 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0420 11:38:10.145785 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0420 11:38:10.145799 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0420 11:38:10.145802 3689 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0420 11:38:10.145808 3689 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0420 11:38:10.146073 3689 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0420 11:38:10.146087 3689 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0420 11:38:10.146094 3689 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0420 11:38:10.146102 3689 egressqos.go:193] Setting up event handlers for EgressQoS I0420 11:38:10.146175 3689 egressfirewall.go:723] Syncing node "ip-10-0-1-126.ec2.internal" for egress firewall I0420 11:38:10.146188 3689 egressfirewall.go:723] Syncing node "ip-10-0-2-190.ec2.internal" for egress firewall I0420 11:38:10.146223 3689 egressqos.go:245] Starting EgressQoS Controller I0420 11:38:10.146234 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0420 11:38:10.146240 3689 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0420 11:38:10.146245 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0420 11:38:10.146250 3689 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0420 11:38:10.146258 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0420 11:38:10.146265 3689 shared_informer.go:356] "Caches are synced" controller="egressqos" I0420 11:38:10.146271 3689 egressqos.go:259] Repairing EgressQoSes I0420 11:38:10.146275 3689 egressqos.go:399] Starting repairing loop for egressqos I0420 11:38:10.146348 3689 egressqos.go:401] Finished repairing loop for egressqos: 72.617µs I0420 11:38:10.146370 3689 egressservice_zone.go:132] Setting up event handlers for Egress Services I0420 11:38:10.146389 3689 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-126.ec2.internal I0420 11:38:10.146437 3689 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-126.ec2.internal : 48.442µs I0420 11:38:10.146493 3689 egressservice_zone.go:210] Starting Egress Services Controller I0420 11:38:10.146506 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0420 11:38:10.146514 3689 shared_informer.go:356] "Caches are synced" controller="egressservices" I0420 11:38:10.146525 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0420 11:38:10.146532 3689 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0420 11:38:10.146540 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0420 11:38:10.146547 3689 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0420 11:38:10.146556 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0420 11:38:10.146562 3689 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0420 11:38:10.146570 3689 egressservice_zone.go:228] Repairing Egress Services I0420 11:38:10.146871 3689 master_controller.go:88] Starting Admin Policy Based Route Controller I0420 11:38:10.146879 3689 external_controller.go:278] Starting Admin Policy Based Route Controller I0420 11:38:10.146912 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-126.ec2.internal I0420 11:38:10.146926 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-126.ec2.internal: 27.795µs I0420 11:38:10.146936 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-190.ec2.internal I0420 11:38:10.146940 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-190.ec2.internal: 4.53µs I0420 11:38:10.147077 3689 default_network_controller.go:585] Completing all the Watchers took 311.767007ms I0420 11:38:10.147088 3689 default_network_controller.go:589] Starting unidling controllers I0420 11:38:10.147098 3689 unidle.go:46] Registering OVN SB ControllerEvent handler I0420 11:38:10.147109 3689 unidle.go:63] Populating Initial ContollerEvent events I0420 11:38:10.147122 3689 unidle.go:79] Setting up event handlers for services I0420 11:38:10.147283 3689 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0420 11:38:10.430733 3689 default_node_network_controller.go:519] Node connection status = connected I0420 11:38:10.931665 3689 default_node_network_controller.go:519] Node connection status = connected I0420 11:38:11.430780 3689 default_node_network_controller.go:519] Node connection status = connected I0420 11:38:11.895784 3689 egressfirewall.go:723] Syncing node "ip-10-0-0-85.ec2.internal" for egress firewall I0420 11:38:11.895799 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-85.ec2.internal in network "default" I0420 11:38:11.895814 3689 node_tracker.go:220] Node ip-10-0-0-85.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0420 11:38:11.895828 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-85.ec2.internal I0420 11:38:11.895841 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-85.ec2.internal: 16.173µs E0420 11:38:11.895881 3689 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-85.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0420 11:38:11.895914 3689 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-85.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-85.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-85.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-85.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-85.ec2.internal, error parsing annotation for node ip-10-0-0-85.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0420 11:38:11.909096 3689 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-85.ec2.internal annotation: unexpected end of JSON input I0420 11:38:11.909144 3689 default_network_controller.go:1025] Node "ip-10-0-0-85.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0420 11:38:11.909186 3689 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-85.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0420 11:38:11.923528 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-85.ec2.internal I0420 11:38:11.923547 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-85.ec2.internal: 23.288µs I0420 11:38:11.923550 3689 default_network_controller.go:1025] Node "ip-10-0-0-85.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0420 11:38:11.923558 3689 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-85.ec2.internal annotation: unexpected end of JSON input E0420 11:38:11.923588 3689 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-85.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0420 11:38:11.923594 3689 egressfirewall.go:723] Syncing node "ip-10-0-0-85.ec2.internal" for egress firewall I0420 11:38:11.930687 3689 default_node_network_controller.go:519] Node connection status = connected I0420 11:38:11.980381 3689 bridgeconfig.go:537] Gateway is ready I0420 11:38:11.980397 3689 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0420 11:38:11.992428 3689 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.126. Updating OVN encap IP. I0420 11:38:11.992675 3689 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0420 11:38:11.997790 3689 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.126 - it is already configured I0420 11:38:11.997803 3689 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.126/24:{} 172.20.0.1/32:{}]. Updating annotations. I0420 11:38:11.998012 3689 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.126/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-126.ec2.internal","mac-address":"0a:ff:c0:b3:8d:ff","ip-addresses":["10.0.1.126/24"],"ip-address":"10.0.1.126/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:a444c116-e33b-448a-baa5-3fbe85655511 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.126/24"}] on node ip-10-0-1-126.ec2.internal E0420 11:38:12.014184 3689 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-85.ec2.internal annotation: unexpected end of JSON input E0420 11:38:12.014205 3689 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-85.ec2.internal annotation: unexpected end of JSON input I0420 11:38:12.014227 3689 default_network_controller.go:1025] Node "ip-10-0-0-85.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0420 11:38:12.014278 3689 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-85.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-85.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-85.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-85.ec2.internal" I0420 11:38:12.019197 3689 egressfirewall.go:723] Syncing node "ip-10-0-1-126.ec2.internal" for egress firewall I0420 11:38:12.019249 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-126.ec2.internal I0420 11:38:12.019263 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-126.ec2.internal: 15.019µs I0420 11:38:12.019278 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-126.ec2.internal in network "default" I0420 11:38:12.019335 3689 master.go:548] Adding or Updating local node "ip-10-0-1-126.ec2.internal" for network "default" I0420 11:38:12.019361 3689 node_tracker.go:169] Node ip-10-0-1-126.ec2.internal switch + router changed, syncing services in network "default" I0420 11:38:12.019385 3689 services_controller.go:574] Full service sync requested for network=default I0420 11:38:12.019480 3689 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.1.126 I0420 11:38:12.019581 3689 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-126.ec2.internal I0420 11:38:12.023573 3689 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-126.ec2.internal. Updating no re-route policies I0420 11:38:12.023648 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-spg24 I0420 11:38:12.023660 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-hlcth I0420 11:38:12.023667 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-96r95 I0420 11:38:12.023677 3689 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-126.ec2.internal I0420 11:38:12.023680 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-96r95 I0420 11:38:12.023682 3689 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-126.ec2.internal I0420 11:38:12.023684 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-8lx2b I0420 11:38:12.023685 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-sgztn I0420 11:38:12.023690 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-8lx2b I0420 11:38:12.023691 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-twtm7 I0420 11:38:12.023699 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-sgztn I0420 11:38:12.023699 3689 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-126.ec2.internal after 0 failed attempt(s) I0420 11:38:12.023706 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-twtm7 I0420 11:38:12.023709 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-8lx2b after 0 failed attempt(s) I0420 11:38:12.023670 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-hlcth I0420 11:38:12.023715 3689 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-jnvc5 I0420 11:38:12.023723 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-twtm7 after 0 failed attempt(s) I0420 11:38:12.023727 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-hlcth after 0 failed attempt(s) I0420 11:38:12.023727 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-tmb2r I0420 11:38:12.023724 3689 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-jnvc5 I0420 11:38:12.023736 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-tmb2r I0420 11:38:12.023735 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-6sjft I0420 11:38:12.023735 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-126.ec2.internal I0420 11:38:12.023740 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-s45r5 I0420 11:38:12.023742 3689 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-sgztn, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-sgztn: suppressed error logged: pod openshift-multus/network-metrics-daemon-sgztn: no pod IPs found I0420 11:38:12.023751 3689 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-jnvc5 after 0 failed attempt(s) I0420 11:38:12.023752 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-126.ec2.internal I0420 11:38:12.023670 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-58qj7 I0420 11:38:12.023759 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-s45r5 I0420 11:38:12.023765 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-58qj7 I0420 11:38:12.023770 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-126.ec2.internal after 0 failed attempt(s) I0420 11:38:12.023789 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-s45r5 after 0 failed attempt(s) I0420 11:38:12.023791 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-tmb2r after 0 failed attempt(s) I0420 11:38:12.023798 3689 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-58qj7] creating logical port openshift-multus_network-metrics-daemon-58qj7 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:38:12.023662 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-spg24 I0420 11:38:12.023807 3689 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-96r95, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-96r95: suppressed error logged: pod openshift-network-diagnostics/network-check-target-96r95: no pod IPs found I0420 11:38:12.023833 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-spg24 after 0 failed attempt(s) I0420 11:38:12.023648 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-55hzw I0420 11:38:12.023846 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-55hzw I0420 11:38:12.023856 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-55hzw after 0 failed attempt(s) I0420 11:38:12.023745 3689 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-7zq9w I0420 11:38:12.023864 3689 obj_retry.go:398] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-7zq9w I0420 11:38:12.023721 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-7cvhj I0420 11:38:12.023877 3689 obj_retry.go:418] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-7zq9w after 0 failed attempt(s) I0420 11:38:12.023747 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-6sjft I0420 11:38:12.023878 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-7cvhj I0420 11:38:12.023924 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-7cvhj after 0 failed attempt(s) I0420 11:38:12.023928 3689 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-6sjft] creating logical port openshift-network-diagnostics_network-check-target-6sjft for pod on switch ip-10-0-1-126.ec2.internal I0420 11:38:12.024693 3689 pods.go:271] [openshift-multus/network-metrics-daemon-58qj7] addLogicalPort took 902.883µs, libovsdb time 401.751µs I0420 11:38:12.024709 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-58qj7 after 0 failed attempt(s) I0420 11:38:12.024933 3689 pods.go:271] [openshift-network-diagnostics/network-check-target-6sjft] addLogicalPort took 1.014515ms, libovsdb time 227.762µs I0420 11:38:12.024942 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-6sjft after 0 failed attempt(s) I0420 11:38:12.030065 3689 default_network_controller.go:1025] Node "ip-10-0-0-85.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0420 11:38:12.038535 3689 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0420 11:38:12.049662 3689 iptables.go:108] Creating table: mangle chain: OUTPUT I0420 11:38:12.052713 3689 iptables.go:108] Creating table: nat chain: OUTPUT I0420 11:38:12.088467 3689 iptables.go:108] Creating table: nat chain: PREROUTING I0420 11:38:12.140315 3689 egressfirewall.go:723] Syncing node "ip-10-0-0-85.ec2.internal" for egress firewall I0420 11:38:12.140327 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-85.ec2.internal I0420 11:38:12.140337 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-85.ec2.internal: 14.836µs I0420 11:38:12.140380 3689 default_network_controller.go:1025] Node "ip-10-0-0-85.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 11:38:12.146591 3689 iptables.go:108] Creating table: filter chain: FORWARD I0420 11:38:12.153844 3689 default_network_controller.go:1025] Node "ip-10-0-0-85.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 11:38:12.162126 3689 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0420 11:38:12.162220 3689 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0420 11:38:12.162275 3689 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0420 11:38:12.162360 3689 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0420 11:38:12.162412 3689 gateway_init.go:325] Gateway and management port readiness took 2.235972687s I0420 11:38:12.162437 3689 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0420 11:38:12.162478 3689 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0420 11:38:12.162489 3689 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0420 11:38:12.162497 3689 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0420 11:38:12.162552 3689 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0420 11:38:12.162635 3689 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 55.951µs I0420 11:38:12.162859 3689 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0420 11:38:12.162883 3689 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 14.432µs I0420 11:38:12.162891 3689 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 32.007µs I0420 11:38:12.162911 3689 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0420 11:38:12.162923 3689 nad_controller.go:379] [node-nad-controller NAD controller]: started I0420 11:38:12.162934 3689 default_node_network_controller.go:899] Starting the default node network controller I0420 11:38:12.162940 3689 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0420 11:38:12.163009 3689 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 12.98µs I0420 11:38:12.163100 3689 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0420 11:38:12.163108 3689 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0420 11:38:12.163129 3689 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b89110 0xc0002d1b90 0xc0002d1c00 0xc0002d1c70 0xc0002d1ce0 0xc0002d1d50 0xc0002d1dc0 0xc000bb93b0 0xc0002d1e30 0xc0002d1ea0 0xc000aaa410 0xc00656f3b0 0xc0002d1f10 map[0x2ff9080:0xc003197500 0x2ff95c0:0xc003197800 0x2ffa2e0:0xc003197440 0x2ffa580:0xc0031974a0 0x2ffad60:0xc003197560 0x2ffb000:0xc0031975c0 0x2ffb2a0:0xc003197620 0x2ffba80:0xc003197860 0x2ffbd20:0xc0031978c0 0x3026980:0xc00072f140 0x3026d20:0xc00072f1a0 0x3027800:0xc000bf1500 0x3029160:0xc001a9a000 0x30298a0:0xc00072f500 0x302a720:0xc000c08240] 0xc000bac230 0} I0420 11:38:12.163195 3689 gateway.go:151] Starting gateway service sync I0420 11:38:12.163602 3689 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0420 11:38:12.165647 3689 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0420 11:38:12.189853 3689 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0420 11:38:12.190966 3689 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0420 11:38:12.192037 3689 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0420 11:38:12.225638 3689 gateway.go:168] Gateway service sync done. Time taken: 62.429384ms I0420 11:38:12.226124 3689 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30311 I0420 11:38:12.226387 3689 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b89110 0xc0002d1b90 0xc0002d1c00 0xc0002d1c70 0xc0002d1ce0 0xc0002d1d50 0xc0002d1dc0 0xc000bb93b0 0xc0002d1e30 0xc0002d1ea0 0xc000aaa410 0xc00656f3b0 0xc0002d1f10 map[0x2ff9080:0xc003197500 0x2ff95c0:0xc003197800 0x2ffa2e0:0xc003197440 0x2ffa580:0xc0031974a0 0x2ffad60:0xc003197560 0x2ffb000:0xc0031975c0 0x2ffb2a0:0xc003197620 0x2ffba80:0xc003197860 0x2ffbd20:0xc0031978c0 0x3026980:0xc00072f140 0x3026d20:0xc00072f1a0 0x3027800:0xc000bf1500 0x3029160:0xc001a9a000 0x30298a0:0xc00072f500 0x302a720:0xc000c08240] 0xc000bac230 0} I0420 11:38:12.226795 3689 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b89110 0xc0002d1b90 0xc0002d1c00 0xc0002d1c70 0xc0002d1ce0 0xc0002d1d50 0xc0002d1dc0 0xc000bb93b0 0xc0002d1e30 0xc0002d1ea0 0xc000aaa410 0xc00656f3b0 0xc0002d1f10 map[0x2ff9080:0xc003197500 0x2ff95c0:0xc003197800 0x2ffa2e0:0xc003197440 0x2ffa580:0xc0031974a0 0x2ffad60:0xc003197560 0x2ffb000:0xc0031975c0 0x2ffb2a0:0xc003197620 0x2ffba80:0xc003197860 0x2ffbd20:0xc0031978c0 0x3026980:0xc00072f140 0x3026d20:0xc00072f1a0 0x3027800:0xc000bf1500 0x3029160:0xc001a9a000 0x30298a0:0xc00072f500 0x302a720:0xc000c08240] 0xc000bac230 0} I0420 11:38:12.226829 3689 gateway.go:532] Reconciling gateway with updates I0420 11:38:12.232164 3689 gateway.go:339] Spawning Conntrack Rule Check Thread I0420 11:38:12.232234 3689 default_node_network_controller.go:944] Gateway and management port readiness took 69.119437ms I0420 11:38:12.232256 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0420 11:38:12.232273 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0420 11:38:12.232289 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0420 11:38:12.232297 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0420 11:38:12.232299 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0420 11:38:12.232302 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 11:38:12.232316 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0420 11:38:12.232318 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0420 11:38:12.232324 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 11:38:12.232344 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0420 11:38:12.232364 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0420 11:38:12.232404 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0420 11:38:12.232422 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0420 11:38:12.232420 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0420 11:38:12.232429 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0420 11:38:12.232280 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0420 11:38:12.232438 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0420 11:38:12.232441 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0420 11:38:12.232444 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0420 11:38:12.232454 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0420 11:38:12.232463 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0420 11:38:12.232480 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0420 11:38:12.232490 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0420 11:38:12.232504 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0420 11:38:12.232506 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0420 11:38:12.232513 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0420 11:38:12.232277 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0420 11:38:12.232515 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0420 11:38:12.232530 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0420 11:38:12.232541 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0420 11:38:12.232541 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0420 11:38:12.232530 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0420 11:38:12.232552 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0420 11:38:12.232559 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0420 11:38:12.232561 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0420 11:38:12.232562 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0420 11:38:12.232572 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0420 11:38:12.232577 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0420 11:38:12.232577 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0420 11:38:12.232581 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0420 11:38:12.232532 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0420 11:38:12.232593 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0420 11:38:12.232601 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0420 11:38:12.232602 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0420 11:38:12.232605 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0420 11:38:12.232567 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0420 11:38:12.232552 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0420 11:38:12.232622 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0420 11:38:12.232608 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0420 11:38:12.232634 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0420 11:38:12.232625 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0420 11:38:12.232639 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0420 11:38:12.232643 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0420 11:38:12.232643 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0420 11:38:12.232645 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0420 11:38:12.232649 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0420 11:38:12.232652 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0420 11:38:12.232653 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0420 11:38:12.232655 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0420 11:38:12.232658 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0420 11:38:12.232662 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0420 11:38:12.232664 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0420 11:38:12.232668 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0420 11:38:12.232673 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0420 11:38:12.232679 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0420 11:38:12.232684 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0420 11:38:12.232684 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0420 11:38:12.232683 3689 port_claim.go:71] Svc openshift-ingress/router-default: port 30155 is already open, no action needed I0420 11:38:12.232693 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0420 11:38:12.232693 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0420 11:38:12.232691 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0420 11:38:12.232703 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0420 11:38:12.232712 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0420 11:38:12.232634 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0420 11:38:12.232672 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0420 11:38:12.232719 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0420 11:38:12.232735 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0420 11:38:12.232648 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0420 11:38:12.232547 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0420 11:38:12.232615 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0420 11:38:12.232661 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0420 11:38:12.232799 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0420 11:38:12.232809 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0420 11:38:12.232646 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0420 11:38:12.232683 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0420 11:38:12.232862 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0420 11:38:12.232662 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0420 11:38:12.232670 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0420 11:38:12.232933 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0420 11:38:12.232684 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0420 11:38:12.233021 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0420 11:38:12.232696 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0420 11:38:12.232703 3689 port_claim.go:71] Svc openshift-ingress/router-default: port 32487 is already open, no action needed I0420 11:38:12.232639 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0420 11:38:12.233164 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0420 11:38:12.233169 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0420 11:38:12.232657 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0420 11:38:12.233177 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0420 11:38:12.232763 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0420 11:38:12.233628 3689 node_ip_handler_linux.go:160] Node IP manager is running I0420 11:38:12.253151 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0420 11:38:12.253210 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0420 11:38:12.253231 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0420 11:38:12.253239 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0420 11:38:12.253245 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0420 11:38:12.253250 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0420 11:38:12.253257 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0420 11:38:12.253262 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0420 11:38:12.253294 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0420 11:38:12.253309 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0420 11:38:12.253315 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0420 11:38:12.253320 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0420 11:38:12.253325 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0420 11:38:12.253330 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0420 11:38:12.253335 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0420 11:38:12.253340 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0420 11:38:12.253345 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0420 11:38:12.253349 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0420 11:38:12.253354 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0420 11:38:12.253369 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0420 11:38:12.253378 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) W0420 11:38:12.269651 3689 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0420 11:38:12.269843 3689 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0420 11:38:12.270021 3689 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0420 11:38:12.299462 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0420 11:38:12.299492 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0420 11:38:12.299498 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0420 11:38:12.299505 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0420 11:38:12.299515 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0420 11:38:12.299521 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0420 11:38:12.299543 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0420 11:38:12.299549 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0420 11:38:12.299556 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0420 11:38:12.299561 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0420 11:38:12.299572 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0420 11:38:12.338895 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0420 11:38:12.338914 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0420 11:38:12.338938 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0420 11:38:12.413069 3689 default_network_controller.go:1025] Node "ip-10-0-0-85.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 11:38:14.236559 3689 default_node_network_controller.go:1380] Starting node controller node sync W0420 11:38:14.236582 3689 egressip_healthcheck.go:76] Health checking using insecure connection I0420 11:38:14.236625 3689 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0420 11:38:14.248816 3689 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 12.234943ms I0420 11:38:14.248989 3689 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0420 11:38:14.249277 3689 egressservice_node.go:93] Setting up event handlers for Egress Services I0420 11:38:14.249442 3689 egressservice_node.go:183] Starting Egress Services Controller I0420 11:38:14.249459 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0420 11:38:14.249471 3689 shared_informer.go:356] "Caches are synced" controller="egressservices" I0420 11:38:14.249481 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0420 11:38:14.249488 3689 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0420 11:38:14.249497 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0420 11:38:14.249504 3689 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0420 11:38:14.249510 3689 egressservice_node.go:197] Repairing Egress Services I0420 11:38:14.260788 3689 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0420 11:38:14.260802 3689 external_controller.go:278] Starting Admin Policy Based Route Controller I0420 11:38:14.261012 3689 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0420 11:38:14.261025 3689 link_network_manager.go:119] Link manager is running I0420 11:38:14.261039 3689 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0420 11:38:14.261229 3689 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0420 11:38:14.261368 3689 vrf_manager.go:128] VRF manager is running I0420 11:38:14.261463 3689 node_controller_manager.go:437] Removing flows to drop GARP I0420 11:38:14.261476 3689 gateway.go:532] Reconciling gateway with updates I0420 11:38:14.290959 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0420 11:38:14.290971 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0420 11:38:14.290977 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0420 11:38:14.290981 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0420 11:38:14.290987 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0420 11:38:14.290989 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0420 11:38:14.290991 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0420 11:38:14.290995 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0420 11:38:14.290995 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0420 11:38:14.290997 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0420 11:38:14.291001 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0420 11:38:14.291001 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0420 11:38:14.291009 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0420 11:38:14.291013 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0420 11:38:14.291015 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0420 11:38:14.291023 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0420 11:38:14.291025 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0420 11:38:14.291030 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0420 11:38:14.291033 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0420 11:38:14.290980 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0420 11:38:14.291042 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0420 11:38:14.291045 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0420 11:38:14.290962 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0420 11:38:14.291050 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0420 11:38:14.291053 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0420 11:38:14.291057 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0420 11:38:14.291059 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0420 11:38:14.291052 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 11:38:14.291070 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0420 11:38:14.291075 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0420 11:38:14.291063 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0420 11:38:14.291017 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0420 11:38:14.291081 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0420 11:38:14.291090 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0420 11:38:14.291091 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0420 11:38:14.291093 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0420 11:38:14.291050 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0420 11:38:14.291095 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0420 11:38:14.291108 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0420 11:38:14.291107 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0420 11:38:14.291112 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0420 11:38:14.291117 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0420 11:38:14.291091 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0420 11:38:14.291123 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0420 11:38:14.291123 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0420 11:38:14.291124 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0420 11:38:14.291127 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0420 11:38:14.291123 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0420 11:38:14.291135 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0420 11:38:14.291131 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0420 11:38:14.291138 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0420 11:38:14.291139 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0420 11:38:14.291143 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0420 11:38:14.291142 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0420 11:38:14.291142 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0420 11:38:14.290986 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0420 11:38:14.291153 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0420 11:38:14.291152 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0420 11:38:14.291158 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0420 11:38:14.291163 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0420 11:38:14.291166 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0420 11:38:14.291167 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0420 11:38:14.291181 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0420 11:38:14.291189 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0420 11:38:14.290973 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0420 11:38:14.291218 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0420 11:38:14.291232 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0420 11:38:14.291096 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0420 11:38:14.291250 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0420 11:38:14.291046 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0420 11:38:14.291157 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0420 11:38:14.291311 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0420 11:38:14.291063 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0420 11:38:14.291321 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0420 11:38:14.291118 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0420 11:38:14.291109 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0420 11:38:14.291054 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0420 11:38:14.291098 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0420 11:38:14.291432 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0420 11:38:14.291102 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0420 11:38:14.291456 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0420 11:38:14.291108 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0420 11:38:14.291107 3689 port_claim.go:71] Svc openshift-ingress/router-default: port 30155 is already open, no action needed I0420 11:38:14.291489 3689 port_claim.go:71] Svc openshift-ingress/router-default: port 32487 is already open, no action needed I0420 11:38:14.291081 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 11:38:14.291036 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0420 11:38:14.291574 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0420 11:38:14.291111 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0420 11:38:14.291119 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0420 11:38:14.291600 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0420 11:38:14.291604 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0420 11:38:14.291126 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0420 11:38:14.291129 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0420 11:38:14.291124 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0420 11:38:14.291633 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0420 11:38:14.291131 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0420 11:38:14.291127 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0420 11:38:14.291690 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0420 11:38:14.291134 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0420 11:38:14.291133 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0420 11:38:14.291717 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0420 11:38:14.291079 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0420 11:38:14.291138 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0420 11:38:14.291146 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0420 11:38:14.291143 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0420 11:38:14.291749 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0420 11:38:14.291150 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0420 11:38:14.291770 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0420 11:38:14.291150 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0420 11:38:14.291157 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0420 11:38:14.333879 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0420 11:38:14.333915 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0420 11:38:14.378477 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0420 11:38:14.378498 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0420 11:38:14.378505 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0420 11:38:14.378512 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0420 11:38:14.378518 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0420 11:38:14.378524 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0420 11:38:14.378529 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0420 11:38:14.378535 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0420 11:38:14.378539 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0420 11:38:14.378546 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0420 11:38:14.378551 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0420 11:38:14.378556 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0420 11:38:14.378561 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0420 11:38:14.417851 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0420 11:38:14.417861 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0420 11:38:14.417878 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0420 11:38:14.417885 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0420 11:38:14.417892 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0420 11:38:14.417917 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0420 11:38:14.417934 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0420 11:38:14.417950 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0420 11:38:14.417956 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0420 11:38:14.491164 3689 obj_retry.go:498] Detected object kube-system/ecr-credential-refresh-initial-7zq9w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:38:14.491254 3689 obj_retry.go:498] Detected object kube-system/ecr-credential-refresh-initial-7zq9w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:38:15.483933 3689 obj_retry.go:492] Detected object kube-system/ecr-credential-refresh-initial-7zq9w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:38:15.483966 3689 obj_retry.go:492] Detected object kube-system/ecr-credential-refresh-initial-7zq9w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:38:16.093729 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-126.ec2.internal I0420 11:38:16.093764 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-126.ec2.internal: 41.248µs I0420 11:38:16.117941 3689 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-5c66559854-bgrcm] creating logical port openshift-image-registry_image-registry-5c66559854-bgrcm for pod on switch ip-10-0-1-126.ec2.internal I0420 11:38:16.118129 3689 kube.go:256] Updating pod openshift-image-registry/image-registry-5c66559854-bgrcm I0420 11:38:16.143391 3689 pod.go:62] [openshift-image-registry/image-registry-5c66559854-bgrcm] pod update took 25.29221ms I0420 11:38:16.143410 3689 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-5c66559854-bgrcm] addLogicalPort annotation time took 25.316227ms I0420 11:38:16.144976 3689 pods.go:271] [openshift-image-registry/image-registry-5c66559854-bgrcm] addLogicalPort took 27.056538ms, libovsdb time 1.291794ms I0420 11:38:16.186668 3689 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-l5dvc] creating logical port openshift-dns_dns-default-l5dvc for pod on switch ip-10-0-1-126.ec2.internal I0420 11:38:16.186754 3689 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-nn4xr] creating logical port openshift-ingress-canary_ingress-canary-nn4xr for pod on switch ip-10-0-1-126.ec2.internal I0420 11:38:16.186819 3689 kube.go:256] Updating pod openshift-dns/dns-default-l5dvc I0420 11:38:16.186829 3689 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-t4zx6] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-t4zx6 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:38:16.186880 3689 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-nn4xr I0420 11:38:16.186949 3689 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-t4zx6 I0420 11:38:16.216327 3689 pod.go:62] [openshift-ingress-canary/ingress-canary-nn4xr] pod update took 29.471966ms I0420 11:38:16.216346 3689 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-nn4xr] addLogicalPort annotation time took 29.497244ms I0420 11:38:16.217528 3689 pods.go:271] [openshift-ingress-canary/ingress-canary-nn4xr] addLogicalPort took 30.786886ms, libovsdb time 912.634µs I0420 11:38:16.230261 3689 pod.go:62] [openshift-dns/dns-default-l5dvc] pod update took 43.479366ms I0420 11:38:16.230281 3689 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-l5dvc] addLogicalPort annotation time took 43.50373ms I0420 11:38:16.231434 3689 pods.go:271] [openshift-dns/dns-default-l5dvc] addLogicalPort took 44.806533ms, libovsdb time 886.96µs I0420 11:38:16.231853 3689 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-t4zx6] pod update took 44.927479ms I0420 11:38:16.231871 3689 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-t4zx6] addLogicalPort annotation time took 44.949204ms I0420 11:38:16.232867 3689 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-t4zx6] addLogicalPort took 46.047617ms, libovsdb time 789.638µs I0420 11:38:17.288690 3689 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-t4zx6 830335e33e080f97c6fb97ad3c001fa562339a7d73e15bd480162793554e4ec0 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-t4zx6 830335e33e080f97c6fb97ad3c001fa562339a7d73e15bd480162793554e4ec0 network default NAD default NAD key ] I0420 11:38:17.288752 3689 cni.go:452] [openshift-ingress-canary/ingress-canary-nn4xr 92b086eeb4e7a42b7849ca3b993e1510b175d0a253fb56ad1c6bfecd3ced5e94 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-nn4xr 92b086eeb4e7a42b7849ca3b993e1510b175d0a253fb56ad1c6bfecd3ced5e94 network default NAD default NAD key ] I0420 11:38:17.327522 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-nn4xr, hostIfaceName: 92b086eeb4e7a42, network: default, NAD default, SandboxID: "92b086eeb4e7a42b7849ca3b993e1510b175d0a253fb56ad1c6bfecd3ced5e94", PCI device ID: , UID: "f831ec27-9613-47bc-aaf7-3a7742f596dc", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0420 11:38:17.328048 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-t4zx6, hostIfaceName: 830335e33e080f9, network: default, NAD default, SandboxID: "830335e33e080f97c6fb97ad3c001fa562339a7d73e15bd480162793554e4ec0", PCI device ID: , UID: "d57895d5-dc34-4a68-af95-aa69b37b6ab3", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0420 11:38:17.380974 3689 cni.go:473] [openshift-ingress-canary/ingress-canary-nn4xr 92b086eeb4e7a42b7849ca3b993e1510b175d0a253fb56ad1c6bfecd3ced5e94 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-nn4xr 92b086eeb4e7a42b7849ca3b993e1510b175d0a253fb56ad1c6bfecd3ced5e94 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:fe:2c:f9:f5:5e\",\"name\":\"92b086eeb4e7a42\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/21511992-2331-44d8-8ad6-ab40300fa200\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:38:17.381007 3689 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-t4zx6 830335e33e080f97c6fb97ad3c001fa562339a7d73e15bd480162793554e4ec0 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-t4zx6 830335e33e080f97c6fb97ad3c001fa562339a7d73e15bd480162793554e4ec0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:a5:f3:b6:32:61\",\"name\":\"830335e33e080f9\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/29f37267-4bad-421d-a36a-f459347c6146\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:38:18.055185 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-190.ec2.internal I0420 11:38:18.055212 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-190.ec2.internal: 35.311µs I0420 11:38:18.055243 3689 egressfirewall.go:723] Syncing node "ip-10-0-2-190.ec2.internal" for egress firewall I0420 11:38:18.055244 3689 default_network_controller.go:1025] Node "ip-10-0-2-190.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 11:38:19.444136 3689 cni.go:452] [openshift-network-diagnostics/network-check-target-6sjft 125d5cabff41f94de7e0256e9e48b1738bf61513c7f375c21673346cef49a11b network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-6sjft 125d5cabff41f94de7e0256e9e48b1738bf61513c7f375c21673346cef49a11b network default NAD default NAD key ] I0420 11:38:19.454772 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-6sjft, hostIfaceName: 125d5cabff41f94, network: default, NAD default, SandboxID: "125d5cabff41f94de7e0256e9e48b1738bf61513c7f375c21673346cef49a11b", PCI device ID: , UID: "48b0e8a8-83ac-4fb4-a327-8adc2d5f8f09", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0420 11:38:19.501957 3689 cni.go:473] [openshift-network-diagnostics/network-check-target-6sjft 125d5cabff41f94de7e0256e9e48b1738bf61513c7f375c21673346cef49a11b network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-6sjft 125d5cabff41f94de7e0256e9e48b1738bf61513c7f375c21673346cef49a11b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:38:93:4d:e5:e1\",\"name\":\"125d5cabff41f94\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f81919e4-6901-4080-8de4-106382b2f49f\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:38:22.011430 3689 default_network_controller.go:1025] Node "ip-10-0-0-85.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 11:38:23.178447 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-190.ec2.internal in network "default" I0420 11:38:23.178750 3689 node_tracker.go:235] Node ip-10-0-2-190.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-190.ec2.internal" I0420 11:38:23.178770 3689 node_tracker.go:169] Node ip-10-0-2-190.ec2.internal switch + router changed, syncing services in network "default" I0420 11:38:23.178467 3689 default_network_controller.go:1025] Node "ip-10-0-2-190.ec2.internal" in remote zone "ip-10-0-2-190.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 11:38:23.178785 3689 services_controller.go:574] Full service sync requested for network=default I0420 11:38:23.217639 3689 default_network_controller.go:1025] Node "ip-10-0-2-190.ec2.internal" in remote zone "ip-10-0-2-190.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 11:38:23.300216 3689 default_network_controller.go:1025] Node "ip-10-0-2-190.ec2.internal" in remote zone "ip-10-0-2-190.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 11:38:23.300226 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-190.ec2.internal in network "default" I0420 11:38:23.300284 3689 node_tracker.go:169] Node ip-10-0-2-190.ec2.internal switch + router changed, syncing services in network "default" I0420 11:38:23.300299 3689 services_controller.go:574] Full service sync requested for network=default I0420 11:38:23.301264 3689 zone_ic_handler.go:195] Time taken to create transit switch: 273.544µs I0420 11:38:23.301279 3689 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-190.ec2.internal for the network default I0420 11:38:23.302256 3689 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-190.ec2.internal" on network "default" took: 1.277331ms W0420 11:38:24.491722 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0420 11:38:25.354334 3689 egressfirewall.go:723] Syncing node "ip-10-0-2-190.ec2.internal" for egress firewall I0420 11:38:25.354400 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-190.ec2.internal I0420 11:38:25.354414 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-190.ec2.internal: 18.526µs I0420 11:38:25.354423 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-190.ec2.internal in network "default" I0420 11:38:25.354425 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-190.ec2.internal", IP: 10.0.2.190 to PMTUD blocking rules I0420 11:38:25.354441 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-190.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0420 11:38:25.354471 3689 node_tracker.go:169] Node ip-10-0-2-190.ec2.internal switch + router changed, syncing services in network "default" I0420 11:38:25.354487 3689 services_controller.go:574] Full service sync requested for network=default W0420 11:38:25.354684 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0420 11:38:25.361131 3689 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-190.ec2.internal. Updating no re-route policies I0420 11:38:25.370832 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-190.ec2.internal", IP: 10.0.2.190 to PMTUD blocking rules I0420 11:38:25.370848 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-190.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0420 11:38:25.493336 3689 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 11:38:28.866673 3689 default_network_controller.go:1025] Node "ip-10-0-0-85.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 11:38:29.414024 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-190.ec2.internal I0420 11:38:29.414052 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-190.ec2.internal: 33.916µs I0420 11:38:29.819403 3689 default_network_controller.go:1025] Node "ip-10-0-0-85.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 11:38:29.819416 3689 egressfirewall.go:723] Syncing node "ip-10-0-0-85.ec2.internal" for egress firewall I0420 11:38:29.819417 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-85.ec2.internal I0420 11:38:29.819440 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-85.ec2.internal: 28.217µs W0420 11:38:35.209973 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0420 11:38:37.419931 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-85.ec2.internal in network "default" I0420 11:38:37.419961 3689 node_tracker.go:235] Node ip-10-0-0-85.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-85.ec2.internal" I0420 11:38:37.419962 3689 default_network_controller.go:1025] Node "ip-10-0-0-85.ec2.internal" in remote zone "ip-10-0-0-85.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 11:38:37.419977 3689 node_tracker.go:169] Node ip-10-0-0-85.ec2.internal switch + router changed, syncing services in network "default" I0420 11:38:37.419987 3689 services_controller.go:574] Full service sync requested for network=default W0420 11:38:37.420181 3689 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0420 11:38:37.420602 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0420 11:38:37.590164 3689 default_network_controller.go:1025] Node "ip-10-0-0-85.ec2.internal" in remote zone "ip-10-0-0-85.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 11:38:37.590179 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-85.ec2.internal in network "default" I0420 11:38:37.590228 3689 node_tracker.go:169] Node ip-10-0-0-85.ec2.internal switch + router changed, syncing services in network "default" I0420 11:38:37.590241 3689 services_controller.go:574] Full service sync requested for network=default W0420 11:38:37.590579 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0420 11:38:37.590684 3689 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 11:38:37.591134 3689 zone_ic_handler.go:195] Time taken to create transit switch: 277.495µs I0420 11:38:37.591149 3689 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-85.ec2.internal for the network default I0420 11:38:37.592230 3689 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-85.ec2.internal" on network "default" took: 1.384316ms I0420 11:38:38.147689 3689 egressfirewall.go:723] Syncing node "ip-10-0-0-85.ec2.internal" for egress firewall I0420 11:38:38.147705 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-85.ec2.internal in network "default" I0420 11:38:38.147735 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-85.ec2.internal", IP: 10.0.0.85 to PMTUD blocking rules I0420 11:38:38.147745 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-85.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0420 11:38:38.147756 3689 node_tracker.go:169] Node ip-10-0-0-85.ec2.internal switch + router changed, syncing services in network "default" I0420 11:38:38.147769 3689 services_controller.go:574] Full service sync requested for network=default I0420 11:38:38.147708 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-85.ec2.internal I0420 11:38:38.147794 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-85.ec2.internal: 88.241µs W0420 11:38:38.148220 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0420 11:38:38.148305 3689 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 11:38:38.153835 3689 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-85.ec2.internal. Updating no re-route policies I0420 11:38:38.166891 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-85.ec2.internal", IP: 10.0.0.85 to PMTUD blocking rules I0420 11:38:38.166921 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-85.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0420 11:38:38.534812 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 11:38:42.074259 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-85.ec2.internal I0420 11:38:42.074290 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-85.ec2.internal: 37.735µs W0420 11:38:50.019513 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0420 11:38:55.495274 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0420 11:39:03.144832 3689 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl] creating logical port openshift-deployment-validation-operator_ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl for pod on switch ip-10-0-1-126.ec2.internal I0420 11:39:03.144984 3689 kube.go:256] Updating pod openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl I0420 11:39:03.161635 3689 pod.go:62] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl] pod update took 16.678771ms I0420 11:39:03.161655 3689 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl] addLogicalPort annotation time took 16.703215ms I0420 11:39:03.162838 3689 pods.go:271] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl] addLogicalPort took 18.057883ms, libovsdb time 878.749µs I0420 11:39:03.502206 3689 cni.go:452] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl f15a1cbee75aabba091a435a298127478aedcb4164706e5d0598d229bc311229 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl f15a1cbee75aabba091a435a298127478aedcb4164706e5d0598d229bc311229 network default NAD default NAD key ] I0420 11:39:03.511580 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl, hostIfaceName: f15a1cbee75aabb, network: default, NAD default, SandboxID: "f15a1cbee75aabba091a435a298127478aedcb4164706e5d0598d229bc311229", PCI device ID: , UID: "d013594c-5dc4-4613-9814-850f6a2a6e3c", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0420 11:39:03.556704 3689 cni.go:473] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl f15a1cbee75aabba091a435a298127478aedcb4164706e5d0598d229bc311229 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl f15a1cbee75aabba091a435a298127478aedcb4164706e5d0598d229bc311229 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:bc:e5:7f:68:63\",\"name\":\"f15a1cbee75aabb\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0b8330d8-8346-4e3c-9e5e-c91a712506b6\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:39:21.639485 3689 cni.go:452] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl f15a1cbee75aabba091a435a298127478aedcb4164706e5d0598d229bc311229 network default NAD default NAD key ] DEL starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl f15a1cbee75aabba091a435a298127478aedcb4164706e5d0598d229bc311229 network default NAD default NAD key ] I0420 11:39:21.704199 3689 cni.go:473] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl f15a1cbee75aabba091a435a298127478aedcb4164706e5d0598d229bc311229 network default NAD default NAD key default] DEL finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl f15a1cbee75aabba091a435a298127478aedcb4164706e5d0598d229bc311229 network default NAD default NAD key default], result "{}", err I0420 11:39:21.746205 3689 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:39:21.746236 3689 pods.go:176] Deleting pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl I0420 11:39:21.747476 3689 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl, ips: 10.128.0.9 I0420 11:39:21.747502 3689 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:39:21.747518 3689 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:39:21.747526 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0420 11:39:21.747766 3689 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:39:21.747781 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0420 11:39:22.777497 3689 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:39:22.777523 3689 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:39:22.777538 3689 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:39:22.777563 3689 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390xbtjl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:39:23.243174 3689 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0420 11:39:23.256631 3689 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0420 11:40:27.028512 3689 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-6nmq4] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67f4d568c7-6nmq4 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:40:27.028653 3689 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-6nmq4 I0420 11:40:27.035223 3689 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-lcpgd] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-lcpgd for pod on switch ip-10-0-1-126.ec2.internal I0420 11:40:27.035346 3689 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-lcpgd I0420 11:40:27.043382 3689 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-2vmwd] creating logical port openshift-insights_insights-operator-6dbb575d6c-2vmwd for pod on switch ip-10-0-1-126.ec2.internal I0420 11:40:27.043493 3689 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-2vmwd I0420 11:40:27.053160 3689 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-6nmq4] pod update took 24.545811ms I0420 11:40:27.053180 3689 base_network_controller_pods.go:951] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-6nmq4] addLogicalPort annotation time took 24.568205ms I0420 11:40:27.054485 3689 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-6nmq4] addLogicalPort took 25.988095ms, libovsdb time 1.050169ms I0420 11:40:27.060942 3689 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-lcpgd] pod update took 25.616366ms I0420 11:40:27.060965 3689 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-lcpgd] addLogicalPort annotation time took 25.643837ms I0420 11:40:27.061862 3689 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-lcpgd] addLogicalPort took 26.653799ms, libovsdb time 680.146µs I0420 11:40:27.068733 3689 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-2vmwd] pod update took 25.260268ms I0420 11:40:27.068750 3689 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-2vmwd] addLogicalPort annotation time took 25.281424ms I0420 11:40:27.069809 3689 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-2vmwd] addLogicalPort took 26.439404ms, libovsdb time 801.487µs I0420 11:40:27.137696 3689 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-qgzbl] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-qgzbl for pod on switch ip-10-0-1-126.ec2.internal I0420 11:40:27.137796 3689 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-qgzbl I0420 11:40:27.137806 3689 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-lwr79] creating logical port openshift-console-operator_console-operator-66dbbb6468-lwr79 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:40:27.137914 3689 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-lwr79 I0420 11:40:27.162071 3689 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-qgzbl] pod update took 24.306145ms I0420 11:40:27.162092 3689 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-qgzbl] addLogicalPort annotation time took 24.330907ms I0420 11:40:27.163176 3689 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-qgzbl] addLogicalPort took 25.49404ms, libovsdb time 841.371µs I0420 11:40:27.165477 3689 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-lwr79] pod update took 27.603402ms I0420 11:40:27.165496 3689 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-lwr79] addLogicalPort annotation time took 27.625091ms I0420 11:40:27.166412 3689 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-lwr79] addLogicalPort took 28.619484ms, libovsdb time 691.042µs I0420 11:40:27.178000 3689 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-94f4ccbc7-hp4j9] creating logical port openshift-ingress_router-default-94f4ccbc7-hp4j9 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:40:27.178086 3689 kube.go:256] Updating pod openshift-ingress/router-default-94f4ccbc7-hp4j9 I0420 11:40:27.200912 3689 pod.go:62] [openshift-ingress/router-default-94f4ccbc7-hp4j9] pod update took 22.84054ms I0420 11:40:27.200930 3689 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-94f4ccbc7-hp4j9] addLogicalPort annotation time took 22.880408ms I0420 11:40:27.201846 3689 pods.go:271] [openshift-ingress/router-default-94f4ccbc7-hp4j9] addLogicalPort took 23.856421ms, libovsdb time 623.367µs I0420 11:40:27.397637 3689 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-lcpgd 8b27146726e86a7752e3eee1aa535c285afc22033c1bef7e74996d2ebff6dcca network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-lcpgd 8b27146726e86a7752e3eee1aa535c285afc22033c1bef7e74996d2ebff6dcca network default NAD default NAD key ] I0420 11:40:27.400891 3689 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-2vmwd 6ad410ff782a7df67bb6dbcbf445f0a0a4002082b1f21462b632a9849fa0bbe9 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-2vmwd 6ad410ff782a7df67bb6dbcbf445f0a0a4002082b1f21462b632a9849fa0bbe9 network default NAD default NAD key ] I0420 11:40:27.406914 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-lcpgd, hostIfaceName: 8b27146726e86a7, network: default, NAD default, SandboxID: "8b27146726e86a7752e3eee1aa535c285afc22033c1bef7e74996d2ebff6dcca", PCI device ID: , UID: "a75c35fe-367c-413e-a2c6-25372bdc65a9", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0420 11:40:27.410260 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-2vmwd, hostIfaceName: 6ad410ff782a7df, network: default, NAD default, SandboxID: "6ad410ff782a7df67bb6dbcbf445f0a0a4002082b1f21462b632a9849fa0bbe9", PCI device ID: , UID: "bf091148-1f22-4d3a-aed2-c3f3dbe418cd", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0420 11:40:27.451607 3689 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-lcpgd 8b27146726e86a7752e3eee1aa535c285afc22033c1bef7e74996d2ebff6dcca network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-lcpgd 8b27146726e86a7752e3eee1aa535c285afc22033c1bef7e74996d2ebff6dcca network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:c0:f0:47:a3:cd\",\"name\":\"8b27146726e86a7\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/669409ae-0680-4680-bdf7-dac553083f4a\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:40:27.469460 3689 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-2vmwd 6ad410ff782a7df67bb6dbcbf445f0a0a4002082b1f21462b632a9849fa0bbe9 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-2vmwd 6ad410ff782a7df67bb6dbcbf445f0a0a4002082b1f21462b632a9849fa0bbe9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:f4:52:77:c6:ff\",\"name\":\"6ad410ff782a7df\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/06d0f00d-64fa-4acb-8696-44961a6ef7a5\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:40:27.502413 3689 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-lwr79 1e9863ed697789a452f190b4431005d164b7a57050aad73e6019f21f20750ac0 network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-lwr79 1e9863ed697789a452f190b4431005d164b7a57050aad73e6019f21f20750ac0 network default NAD default NAD key ] I0420 11:40:27.511878 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-lwr79, hostIfaceName: 1e9863ed697789a, network: default, NAD default, SandboxID: "1e9863ed697789a452f190b4431005d164b7a57050aad73e6019f21f20750ac0", PCI device ID: , UID: "838052a2-4570-492f-ae6a-2b1abefda6ac", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0420 11:40:27.556834 3689 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-lwr79 1e9863ed697789a452f190b4431005d164b7a57050aad73e6019f21f20750ac0 network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-lwr79 1e9863ed697789a452f190b4431005d164b7a57050aad73e6019f21f20750ac0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:5b:bd:d6:54:a1\",\"name\":\"1e9863ed697789a\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b26e0bc2-ef3a-44c6-acdd-d292dabec63b\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:40:30.877693 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0420 11:40:31.288457 3689 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0420 11:40:31.289453 3689 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 976.119µs I0420 11:40:31.325153 3689 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0420 11:40:31.463156 3689 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-rgf6p] creating logical port openshift-kube-storage-version-migrator_migrator-846d9b8bdc-rgf6p for pod on switch ip-10-0-1-126.ec2.internal I0420 11:40:31.463251 3689 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-846d9b8bdc-rgf6p I0420 11:40:31.485666 3689 pod.go:62] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-rgf6p] pod update took 22.439846ms I0420 11:40:31.485691 3689 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-rgf6p] addLogicalPort annotation time took 22.469487ms I0420 11:40:31.486883 3689 pods.go:271] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-rgf6p] addLogicalPort took 23.742587ms, libovsdb time 900.585µs I0420 11:40:31.821777 3689 cni.go:452] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-rgf6p be187bbb03768d8d8a8441e941105fe646aaca2262f834a8cca38f77d2eab1dd network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-rgf6p be187bbb03768d8d8a8441e941105fe646aaca2262f834a8cca38f77d2eab1dd network default NAD default NAD key ] I0420 11:40:31.831553 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-846d9b8bdc-rgf6p, hostIfaceName: be187bbb03768d8, network: default, NAD default, SandboxID: "be187bbb03768d8d8a8441e941105fe646aaca2262f834a8cca38f77d2eab1dd", PCI device ID: , UID: "6a651bbe-459b-4787-b062-5e6e4f0edf0c", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0420 11:40:31.881258 3689 cni.go:473] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-rgf6p be187bbb03768d8d8a8441e941105fe646aaca2262f834a8cca38f77d2eab1dd network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-rgf6p be187bbb03768d8d8a8441e941105fe646aaca2262f834a8cca38f77d2eab1dd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d2:98:30:f7:ef:dc\",\"name\":\"be187bbb03768d8\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e3309b66-cdf5-406f-8ef4-8cbf172c6e2d\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:40:31.912369 3689 namespace.go:98] [openshift-service-ca] adding namespace I0420 11:40:31.913275 3689 namespace.go:102] [openshift-service-ca] adding namespace took 882.412µs I0420 11:40:31.934379 3689 namespace.go:142] [openshift-service-ca] updating namespace W0420 11:40:40.446159 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0420 11:40:42.984695 3689 cni.go:452] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-6nmq4 e4a9828a5abf4ff55102133bff94f4e8163237359673fb05ff3bc6ca69f219f8 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-6nmq4 e4a9828a5abf4ff55102133bff94f4e8163237359673fb05ff3bc6ca69f219f8 network default NAD default NAD key ] I0420 11:40:42.994413 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67f4d568c7-6nmq4, hostIfaceName: e4a9828a5abf4ff, network: default, NAD default, SandboxID: "e4a9828a5abf4ff55102133bff94f4e8163237359673fb05ff3bc6ca69f219f8", PCI device ID: , UID: "0b150414-b637-434f-9e08-0e8b0a7bb7c1", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0420 11:40:43.040183 3689 cni.go:473] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-6nmq4 e4a9828a5abf4ff55102133bff94f4e8163237359673fb05ff3bc6ca69f219f8 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-6nmq4 e4a9828a5abf4ff55102133bff94f4e8163237359673fb05ff3bc6ca69f219f8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:25:d9:9f:8e:b2\",\"name\":\"e4a9828a5abf4ff\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1a980055-5438-49e1-823e-0a8d18ad96ab\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:40:52.266035 3689 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0420 11:40:52.322156 3689 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0420 11:40:52.322175 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0420 11:40:52.322180 3689 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0420 11:40:52.355109 3689 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0420 11:40:52.370973 3689 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-9vz9k] creating logical port openshift-insights_insights-runtime-extractor-9vz9k for pod on switch ip-10-0-1-126.ec2.internal I0420 11:40:52.371059 3689 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-9vz9k I0420 11:40:52.414206 3689 pod.go:62] [openshift-insights/insights-runtime-extractor-9vz9k] pod update took 43.175421ms I0420 11:40:52.414226 3689 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-9vz9k] addLogicalPort annotation time took 43.198688ms I0420 11:40:52.415288 3689 pods.go:271] [openshift-insights/insights-runtime-extractor-9vz9k] addLogicalPort took 44.334256ms, libovsdb time 784.53µs I0420 11:40:52.724763 3689 cni.go:452] [openshift-insights/insights-runtime-extractor-9vz9k 52626a1e8d9d02d3f10c361cf2510da218d5c01845b709bc9d503da202f4dea3 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-9vz9k 52626a1e8d9d02d3f10c361cf2510da218d5c01845b709bc9d503da202f4dea3 network default NAD default NAD key ] I0420 11:40:52.733673 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-9vz9k, hostIfaceName: 52626a1e8d9d02d, network: default, NAD default, SandboxID: "52626a1e8d9d02d3f10c361cf2510da218d5c01845b709bc9d503da202f4dea3", PCI device ID: , UID: "fed70b91-8c7c-4c6c-917f-65d6ef00b6af", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0420 11:40:52.780380 3689 cni.go:473] [openshift-insights/insights-runtime-extractor-9vz9k 52626a1e8d9d02d3f10c361cf2510da218d5c01845b709bc9d503da202f4dea3 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-9vz9k 52626a1e8d9d02d3f10c361cf2510da218d5c01845b709bc9d503da202f4dea3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:48:e2:3c:e9:98\",\"name\":\"52626a1e8d9d02d\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b35c40f6-69a9-467d-9978-d1bfa172fa97\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:40:53.517997 3689 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5c66559854-dmmkd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:40:53.518598 3689 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5c66559854-dmmkd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:40:53.537662 3689 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5c66559854-dmmkd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:40:53.537680 3689 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5c66559854-dmmkd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:40:53.552615 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5c66559854-dmmkd I0420 11:40:53.552635 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5c66559854-dmmkd W0420 11:40:53.947689 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics I0420 11:40:54.063162 3689 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-m9l8f] creating logical port openshift-console_downloads-5c54f4944f-m9l8f for pod on switch ip-10-0-1-126.ec2.internal I0420 11:40:54.063275 3689 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-m9l8f I0420 11:40:54.083555 3689 pod.go:62] [openshift-console/downloads-5c54f4944f-m9l8f] pod update took 20.315935ms I0420 11:40:54.083576 3689 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-m9l8f] addLogicalPort annotation time took 20.345185ms I0420 11:40:54.084707 3689 pods.go:271] [openshift-console/downloads-5c54f4944f-m9l8f] addLogicalPort took 21.586336ms, libovsdb time 817.312µs I0420 11:40:54.424132 3689 cni.go:452] [openshift-console/downloads-5c54f4944f-m9l8f 7059f41fa2bac2c18607e0510963eab06e29825142686c77780cce2758114dd5 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-m9l8f 7059f41fa2bac2c18607e0510963eab06e29825142686c77780cce2758114dd5 network default NAD default NAD key ] I0420 11:40:54.435836 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-m9l8f, hostIfaceName: 7059f41fa2bac2c, network: default, NAD default, SandboxID: "7059f41fa2bac2c18607e0510963eab06e29825142686c77780cce2758114dd5", PCI device ID: , UID: "065d5e3c-d07d-4fc7-9ff9-5a7d2abf977e", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0420 11:40:54.483343 3689 cni.go:473] [openshift-console/downloads-5c54f4944f-m9l8f 7059f41fa2bac2c18607e0510963eab06e29825142686c77780cce2758114dd5 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-m9l8f 7059f41fa2bac2c18607e0510963eab06e29825142686c77780cce2758114dd5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:e2:62:1d:4c:57\",\"name\":\"7059f41fa2bac2c\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9efbd1bd-b6c0-4f45-96d5-6379d8f94503\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:40:54.535912 3689 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0420 11:40:59.309927 3689 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-qgzbl 31639a46771610c01fc10d680150d1e8413b1a2ddf78d61972fbe4c8533d358b network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-qgzbl 31639a46771610c01fc10d680150d1e8413b1a2ddf78d61972fbe4c8533d358b network default NAD default NAD key ] I0420 11:40:59.331336 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-qgzbl, hostIfaceName: 31639a46771610c, network: default, NAD default, SandboxID: "31639a46771610c01fc10d680150d1e8413b1a2ddf78d61972fbe4c8533d358b", PCI device ID: , UID: "facdb827-af1c-4bdc-8b58-433c00b8734d", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0420 11:40:59.367644 3689 cni.go:452] [openshift-ingress/router-default-94f4ccbc7-hp4j9 9b75f5479b66d6b2a5b48239620370bd920ea8516d87b959b7a90ba62ea7ae0a network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-94f4ccbc7-hp4j9 9b75f5479b66d6b2a5b48239620370bd920ea8516d87b959b7a90ba62ea7ae0a network default NAD default NAD key ] I0420 11:40:59.378590 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-94f4ccbc7-hp4j9, hostIfaceName: 9b75f5479b66d6b, network: default, NAD default, SandboxID: "9b75f5479b66d6b2a5b48239620370bd920ea8516d87b959b7a90ba62ea7ae0a", PCI device ID: , UID: "c90bef2d-0da7-4ccd-99f5-5f8001a230d7", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0420 11:40:59.384989 3689 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-qgzbl 31639a46771610c01fc10d680150d1e8413b1a2ddf78d61972fbe4c8533d358b network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-qgzbl 31639a46771610c01fc10d680150d1e8413b1a2ddf78d61972fbe4c8533d358b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:87:30:ad:8a:96\",\"name\":\"31639a46771610c\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/712afa18-e7b4-4cff-8888-ba9cef833fe5\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:40:59.428611 3689 cni.go:473] [openshift-ingress/router-default-94f4ccbc7-hp4j9 9b75f5479b66d6b2a5b48239620370bd920ea8516d87b959b7a90ba62ea7ae0a network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-94f4ccbc7-hp4j9 9b75f5479b66d6b2a5b48239620370bd920ea8516d87b959b7a90ba62ea7ae0a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:9c:4d:37:b0:dc\",\"name\":\"9b75f5479b66d6b\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3eb9c593-d2ee-413f-808d-6b7140013b67\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:40:59.948008 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 11:40:59.948034 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 11:40:59.948045 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 11:40:59.948507 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0420 11:40:59.948533 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0420 11:41:00.326990 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0420 11:41:00.327009 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0420 11:41:00.329984 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 11:41:00.330000 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 11:41:00.330007 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 11:41:00.364429 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 11:41:00.364446 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 11:41:00.364455 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 11:41:00.371604 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0420 11:41:00.371626 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0420 11:41:00.985651 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0420 11:41:00.985678 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0420 11:41:01.102916 3689 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0420 11:41:01.116225 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 11:41:01.116240 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 11:41:01.116245 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 11:41:04.354347 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0420 11:41:04.563632 3689 base_network_controller_pods.go:487] [default/openshift-console/console-5bfdd69657-w4nq2] creating logical port openshift-console_console-5bfdd69657-w4nq2 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:41:04.563806 3689 kube.go:256] Updating pod openshift-console/console-5bfdd69657-w4nq2 I0420 11:41:04.579475 3689 pod.go:62] [openshift-console/console-5bfdd69657-w4nq2] pod update took 15.703486ms I0420 11:41:04.579499 3689 base_network_controller_pods.go:951] [default/openshift-console/console-5bfdd69657-w4nq2] addLogicalPort annotation time took 15.729011ms I0420 11:41:04.580819 3689 pods.go:271] [openshift-console/console-5bfdd69657-w4nq2] addLogicalPort took 17.202322ms, libovsdb time 964.035µs I0420 11:41:09.114847 3689 cni.go:452] [openshift-console/console-5bfdd69657-w4nq2 0377a9723f12fc7c88a71153f0ba45791a0b3d6a3e11c370f8526910ea44606e network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-5bfdd69657-w4nq2 0377a9723f12fc7c88a71153f0ba45791a0b3d6a3e11c370f8526910ea44606e network default NAD default NAD key ] I0420 11:41:09.123990 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-5bfdd69657-w4nq2, hostIfaceName: 0377a9723f12fc7, network: default, NAD default, SandboxID: "0377a9723f12fc7c88a71153f0ba45791a0b3d6a3e11c370f8526910ea44606e", PCI device ID: , UID: "a82dba23-7c84-43ed-8c60-65726f7d26ad", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0420 11:41:09.170434 3689 cni.go:473] [openshift-console/console-5bfdd69657-w4nq2 0377a9723f12fc7c88a71153f0ba45791a0b3d6a3e11c370f8526910ea44606e network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-5bfdd69657-w4nq2 0377a9723f12fc7c88a71153f0ba45791a0b3d6a3e11c370f8526910ea44606e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:19:0b:76:a6:18\",\"name\":\"0377a9723f12fc7\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b71a0dee-8aba-40f0-8ff4-27cdea1fcbd4\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:41:09.609145 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0420 11:41:10.012492 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0420 11:41:11.998168 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0420 11:41:12.615461 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0420 11:41:13.379545 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0420 11:41:15.031997 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0420 11:41:17.035947 3689 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5c66559854-bgrcm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:41:17.035994 3689 pods.go:176] Deleting pod: openshift-image-registry/image-registry-5c66559854-bgrcm I0420 11:41:17.037249 3689 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-5c66559854-bgrcm, ips: 10.128.0.5 I0420 11:41:17.037279 3689 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5c66559854-bgrcm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:41:17.051010 3689 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6767b958b6-kkgzd] creating logical port openshift-image-registry_image-registry-6767b958b6-kkgzd for pod on switch ip-10-0-1-126.ec2.internal I0420 11:41:17.051112 3689 kube.go:256] Updating pod openshift-image-registry/image-registry-6767b958b6-kkgzd I0420 11:41:17.068958 3689 pod.go:62] [openshift-image-registry/image-registry-6767b958b6-kkgzd] pod update took 17.882126ms I0420 11:41:17.068979 3689 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-6767b958b6-kkgzd] addLogicalPort annotation time took 17.90582ms I0420 11:41:17.070172 3689 pods.go:271] [openshift-image-registry/image-registry-6767b958b6-kkgzd] addLogicalPort took 19.176207ms, libovsdb time 869.971µs I0420 11:41:17.070225 3689 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5c66559854-bgrcm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:41:17.070241 3689 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5c66559854-bgrcm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:41:17.070259 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5c66559854-bgrcm I0420 11:41:17.070267 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5c66559854-bgrcm I0420 11:41:17.420445 3689 cni.go:452] [openshift-image-registry/image-registry-6767b958b6-kkgzd d797fb5f767f3ed2878e015bd6f59fdb93d13bb8006c310dfc1018974c0d6914 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-6767b958b6-kkgzd d797fb5f767f3ed2878e015bd6f59fdb93d13bb8006c310dfc1018974c0d6914 network default NAD default NAD key ] I0420 11:41:17.431080 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6767b958b6-kkgzd, hostIfaceName: d797fb5f767f3ed, network: default, NAD default, SandboxID: "d797fb5f767f3ed2878e015bd6f59fdb93d13bb8006c310dfc1018974c0d6914", PCI device ID: , UID: "ec0fa903-ae17-476a-8a5a-553d2aaddd08", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0420 11:41:17.484311 3689 cni.go:473] [openshift-image-registry/image-registry-6767b958b6-kkgzd d797fb5f767f3ed2878e015bd6f59fdb93d13bb8006c310dfc1018974c0d6914 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-6767b958b6-kkgzd d797fb5f767f3ed2878e015bd6f59fdb93d13bb8006c310dfc1018974c0d6914 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:cf:b0:ea:1e:4b\",\"name\":\"d797fb5f767f3ed\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e4cb3cd2-532c-408d-8557-f1aa2a3b8dc1\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:41:18.559924 3689 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6fb77b54c7-htcr8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:41:18.560287 3689 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6fb77b54c7-htcr8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:41:19.160569 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:41:19.160676 3689 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0420 11:41:19.194392 3689 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 33.753949ms I0420 11:41:19.194415 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 33.783173ms I0420 11:41:19.195853 3689 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 35.296185ms, libovsdb time 1.048077ms I0420 11:41:19.411404 3689 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6fb77b54c7-htcr8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:41:19.411428 3689 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6fb77b54c7-htcr8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:41:19.420086 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6fb77b54c7-htcr8 I0420 11:41:19.420110 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6fb77b54c7-htcr8 I0420 11:41:19.520300 3689 cni.go:452] [openshift-monitoring/alertmanager-main-1 3ca4950b5107eaa7798418d11c40b3ab5381618a4912f88ff07736494195c7ee network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 3ca4950b5107eaa7798418d11c40b3ab5381618a4912f88ff07736494195c7ee network default NAD default NAD key ] I0420 11:41:19.530718 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 3ca4950b5107eaa, network: default, NAD default, SandboxID: "3ca4950b5107eaa7798418d11c40b3ab5381618a4912f88ff07736494195c7ee", PCI device ID: , UID: "2dc661e0-1bfd-48a9-b724-07f0fcc2e63f", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0420 11:41:19.578151 3689 cni.go:473] [openshift-monitoring/alertmanager-main-1 3ca4950b5107eaa7798418d11c40b3ab5381618a4912f88ff07736494195c7ee network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 3ca4950b5107eaa7798418d11c40b3ab5381618a4912f88ff07736494195c7ee network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:4f:bd:74:c6:b6\",\"name\":\"3ca4950b5107eaa\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/84d62d6a-46b6-4527-b726-f8895d387ed4\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:41:21.113304 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-774f75c9d7-bdpl9] creating logical port openshift-monitoring_thanos-querier-774f75c9d7-bdpl9 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:41:21.113394 3689 kube.go:256] Updating pod openshift-monitoring/thanos-querier-774f75c9d7-bdpl9 I0420 11:41:21.131722 3689 pod.go:62] [openshift-monitoring/thanos-querier-774f75c9d7-bdpl9] pod update took 18.355594ms I0420 11:41:21.131740 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-774f75c9d7-bdpl9] addLogicalPort annotation time took 18.379961ms I0420 11:41:21.132847 3689 pods.go:271] [openshift-monitoring/thanos-querier-774f75c9d7-bdpl9] addLogicalPort took 19.559974ms, libovsdb time 803.956µs W0420 11:41:21.458678 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:21.458698 3689 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:21.458705 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0420 11:41:21.513635 3689 cni.go:452] [openshift-monitoring/thanos-querier-774f75c9d7-bdpl9 e76ab90d31808948a1dfe00225b255cfdb5b6cdb7907481110a8ebe24c4ce92a network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-774f75c9d7-bdpl9 e76ab90d31808948a1dfe00225b255cfdb5b6cdb7907481110a8ebe24c4ce92a network default NAD default NAD key ] I0420 11:41:21.522746 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-774f75c9d7-bdpl9, hostIfaceName: e76ab90d3180894, network: default, NAD default, SandboxID: "e76ab90d31808948a1dfe00225b255cfdb5b6cdb7907481110a8ebe24c4ce92a", PCI device ID: , UID: "af326dff-e1d9-4e99-8c72-ce37d4d80360", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0420 11:41:21.569546 3689 cni.go:473] [openshift-monitoring/thanos-querier-774f75c9d7-bdpl9 e76ab90d31808948a1dfe00225b255cfdb5b6cdb7907481110a8ebe24c4ce92a network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-774f75c9d7-bdpl9 e76ab90d31808948a1dfe00225b255cfdb5b6cdb7907481110a8ebe24c4ce92a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:1c:b7:bd:2f:f7\",\"name\":\"e76ab90d3180894\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/65e2970d-53de-43c8-a913-eff4df389e5b\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:41:22.052485 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:22.052504 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:22.052511 3689 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0420 11:41:22.638628 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-48tkt] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-48tkt for pod on switch ip-10-0-1-126.ec2.internal I0420 11:41:22.638801 3689 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-48tkt I0420 11:41:22.655474 3689 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-48tkt] pod update took 16.709326ms I0420 11:41:22.655503 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-48tkt] addLogicalPort annotation time took 16.742451ms I0420 11:41:22.656917 3689 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-48tkt] addLogicalPort took 18.293615ms, libovsdb time 1.006304ms I0420 11:41:22.776609 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-556cc9585b-62d96] creating logical port openshift-monitoring_metrics-server-556cc9585b-62d96 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:41:22.776719 3689 kube.go:256] Updating pod openshift-monitoring/metrics-server-556cc9585b-62d96 I0420 11:41:22.793167 3689 pod.go:62] [openshift-monitoring/metrics-server-556cc9585b-62d96] pod update took 16.477023ms I0420 11:41:22.793192 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-556cc9585b-62d96] addLogicalPort annotation time took 16.506082ms I0420 11:41:22.794577 3689 pods.go:271] [openshift-monitoring/metrics-server-556cc9585b-62d96] addLogicalPort took 17.983846ms, libovsdb time 918.555µs W0420 11:41:23.037719 3689 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-6f76dd66f7-bzz5c IP address from the namespace address-set, err: pod openshift-console/console-6f76dd66f7-bzz5c: no pod IPs found I0420 11:41:24.166912 3689 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-48tkt beff856934fcd63c57dd6f8ff9b7dd3e5cb6028259a5de554d23a3d6956416f6 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-48tkt beff856934fcd63c57dd6f8ff9b7dd3e5cb6028259a5de554d23a3d6956416f6 network default NAD default NAD key ] I0420 11:41:24.167817 3689 cni.go:452] [openshift-monitoring/metrics-server-556cc9585b-62d96 c63f346941a202e7a598992ea0dd224efab8574e7ecd81b4c0de03f049e30f9f network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-556cc9585b-62d96 c63f346941a202e7a598992ea0dd224efab8574e7ecd81b4c0de03f049e30f9f network default NAD default NAD key ] I0420 11:41:24.179169 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-48tkt, hostIfaceName: beff856934fcd63, network: default, NAD default, SandboxID: "beff856934fcd63c57dd6f8ff9b7dd3e5cb6028259a5de554d23a3d6956416f6", PCI device ID: , UID: "89d438d9-fc85-4c60-9a57-a6d8da441cdd", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0420 11:41:24.180426 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-556cc9585b-62d96, hostIfaceName: c63f346941a202e, network: default, NAD default, SandboxID: "c63f346941a202e7a598992ea0dd224efab8574e7ecd81b4c0de03f049e30f9f", PCI device ID: , UID: "67c66a70-0505-41a4-82ff-716140b6d000", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0420 11:41:24.229009 3689 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-48tkt beff856934fcd63c57dd6f8ff9b7dd3e5cb6028259a5de554d23a3d6956416f6 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-48tkt beff856934fcd63c57dd6f8ff9b7dd3e5cb6028259a5de554d23a3d6956416f6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:0a:c2:3c:73:da\",\"name\":\"beff856934fcd63\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/53353b48-31b6-46a8-8873-bac50f7ef10e\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:41:24.245105 3689 cni.go:473] [openshift-monitoring/metrics-server-556cc9585b-62d96 c63f346941a202e7a598992ea0dd224efab8574e7ecd81b4c0de03f049e30f9f network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-556cc9585b-62d96 c63f346941a202e7a598992ea0dd224efab8574e7ecd81b4c0de03f049e30f9f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a6:f1:54:13:bd:c7\",\"name\":\"c63f346941a202e\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f7f7beb9-ebaf-4b74-8d2b-c59f5b9a7789\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:41:25.459998 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0420 11:41:25.460017 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0420 11:41:25.486005 3689 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:25.486033 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:25.486040 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:25.486044 3689 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:25.665454 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0420 11:41:26.447245 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0420 11:41:27.111219 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:27.111240 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:27.111245 3689 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:27.111250 3689 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:27.135174 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0420 11:41:27.189083 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0420 11:41:27.728134 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0420 11:41:27.728156 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0420 11:41:32.465106 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:32.465135 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:32.465143 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:32.465151 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:32.465172 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:32.465178 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:32.465184 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:32.465190 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:33.101746 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:33.101772 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:33.101780 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:33.101787 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 11:41:36.492410 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0420 11:41:37.889836 3689 obj_retry.go:498] Detected object openshift-console/console-5bfdd69657-f7ssj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:41:37.890393 3689 obj_retry.go:498] Detected object openshift-console/console-5bfdd69657-f7ssj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:41:38.736306 3689 obj_retry.go:492] Detected object openshift-console/console-5bfdd69657-f7ssj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:41:38.736323 3689 obj_retry.go:492] Detected object openshift-console/console-5bfdd69657-f7ssj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:41:38.744796 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5bfdd69657-f7ssj I0420 11:41:38.744815 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5bfdd69657-f7ssj W0420 11:41:39.028649 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0420 11:41:45.731600 3689 cni.go:452] [openshift-console/console-5bfdd69657-w4nq2 0377a9723f12fc7c88a71153f0ba45791a0b3d6a3e11c370f8526910ea44606e network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-5bfdd69657-w4nq2 0377a9723f12fc7c88a71153f0ba45791a0b3d6a3e11c370f8526910ea44606e network default NAD default NAD key ] I0420 11:41:45.806167 3689 cni.go:473] [openshift-console/console-5bfdd69657-w4nq2 0377a9723f12fc7c88a71153f0ba45791a0b3d6a3e11c370f8526910ea44606e network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-5bfdd69657-w4nq2 0377a9723f12fc7c88a71153f0ba45791a0b3d6a3e11c370f8526910ea44606e network default NAD default NAD key default], result "{}", err I0420 11:41:45.879232 3689 obj_retry.go:498] Detected object openshift-console/console-5bfdd69657-w4nq2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:41:45.879265 3689 pods.go:176] Deleting pod: openshift-console/console-5bfdd69657-w4nq2 I0420 11:41:45.880553 3689 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5bfdd69657-w4nq2, ips: 10.128.0.19 I0420 11:41:45.880586 3689 obj_retry.go:498] Detected object openshift-console/console-5bfdd69657-w4nq2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:41:45.887639 3689 base_network_controller_pods.go:487] [default/openshift-console/console-859bbcd744-6cq2s] creating logical port openshift-console_console-859bbcd744-6cq2s for pod on switch ip-10-0-1-126.ec2.internal I0420 11:41:45.887764 3689 kube.go:256] Updating pod openshift-console/console-859bbcd744-6cq2s I0420 11:41:45.904007 3689 pod.go:62] [openshift-console/console-859bbcd744-6cq2s] pod update took 16.273351ms I0420 11:41:45.904033 3689 base_network_controller_pods.go:951] [default/openshift-console/console-859bbcd744-6cq2s] addLogicalPort annotation time took 16.300646ms I0420 11:41:45.905445 3689 pods.go:271] [openshift-console/console-859bbcd744-6cq2s] addLogicalPort took 17.821412ms, libovsdb time 988.441µs W0420 11:41:46.162560 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0420 11:41:46.162592 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0420 11:41:46.162642 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0420 11:41:46.174792 3689 obj_retry.go:492] Detected object openshift-console/console-5bfdd69657-w4nq2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:41:46.174812 3689 obj_retry.go:492] Detected object openshift-console/console-5bfdd69657-w4nq2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:41:46.186240 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5bfdd69657-w4nq2 I0420 11:41:46.186266 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5bfdd69657-w4nq2 I0420 11:41:46.287279 3689 cni.go:452] [openshift-console/console-859bbcd744-6cq2s 5eca8da96d46f414d6a6e1c811e728d80d8fddb3624fe253695e86f82b80ff1c network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-859bbcd744-6cq2s 5eca8da96d46f414d6a6e1c811e728d80d8fddb3624fe253695e86f82b80ff1c network default NAD default NAD key ] I0420 11:41:46.296459 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-859bbcd744-6cq2s, hostIfaceName: 5eca8da96d46f41, network: default, NAD default, SandboxID: "5eca8da96d46f414d6a6e1c811e728d80d8fddb3624fe253695e86f82b80ff1c", PCI device ID: , UID: "d45db7f7-c76d-47a3-a33a-37c7f05e0956", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0420 11:41:46.343943 3689 cni.go:473] [openshift-console/console-859bbcd744-6cq2s 5eca8da96d46f414d6a6e1c811e728d80d8fddb3624fe253695e86f82b80ff1c network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-859bbcd744-6cq2s 5eca8da96d46f414d6a6e1c811e728d80d8fddb3624fe253695e86f82b80ff1c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:67:d5:04:aa:d5\",\"name\":\"5eca8da96d46f41\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5146936f-9c33-440d-a8c7-6f6285e0c577\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:41:47.177966 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0420 11:41:47.462092 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0420 11:41:47.462111 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0420 11:41:48.090709 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0420 11:41:48.772598 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0420 11:41:48.772622 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0420 11:41:48.772637 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0420 11:41:48.772669 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0420 11:41:48.818596 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0420 11:41:48.818617 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0420 11:41:49.549509 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:49.549532 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:49.549538 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:49.549549 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:49.549553 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:49.549558 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:49.586694 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:49.586713 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:49.586719 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:41:51.193556 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0420 11:41:51.193645 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0420 11:41:51.193699 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0420 11:41:52.203269 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0420 11:41:57.226060 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0420 11:41:57.303151 3689 cni.go:452] [openshift-multus/network-metrics-daemon-58qj7 323d2d6789d59bfc8094933471141e2f5a473d8efca5a1a1de7e41b8d0bb6e89 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-58qj7 323d2d6789d59bfc8094933471141e2f5a473d8efca5a1a1de7e41b8d0bb6e89 network default NAD default NAD key ] I0420 11:41:57.312702 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-58qj7, hostIfaceName: 323d2d6789d59bf, network: default, NAD default, SandboxID: "323d2d6789d59bfc8094933471141e2f5a473d8efca5a1a1de7e41b8d0bb6e89", PCI device ID: , UID: "ff6b9f83-1fdc-4249-9ac2-1873d19174d8", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0420 11:41:57.362877 3689 cni.go:473] [openshift-multus/network-metrics-daemon-58qj7 323d2d6789d59bfc8094933471141e2f5a473d8efca5a1a1de7e41b8d0bb6e89 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-58qj7 323d2d6789d59bfc8094933471141e2f5a473d8efca5a1a1de7e41b8d0bb6e89 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:56:3f:1a:86:ec\",\"name\":\"323d2d6789d59bf\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b95e2428-1ecc-443d-ac3f-5aab0930ba4d\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:42:14.097988 3689 obj_retry.go:498] Detected object openshift-console/console-6f76dd66f7-jzjrm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:42:14.098515 3689 obj_retry.go:498] Detected object openshift-console/console-6f76dd66f7-jzjrm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:42:14.632544 3689 obj_retry.go:492] Detected object openshift-console/console-6f76dd66f7-jzjrm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:42:14.632565 3689 obj_retry.go:492] Detected object openshift-console/console-6f76dd66f7-jzjrm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:42:14.641453 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6f76dd66f7-jzjrm I0420 11:42:14.641474 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6f76dd66f7-jzjrm I0420 11:42:16.334159 3689 obj_retry.go:498] Detected object openshift-ingress/router-default-76d5dd5db5-b869k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:42:16.334699 3689 obj_retry.go:498] Detected object openshift-ingress/router-default-76d5dd5db5-b869k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:42:16.642387 3689 obj_retry.go:492] Detected object openshift-ingress/router-default-76d5dd5db5-b869k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:42:16.642408 3689 obj_retry.go:492] Detected object openshift-ingress/router-default-76d5dd5db5-b869k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:42:16.651262 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-76d5dd5db5-b869k I0420 11:42:16.651287 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-76d5dd5db5-b869k I0420 11:42:26.355128 3689 cni.go:452] [openshift-dns/dns-default-l5dvc 2591e5e61f572d202d1b2b65584ab459d97c621ed4ec252383c136ff95f45498 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-l5dvc 2591e5e61f572d202d1b2b65584ab459d97c621ed4ec252383c136ff95f45498 network default NAD default NAD key ] I0420 11:42:26.364335 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-l5dvc, hostIfaceName: 2591e5e61f572d2, network: default, NAD default, SandboxID: "2591e5e61f572d202d1b2b65584ab459d97c621ed4ec252383c136ff95f45498", PCI device ID: , UID: "7661dcfc-010d-4244-aca2-68e22c2ee237", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0420 11:42:26.413403 3689 cni.go:473] [openshift-dns/dns-default-l5dvc 2591e5e61f572d202d1b2b65584ab459d97c621ed4ec252383c136ff95f45498 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-l5dvc 2591e5e61f572d202d1b2b65584ab459d97c621ed4ec252383c136ff95f45498 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:be:0e:2a:db:e4\",\"name\":\"2591e5e61f572d2\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bda8eee7-a000-4698-b132-14f9648730dd\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:42:38.361969 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0420 11:42:38.361989 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0420 11:42:38.361997 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0420 11:42:40.237206 3689 cni.go:452] [openshift-monitoring/alertmanager-main-1 3ca4950b5107eaa7798418d11c40b3ab5381618a4912f88ff07736494195c7ee network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 3ca4950b5107eaa7798418d11c40b3ab5381618a4912f88ff07736494195c7ee network default NAD default NAD key ] I0420 11:42:40.306992 3689 cni.go:473] [openshift-monitoring/alertmanager-main-1 3ca4950b5107eaa7798418d11c40b3ab5381618a4912f88ff07736494195c7ee network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 3ca4950b5107eaa7798418d11c40b3ab5381618a4912f88ff07736494195c7ee network default NAD default NAD key default], result "{}", err I0420 11:42:40.373032 3689 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:42:40.373073 3689 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0420 11:42:40.374535 3689 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.128.0.21 I0420 11:42:40.374562 3689 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:42:40.709472 3689 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:42:40.709500 3689 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:42:40.719100 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0420 11:42:40.719130 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0420 11:42:40.765440 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:42:40.765613 3689 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0420 11:42:40.792341 3689 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 26.782441ms I0420 11:42:40.792363 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 26.808812ms I0420 11:42:40.793581 3689 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 28.16171ms, libovsdb time 895.62µs I0420 11:42:41.124548 3689 cni.go:452] [openshift-monitoring/alertmanager-main-1 12e01ae4448d5324aea8bf455073662986b2901158dda0da55e51183d9f4b0b0 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 12e01ae4448d5324aea8bf455073662986b2901158dda0da55e51183d9f4b0b0 network default NAD default NAD key ] I0420 11:42:41.134629 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 12e01ae4448d532, network: default, NAD default, SandboxID: "12e01ae4448d5324aea8bf455073662986b2901158dda0da55e51183d9f4b0b0", PCI device ID: , UID: "b31d207c-4710-4d87-ac93-f11fb8be8589", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0420 11:42:41.182458 3689 cni.go:473] [openshift-monitoring/alertmanager-main-1 12e01ae4448d5324aea8bf455073662986b2901158dda0da55e51183d9f4b0b0 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 12e01ae4448d5324aea8bf455073662986b2901158dda0da55e51183d9f4b0b0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:62:86:3a:e0:68\",\"name\":\"12e01ae4448d532\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/229eaeef-75ea-456f-975f-29cbbe2677dd\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:42:43.156171 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-7c75746957-nfq8m] creating logical port openshift-monitoring_telemeter-client-7c75746957-nfq8m for pod on switch ip-10-0-1-126.ec2.internal I0420 11:42:43.156271 3689 kube.go:256] Updating pod openshift-monitoring/telemeter-client-7c75746957-nfq8m I0420 11:42:43.175491 3689 pod.go:62] [openshift-monitoring/telemeter-client-7c75746957-nfq8m] pod update took 19.247612ms I0420 11:42:43.175521 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/telemeter-client-7c75746957-nfq8m] addLogicalPort annotation time took 19.28077ms I0420 11:42:43.178081 3689 pods.go:271] [openshift-monitoring/telemeter-client-7c75746957-nfq8m] addLogicalPort took 21.930232ms, libovsdb time 1.821395ms I0420 11:42:43.533288 3689 cni.go:452] [openshift-monitoring/telemeter-client-7c75746957-nfq8m 4822e5bbc9ba4b8243f52426c318363ae6b4b25f535d02b42dd11c77aebccd61 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/telemeter-client-7c75746957-nfq8m 4822e5bbc9ba4b8243f52426c318363ae6b4b25f535d02b42dd11c77aebccd61 network default NAD default NAD key ] I0420 11:42:43.542238 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-7c75746957-nfq8m, hostIfaceName: 4822e5bbc9ba4b8, network: default, NAD default, SandboxID: "4822e5bbc9ba4b8243f52426c318363ae6b4b25f535d02b42dd11c77aebccd61", PCI device ID: , UID: "9ad8e3a2-945c-4727-a6d8-06bd2927ed37", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0420 11:42:43.589554 3689 cni.go:473] [openshift-monitoring/telemeter-client-7c75746957-nfq8m 4822e5bbc9ba4b8243f52426c318363ae6b4b25f535d02b42dd11c77aebccd61 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/telemeter-client-7c75746957-nfq8m 4822e5bbc9ba4b8243f52426c318363ae6b4b25f535d02b42dd11c77aebccd61 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:72:1f:e3:0b:48\",\"name\":\"4822e5bbc9ba4b8\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7d9f154c-124e-481f-bd45-e34d8dd21e5f\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:42:44.088856 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611420-jjt9x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:42:44.089478 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611420-jjt9x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:42:44.763842 3689 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:42:44.764442 3689 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:42:44.985051 3689 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:42:44.985080 3689 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:42:44.986270 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611420-jjt9x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:42:44.986282 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611420-jjt9x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:42:44.997502 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0420 11:42:44.997533 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 W0420 11:43:01.190691 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:43:01.190713 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 11:43:01.190719 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0420 11:43:02.498006 3689 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:43:02.498622 3689 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:43:02.841891 3689 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:43:02.841929 3689 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:43:02.851107 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0420 11:43:02.851131 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0420 11:43:11.702342 3689 obj_retry.go:498] Detected object openshift-console/console-859bbcd744-xvlbn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:43:11.702864 3689 obj_retry.go:498] Detected object openshift-console/console-859bbcd744-xvlbn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:43:12.086553 3689 obj_retry.go:492] Detected object openshift-console/console-859bbcd744-xvlbn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:43:12.086579 3689 obj_retry.go:492] Detected object openshift-console/console-859bbcd744-xvlbn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:43:12.096301 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-859bbcd744-xvlbn I0420 11:43:12.096325 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-859bbcd744-xvlbn I0420 11:43:13.192047 3689 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0420 11:43:22.027708 3689 cni.go:452] [openshift-console/console-859bbcd744-6cq2s 5eca8da96d46f414d6a6e1c811e728d80d8fddb3624fe253695e86f82b80ff1c network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-859bbcd744-6cq2s 5eca8da96d46f414d6a6e1c811e728d80d8fddb3624fe253695e86f82b80ff1c network default NAD default NAD key ] I0420 11:43:22.094123 3689 cni.go:473] [openshift-console/console-859bbcd744-6cq2s 5eca8da96d46f414d6a6e1c811e728d80d8fddb3624fe253695e86f82b80ff1c network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-859bbcd744-6cq2s 5eca8da96d46f414d6a6e1c811e728d80d8fddb3624fe253695e86f82b80ff1c network default NAD default NAD key default], result "{}", err I0420 11:43:22.148838 3689 obj_retry.go:498] Detected object openshift-console/console-859bbcd744-6cq2s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:43:22.148880 3689 pods.go:176] Deleting pod: openshift-console/console-859bbcd744-6cq2s I0420 11:43:22.150399 3689 pods.go:236] Attempting to release IPs for pod: openshift-console/console-859bbcd744-6cq2s, ips: 10.128.0.25 I0420 11:43:22.150434 3689 obj_retry.go:498] Detected object openshift-console/console-859bbcd744-6cq2s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:43:22.590361 3689 obj_retry.go:492] Detected object openshift-console/console-859bbcd744-6cq2s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:43:22.590382 3689 obj_retry.go:492] Detected object openshift-console/console-859bbcd744-6cq2s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:43:22.599019 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-859bbcd744-6cq2s I0420 11:43:22.599049 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-859bbcd744-6cq2s I0420 11:43:37.513999 3689 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0420 11:43:47.581871 3689 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:43:47.582394 3689 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:43:48.031719 3689 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:43:48.031745 3689 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:43:48.043329 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0420 11:43:48.043351 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0420 11:43:49.491217 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0420 11:44:12.396876 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=141 I0420 11:44:25.421366 3689 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xkxtw] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-xkxtw for pod on switch ip-10-0-1-126.ec2.internal I0420 11:44:25.421475 3689 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xkxtw I0420 11:44:25.436083 3689 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xkxtw] pod update took 14.642187ms I0420 11:44:25.436106 3689 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xkxtw] addLogicalPort annotation time took 14.668242ms I0420 11:44:25.437381 3689 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xkxtw] addLogicalPort took 16.031509ms, libovsdb time 954.336µs I0420 11:44:25.781015 3689 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xkxtw c8f4de40c7220d8d82bcdbc8d08378834da9fbef8e5c8c5f083fa9920fb91756 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xkxtw c8f4de40c7220d8d82bcdbc8d08378834da9fbef8e5c8c5f083fa9920fb91756 network default NAD default NAD key ] I0420 11:44:25.790274 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-xkxtw, hostIfaceName: c8f4de40c7220d8, network: default, NAD default, SandboxID: "c8f4de40c7220d8d82bcdbc8d08378834da9fbef8e5c8c5f083fa9920fb91756", PCI device ID: , UID: "d934feb2-e30c-4dda-8696-1f2112b58fcd", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0420 11:44:25.839219 3689 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xkxtw c8f4de40c7220d8d82bcdbc8d08378834da9fbef8e5c8c5f083fa9920fb91756 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-xkxtw c8f4de40c7220d8d82bcdbc8d08378834da9fbef8e5c8c5f083fa9920fb91756 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:58:33:3b:7a:8d\",\"name\":\"c8f4de40c7220d8\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ece86ab-0943-41ef-8b37-b6fa58a88136\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:44:28.862362 3689 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0420 11:44:35.482241 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0420 11:44:35.497035 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 W0420 11:44:39.874394 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0420 11:44:39.919519 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0420 11:44:39.919589 3689 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-tjw2q IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-tjw2q: no pod IPs found I0420 11:44:54.015358 3689 base_network_controller_pods.go:487] [default/openshift-console/console-689bcb7788-2kbb5] creating logical port openshift-console_console-689bcb7788-2kbb5 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:44:54.015471 3689 kube.go:256] Updating pod openshift-console/console-689bcb7788-2kbb5 I0420 11:44:54.038811 3689 pod.go:62] [openshift-console/console-689bcb7788-2kbb5] pod update took 23.372459ms I0420 11:44:54.038834 3689 base_network_controller_pods.go:951] [default/openshift-console/console-689bcb7788-2kbb5] addLogicalPort annotation time took 23.400198ms I0420 11:44:54.040027 3689 pods.go:271] [openshift-console/console-689bcb7788-2kbb5] addLogicalPort took 24.691828ms, libovsdb time 833.274µs I0420 11:44:54.374198 3689 cni.go:452] [openshift-console/console-689bcb7788-2kbb5 78f4ef831c57248ce34d32a5ad873695ccbb600e2d15ea05ca686e47dd2870a9 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-689bcb7788-2kbb5 78f4ef831c57248ce34d32a5ad873695ccbb600e2d15ea05ca686e47dd2870a9 network default NAD default NAD key ] I0420 11:44:54.383488 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-689bcb7788-2kbb5, hostIfaceName: 78f4ef831c57248, network: default, NAD default, SandboxID: "78f4ef831c57248ce34d32a5ad873695ccbb600e2d15ea05ca686e47dd2870a9", PCI device ID: , UID: "7c5635f9-143b-4b8c-b042-ab9dde4e5179", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0420 11:44:54.432562 3689 cni.go:473] [openshift-console/console-689bcb7788-2kbb5 78f4ef831c57248ce34d32a5ad873695ccbb600e2d15ea05ca686e47dd2870a9 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-689bcb7788-2kbb5 78f4ef831c57248ce34d32a5ad873695ccbb600e2d15ea05ca686e47dd2870a9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d2:3a:1d:76:33:2a\",\"name\":\"78f4ef831c57248\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/478d8362-d4e2-410f-8d1e-fa8174ae542d\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:45:05.024831 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0420 11:45:12.405304 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=687 I0420 11:45:19.333823 3689 obj_retry.go:498] Detected object openshift-console/console-77ddb99d5-z7cv8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:45:19.334385 3689 obj_retry.go:498] Detected object openshift-console/console-77ddb99d5-z7cv8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:45:19.701401 3689 obj_retry.go:492] Detected object openshift-console/console-77ddb99d5-z7cv8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:45:19.701426 3689 obj_retry.go:492] Detected object openshift-console/console-77ddb99d5-z7cv8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:45:19.711073 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-77ddb99d5-z7cv8 I0420 11:45:19.711087 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-77ddb99d5-z7cv8 I0420 11:45:26.520075 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0420 11:45:30.351550 3689 obj_retry.go:498] Detected object openshift-console/console-77ddb99d5-dkx4b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:45:30.352153 3689 obj_retry.go:498] Detected object openshift-console/console-77ddb99d5-dkx4b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:45:30.611204 3689 obj_retry.go:492] Detected object openshift-console/console-77ddb99d5-dkx4b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:45:30.611224 3689 obj_retry.go:492] Detected object openshift-console/console-77ddb99d5-dkx4b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:45:30.616717 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-77ddb99d5-dkx4b I0420 11:45:30.616742 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-77ddb99d5-dkx4b I0420 11:45:34.399936 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0420 11:45:34.519982 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0420 11:45:36.485660 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0420 11:45:58.482140 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0420 11:46:16.508857 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0420 11:46:37.398872 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=16 I0420 11:47:23.503372 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0420 11:47:26.396860 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=108 I0420 11:47:57.400974 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=74 I0420 11:49:28.516130 3689 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0420 11:50:03.717751 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611430-9b9wf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:50:03.718382 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611430-9b9wf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:50:04.616400 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611430-9b9wf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:50:04.616422 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611430-9b9wf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:50:16.399082 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=19 I0420 11:50:34.402133 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0420 11:50:55.494259 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0420 11:50:59.999546 3689 namespace.go:98] [tsf] adding namespace I0420 11:51:00.000568 3689 namespace.go:102] [tsf] adding namespace took 997.693µs I0420 11:51:00.017652 3689 namespace.go:142] [tsf] updating namespace I0420 11:51:00.031403 3689 namespace.go:142] [tsf] updating namespace I0420 11:51:01.499235 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0420 11:51:04.020512 3689 namespace.go:98] [rhtpa-operator] adding namespace I0420 11:51:04.020569 3689 namespace.go:98] [konflux-operator] adding namespace I0420 11:51:04.020724 3689 namespace.go:98] [tsf-tpa] adding namespace I0420 11:51:04.021451 3689 namespace.go:102] [rhtpa-operator] adding namespace took 916.016µs I0420 11:51:04.022139 3689 namespace.go:102] [konflux-operator] adding namespace took 1.549479ms I0420 11:51:04.022700 3689 namespace.go:102] [tsf-tpa] adding namespace took 1.955893ms I0420 11:51:04.025329 3689 namespace.go:98] [rhbk-operator] adding namespace I0420 11:51:04.025516 3689 namespace.go:98] [konflux-ui] adding namespace I0420 11:51:04.025945 3689 namespace.go:102] [rhbk-operator] adding namespace took 602.725µs I0420 11:51:04.025969 3689 namespace.go:98] [tsf-tas] adding namespace I0420 11:51:04.026220 3689 namespace.go:98] [tsf-keycloak] adding namespace I0420 11:51:04.026342 3689 namespace.go:98] [cert-manager-operator] adding namespace I0420 11:51:04.026586 3689 namespace.go:102] [tsf-tas] adding namespace took 610.616µs I0420 11:51:04.027196 3689 namespace.go:102] [konflux-ui] adding namespace took 1.663393ms I0420 11:51:04.027728 3689 namespace.go:102] [tsf-keycloak] adding namespace took 1.496492ms I0420 11:51:04.028323 3689 namespace.go:102] [cert-manager-operator] adding namespace took 1.971169ms I0420 11:51:04.047557 3689 namespace.go:142] [rhtpa-operator] updating namespace I0420 11:51:04.067408 3689 namespace.go:142] [rhtpa-operator] updating namespace I0420 11:51:04.079818 3689 namespace.go:142] [konflux-operator] updating namespace I0420 11:51:04.090173 3689 namespace.go:142] [tsf-tpa] updating namespace I0420 11:51:04.090272 3689 namespace.go:142] [rhbk-operator] updating namespace I0420 11:51:04.095832 3689 namespace.go:142] [tsf-tas] updating namespace I0420 11:51:04.095886 3689 namespace.go:142] [konflux-operator] updating namespace I0420 11:51:04.097965 3689 namespace.go:142] [konflux-ui] updating namespace I0420 11:51:04.110680 3689 namespace.go:142] [tsf-keycloak] updating namespace I0420 11:51:04.118653 3689 namespace.go:142] [cert-manager-operator] updating namespace I0420 11:51:04.118778 3689 namespace.go:142] [rhtpa-operator] updating namespace I0420 11:51:04.118794 3689 namespace.go:142] [konflux-operator] updating namespace I0420 11:51:04.169662 3689 namespace.go:142] [rhbk-operator] updating namespace I0420 11:51:04.191943 3689 namespace.go:142] [rhbk-operator] updating namespace I0420 11:51:04.205376 3689 namespace.go:142] [tsf-tas] updating namespace I0420 11:51:04.223495 3689 namespace.go:142] [tsf-tas] updating namespace I0420 11:51:04.237411 3689 namespace.go:142] [konflux-ui] updating namespace I0420 11:51:04.251948 3689 namespace.go:142] [konflux-ui] updating namespace I0420 11:51:04.259200 3689 namespace.go:142] [tsf-keycloak] updating namespace I0420 11:51:04.307078 3689 namespace.go:142] [tsf-keycloak] updating namespace I0420 11:51:04.313535 3689 namespace.go:142] [cert-manager-operator] updating namespace I0420 11:51:04.328771 3689 namespace.go:142] [cert-manager-operator] updating namespace I0420 11:51:04.389024 3689 namespace.go:142] [tsf-tpa] updating namespace I0420 11:51:04.408949 3689 namespace.go:142] [tsf-tpa] updating namespace I0420 11:51:05.334456 3689 namespace.go:142] [tsf-tpa] updating namespace I0420 11:51:05.346599 3689 namespace.go:142] [tsf-keycloak] updating namespace I0420 11:51:05.503933 3689 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-1-126.ec2.internal I0420 11:51:05.504057 3689 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0420 11:51:05.521042 3689 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 17.011475ms I0420 11:51:05.521061 3689 base_network_controller_pods.go:951] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 17.035095ms I0420 11:51:05.522400 3689 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 18.481937ms, libovsdb time 953.646µs I0420 11:51:05.880456 3689 cni.go:452] [tsf/test-tsf-subscriptions cb3bb667147916051f63d2d180271f148f347d991967f7fed5186338d7c42444 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-subscriptions cb3bb667147916051f63d2d180271f148f347d991967f7fed5186338d7c42444 network default NAD default NAD key ] I0420 11:51:05.889717 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: cb3bb6671479160, network: default, NAD default, SandboxID: "cb3bb667147916051f63d2d180271f148f347d991967f7fed5186338d7c42444", PCI device ID: , UID: "56f1310d-7851-4810-8c8c-e64ef868d494", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0420 11:51:05.939169 3689 cni.go:473] [tsf/test-tsf-subscriptions cb3bb667147916051f63d2d180271f148f347d991967f7fed5186338d7c42444 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-subscriptions cb3bb667147916051f63d2d180271f148f347d991967f7fed5186338d7c42444 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:b6:1a:1a:67:2b\",\"name\":\"cb3bb6671479160\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2d5f5f7c-797b-44c8-aa73-85f5834c983d\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:51:07.060244 3689 base_network_controller_pods.go:487] [default/openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd] creating logical port openshift-marketplace_a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd for pod on switch ip-10-0-1-126.ec2.internal I0420 11:51:07.060322 3689 base_network_controller_pods.go:487] [default/openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv] creating logical port openshift-marketplace_53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv for pod on switch ip-10-0-1-126.ec2.internal I0420 11:51:07.060365 3689 kube.go:256] Updating pod openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd I0420 11:51:07.060415 3689 kube.go:256] Updating pod openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv I0420 11:51:07.075061 3689 pod.go:62] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv] pod update took 14.667847ms I0420 11:51:07.075087 3689 base_network_controller_pods.go:951] [default/openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv] addLogicalPort annotation time took 14.695096ms I0420 11:51:07.076360 3689 pods.go:271] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv] addLogicalPort took 16.050312ms, libovsdb time 846.028µs I0420 11:51:07.078636 3689 pod.go:62] [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd] pod update took 18.299361ms I0420 11:51:07.078656 3689 base_network_controller_pods.go:951] [default/openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd] addLogicalPort annotation time took 18.321121ms I0420 11:51:07.080038 3689 pods.go:271] [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd] addLogicalPort took 19.812498ms, libovsdb time 1.034705ms I0420 11:51:07.417292 3689 cni.go:452] [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd ac8c11342df094b8fa6baf02d088dfd959b67b986f930d7cfd073d18abe248de network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd ac8c11342df094b8fa6baf02d088dfd959b67b986f930d7cfd073d18abe248de network default NAD default NAD key ] I0420 11:51:07.423942 3689 cni.go:452] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv f9413b3a1783e3e643ecde077fb4f96a6b0d1bcae0f8d8993c70485c45234d76 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv f9413b3a1783e3e643ecde077fb4f96a6b0d1bcae0f8d8993c70485c45234d76 network default NAD default NAD key ] I0420 11:51:07.426661 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd, hostIfaceName: ac8c11342df094b, network: default, NAD default, SandboxID: "ac8c11342df094b8fa6baf02d088dfd959b67b986f930d7cfd073d18abe248de", PCI device ID: , UID: "a48dfb94-12f7-4f67-b557-b823ba9fd649", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0420 11:51:07.437028 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv, hostIfaceName: f9413b3a1783e3e, network: default, NAD default, SandboxID: "f9413b3a1783e3e643ecde077fb4f96a6b0d1bcae0f8d8993c70485c45234d76", PCI device ID: , UID: "a3fde8bd-4d06-43d9-b8d6-dfa6c887c05b", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0420 11:51:07.478343 3689 cni.go:473] [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd ac8c11342df094b8fa6baf02d088dfd959b67b986f930d7cfd073d18abe248de network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd ac8c11342df094b8fa6baf02d088dfd959b67b986f930d7cfd073d18abe248de network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:21:59:a7:6c:c8\",\"name\":\"ac8c11342df094b\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/31b8b234-2b5b-48c8-b2d1-84d22bc18327\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:51:07.502163 3689 cni.go:473] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv f9413b3a1783e3e643ecde077fb4f96a6b0d1bcae0f8d8993c70485c45234d76 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv f9413b3a1783e3e643ecde077fb4f96a6b0d1bcae0f8d8993c70485c45234d76 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:9e:c2:36:15:f6\",\"name\":\"f9413b3a1783e3e\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7907f784-ba04-4430-899a-0cc434daba1c\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:51:10.366272 3689 base_network_controller_pods.go:487] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r] creating logical port openshift-marketplace_925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r for pod on switch ip-10-0-1-126.ec2.internal I0420 11:51:10.366375 3689 kube.go:256] Updating pod openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r I0420 11:51:10.381151 3689 pod.go:62] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r] pod update took 14.801165ms I0420 11:51:10.381180 3689 base_network_controller_pods.go:951] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r] addLogicalPort annotation time took 14.835609ms I0420 11:51:10.382713 3689 pods.go:271] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r] addLogicalPort took 16.459195ms, libovsdb time 1.032606ms I0420 11:51:10.742123 3689 cni.go:452] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r 9bfad672dab9f8d19eaea1bd774e6190b4525792a71f22c8c8e3689a4672f7d0 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r 9bfad672dab9f8d19eaea1bd774e6190b4525792a71f22c8c8e3689a4672f7d0 network default NAD default NAD key ] I0420 11:51:10.758235 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r, hostIfaceName: 9bfad672dab9f8d, network: default, NAD default, SandboxID: "9bfad672dab9f8d19eaea1bd774e6190b4525792a71f22c8c8e3689a4672f7d0", PCI device ID: , UID: "6275905c-0c29-4001-be2e-a33c6e8b2404", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0420 11:51:10.813109 3689 cni.go:473] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r 9bfad672dab9f8d19eaea1bd774e6190b4525792a71f22c8c8e3689a4672f7d0 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r 9bfad672dab9f8d19eaea1bd774e6190b4525792a71f22c8c8e3689a4672f7d0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:e2:c9:d0:98:71\",\"name\":\"9bfad672dab9f8d\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/73ec0a9d-a2f7-40e3-afe8-47b803d69a09\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:51:11.674448 3689 cni.go:452] [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd ac8c11342df094b8fa6baf02d088dfd959b67b986f930d7cfd073d18abe248de network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd ac8c11342df094b8fa6baf02d088dfd959b67b986f930d7cfd073d18abe248de network default NAD default NAD key ] I0420 11:51:11.677917 3689 cni.go:452] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv f9413b3a1783e3e643ecde077fb4f96a6b0d1bcae0f8d8993c70485c45234d76 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv f9413b3a1783e3e643ecde077fb4f96a6b0d1bcae0f8d8993c70485c45234d76 network default NAD default NAD key ] I0420 11:51:11.739657 3689 base_network_controller_pods.go:487] [default/openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9] creating logical port openshift-marketplace_f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:51:11.739811 3689 kube.go:256] Updating pod openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 I0420 11:51:11.754479 3689 pod.go:62] [openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9] pod update took 14.709609ms I0420 11:51:11.754561 3689 base_network_controller_pods.go:951] [default/openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9] addLogicalPort annotation time took 14.796924ms I0420 11:51:11.755958 3689 pods.go:271] [openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9] addLogicalPort took 16.31984ms, libovsdb time 924.934µs I0420 11:51:11.756472 3689 cni.go:473] [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd ac8c11342df094b8fa6baf02d088dfd959b67b986f930d7cfd073d18abe248de network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd ac8c11342df094b8fa6baf02d088dfd959b67b986f930d7cfd073d18abe248de network default NAD default NAD key default], result "{}", err I0420 11:51:11.795351 3689 cni.go:473] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv f9413b3a1783e3e643ecde077fb4f96a6b0d1bcae0f8d8993c70485c45234d76 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv f9413b3a1783e3e643ecde077fb4f96a6b0d1bcae0f8d8993c70485c45234d76 network default NAD default NAD key default], result "{}", err I0420 11:51:13.170473 3689 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:51:13.170550 3689 pods.go:176] Deleting pod: openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv I0420 11:51:13.172291 3689 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv, ips: 10.128.0.32 I0420 11:51:13.172325 3689 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:51:13.172353 3689 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:51:13.172361 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0420 11:51:13.172602 3689 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:51:13.172621 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0420 11:51:13.587613 3689 base_network_controller_pods.go:487] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt] creating logical port openshift-marketplace_30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt for pod on switch ip-10-0-1-126.ec2.internal I0420 11:51:13.587753 3689 kube.go:256] Updating pod openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt I0420 11:51:13.603402 3689 pod.go:62] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt] pod update took 15.672325ms I0420 11:51:13.603438 3689 base_network_controller_pods.go:951] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt] addLogicalPort annotation time took 15.71507ms I0420 11:51:13.605018 3689 pods.go:271] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt] addLogicalPort took 17.424737ms, libovsdb time 1.051808ms I0420 11:51:13.691475 3689 base_network_controller_pods.go:487] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j] creating logical port openshift-marketplace_18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j for pod on switch ip-10-0-1-126.ec2.internal I0420 11:51:13.691601 3689 kube.go:256] Updating pod openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j I0420 11:51:13.706080 3689 pod.go:62] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j] pod update took 14.506855ms I0420 11:51:13.706109 3689 base_network_controller_pods.go:951] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j] addLogicalPort annotation time took 14.53917ms I0420 11:51:13.707580 3689 pods.go:271] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j] addLogicalPort took 16.125873ms, libovsdb time 964.95µs I0420 11:51:14.199871 3689 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:14.199892 3689 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:14.199927 3689 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:14.199946 3689 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece54bhv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:14.252020 3689 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:51:14.252077 3689 pods.go:176] Deleting pod: openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd I0420 11:51:14.253745 3689 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd, ips: 10.128.0.31 I0420 11:51:14.253782 3689 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:51:14.253802 3689 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:51:14.253809 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0420 11:51:14.254050 3689 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:51:14.254069 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0420 11:51:14.442147 3689 cni.go:452] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j a65449ee66c7698bdfe2d1c11398ecbd1ece01ba623429839fdbfde76df05018 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j a65449ee66c7698bdfe2d1c11398ecbd1ece01ba623429839fdbfde76df05018 network default NAD default NAD key ] I0420 11:51:14.442441 3689 cni.go:452] [openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 a8640211b47241ca56ba7baca041e629c2bcadbe938d04e930480e24981db932 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 a8640211b47241ca56ba7baca041e629c2bcadbe938d04e930480e24981db932 network default NAD default NAD key ] I0420 11:51:14.442876 3689 cni.go:452] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt 804aa9ca9a40344747c34ffdde201b90cde77b3613c38059b54d4fb9d1131aca network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt 804aa9ca9a40344747c34ffdde201b90cde77b3613c38059b54d4fb9d1131aca network default NAD default NAD key ] I0420 11:51:14.456227 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j, hostIfaceName: a65449ee66c7698, network: default, NAD default, SandboxID: "a65449ee66c7698bdfe2d1c11398ecbd1ece01ba623429839fdbfde76df05018", PCI device ID: , UID: "47f935b4-c274-4ad5-a3ff-8cd7191a8a0d", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0420 11:51:14.456275 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9, hostIfaceName: a8640211b47241c, network: default, NAD default, SandboxID: "a8640211b47241ca56ba7baca041e629c2bcadbe938d04e930480e24981db932", PCI device ID: , UID: "56c1a847-ab94-4167-80a5-d6e67c264caf", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0420 11:51:14.456556 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt, hostIfaceName: 804aa9ca9a40344, network: default, NAD default, SandboxID: "804aa9ca9a40344747c34ffdde201b90cde77b3613c38059b54d4fb9d1131aca", PCI device ID: , UID: "4c8a7129-9a14-4ed4-93ac-37cdc1759c26", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0420 11:51:14.511002 3689 cni.go:473] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt 804aa9ca9a40344747c34ffdde201b90cde77b3613c38059b54d4fb9d1131aca network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt 804aa9ca9a40344747c34ffdde201b90cde77b3613c38059b54d4fb9d1131aca network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:43:75:c7:3e:74\",\"name\":\"804aa9ca9a40344\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0445af1a-7876-4054-a9a1-abce9bdd7b60\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:51:14.741351 3689 cni.go:473] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j a65449ee66c7698bdfe2d1c11398ecbd1ece01ba623429839fdbfde76df05018 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j a65449ee66c7698bdfe2d1c11398ecbd1ece01ba623429839fdbfde76df05018 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:cc:82:3c:72:e9\",\"name\":\"a65449ee66c7698\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/07e387c4-e44e-4de0-b795-9697e8a07b28\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:51:14.741434 3689 cni.go:473] [openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 a8640211b47241ca56ba7baca041e629c2bcadbe938d04e930480e24981db932 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 a8640211b47241ca56ba7baca041e629c2bcadbe938d04e930480e24981db932 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:a0:d9:5e:f0:c0\",\"name\":\"a8640211b47241c\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7be03219-46cd-4ee2-8f3c-0dffefe548ce\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:51:15.281361 3689 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:15.281392 3689 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:15.281411 3689 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:15.281428 3689 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2fqqbd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:21.751812 3689 cni.go:452] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt 804aa9ca9a40344747c34ffdde201b90cde77b3613c38059b54d4fb9d1131aca network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt 804aa9ca9a40344747c34ffdde201b90cde77b3613c38059b54d4fb9d1131aca network default NAD default NAD key ] I0420 11:51:21.752455 3689 cni.go:452] [openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 a8640211b47241ca56ba7baca041e629c2bcadbe938d04e930480e24981db932 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 a8640211b47241ca56ba7baca041e629c2bcadbe938d04e930480e24981db932 network default NAD default NAD key ] I0420 11:51:21.752464 3689 cni.go:452] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j a65449ee66c7698bdfe2d1c11398ecbd1ece01ba623429839fdbfde76df05018 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j a65449ee66c7698bdfe2d1c11398ecbd1ece01ba623429839fdbfde76df05018 network default NAD default NAD key ] I0420 11:51:21.752485 3689 cni.go:452] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r 9bfad672dab9f8d19eaea1bd774e6190b4525792a71f22c8c8e3689a4672f7d0 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r 9bfad672dab9f8d19eaea1bd774e6190b4525792a71f22c8c8e3689a4672f7d0 network default NAD default NAD key ] I0420 11:51:21.827548 3689 cni.go:473] [openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 a8640211b47241ca56ba7baca041e629c2bcadbe938d04e930480e24981db932 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 a8640211b47241ca56ba7baca041e629c2bcadbe938d04e930480e24981db932 network default NAD default NAD key default], result "{}", err I0420 11:51:21.872121 3689 cni.go:473] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt 804aa9ca9a40344747c34ffdde201b90cde77b3613c38059b54d4fb9d1131aca network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt 804aa9ca9a40344747c34ffdde201b90cde77b3613c38059b54d4fb9d1131aca network default NAD default NAD key default], result "{}", err I0420 11:51:21.895718 3689 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:51:21.895782 3689 pods.go:176] Deleting pod: openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 I0420 11:51:21.897228 3689 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9, ips: 10.128.0.34 I0420 11:51:21.897257 3689 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:51:21.897276 3689 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:51:21.897283 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0420 11:51:21.897470 3689 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:51:21.897491 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0420 11:51:21.925591 3689 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:51:21.925643 3689 pods.go:176] Deleting pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt I0420 11:51:21.926989 3689 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt, ips: 10.128.0.35 I0420 11:51:21.927019 3689 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:51:21.927036 3689 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:51:21.927042 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0420 11:51:21.927257 3689 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:51:21.927273 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0420 11:51:21.932051 3689 cni.go:473] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r 9bfad672dab9f8d19eaea1bd774e6190b4525792a71f22c8c8e3689a4672f7d0 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r 9bfad672dab9f8d19eaea1bd774e6190b4525792a71f22c8c8e3689a4672f7d0 network default NAD default NAD key default], result "{}", err I0420 11:51:21.933211 3689 cni.go:473] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j a65449ee66c7698bdfe2d1c11398ecbd1ece01ba623429839fdbfde76df05018 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j a65449ee66c7698bdfe2d1c11398ecbd1ece01ba623429839fdbfde76df05018 network default NAD default NAD key default], result "{}", err I0420 11:51:21.990369 3689 obj_retry.go:498] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:51:21.990430 3689 pods.go:176] Deleting pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r I0420 11:51:21.991851 3689 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r, ips: 10.128.0.33 I0420 11:51:21.991875 3689 obj_retry.go:498] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:51:21.991891 3689 obj_retry.go:498] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:51:21.991919 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0420 11:51:21.992155 3689 obj_retry.go:498] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:51:21.992175 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0420 11:51:22.006332 3689 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:51:22.006363 3689 pods.go:176] Deleting pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j I0420 11:51:22.007440 3689 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j, ips: 10.128.0.36 I0420 11:51:22.007466 3689 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:51:22.007485 3689 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:51:22.007491 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0420 11:51:22.007651 3689 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 11:51:22.007662 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0420 11:51:22.930375 3689 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:22.930402 3689 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:22.930418 3689 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:22.930430 3689 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568wxpn9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:22.962490 3689 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:22.962540 3689 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:22.962559 3689 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:22.962573 3689 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnzqpt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:23.020203 3689 obj_retry.go:492] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:23.020228 3689 obj_retry.go:492] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:23.020245 3689 obj_retry.go:492] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:23.020261 3689 obj_retry.go:492] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5c6v6r of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:23.043727 3689 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:23.043770 3689 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:23.043787 3689 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:51:23.043800 3689 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad75kn4j of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0420 11:51:23.799830 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0420 11:51:26.269695 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0420 11:51:26.279531 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0420 11:51:26.284792 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0420 11:51:26.296485 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0420 11:51:32.299766 3689 namespace.go:142] [openshift-operators] updating namespace I0420 11:51:32.312243 3689 namespace.go:142] [openshift-operators] updating namespace I0420 11:51:39.710611 3689 namespace.go:142] [openshift-operators] updating namespace I0420 11:51:44.522214 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 W0420 11:51:44.908699 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0420 11:51:49.974842 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0420 11:51:50.010689 3689 namespace.go:98] [trusted-artifact-signer] adding namespace I0420 11:51:50.011569 3689 namespace.go:102] [trusted-artifact-signer] adding namespace took 853.703µs I0420 11:51:50.087558 3689 namespace.go:142] [trusted-artifact-signer] updating namespace I0420 11:51:50.113201 3689 namespace.go:142] [trusted-artifact-signer] updating namespace I0420 11:51:53.198431 3689 namespace.go:98] [cert-manager] adding namespace I0420 11:51:53.199266 3689 namespace.go:102] [cert-manager] adding namespace took 815.849µs I0420 11:51:53.298486 3689 namespace.go:142] [cert-manager] updating namespace I0420 11:51:53.322159 3689 namespace.go:142] [cert-manager] updating namespace I0420 11:51:56.438139 3689 namespace.go:98] [openshift-pipelines] adding namespace I0420 11:51:56.438968 3689 namespace.go:102] [openshift-pipelines] adding namespace took 806.447µs I0420 11:51:56.475826 3689 namespace.go:142] [openshift-pipelines] updating namespace I0420 11:51:57.579554 3689 namespace.go:142] [cert-manager] updating namespace I0420 11:51:57.763501 3689 namespace.go:142] [cert-manager-operator] updating namespace I0420 11:51:58.150303 3689 namespace.go:142] [konflux-operator] updating namespace I0420 11:51:58.363627 3689 namespace.go:142] [konflux-ui] updating namespace I0420 11:51:58.561377 3689 namespace.go:142] [open-cluster-management-2pprst5nusibgq03117kjfn2t3hbbnqb] updating namespace I0420 11:51:58.923792 3689 namespace.go:142] [rhbk-operator] updating namespace I0420 11:51:59.125366 3689 namespace.go:142] [rhtpa-operator] updating namespace I0420 11:51:59.325334 3689 namespace.go:142] [trusted-artifact-signer] updating namespace I0420 11:51:59.526021 3689 namespace.go:142] [tsf] updating namespace I0420 11:51:59.731309 3689 namespace.go:142] [tsf-keycloak] updating namespace I0420 11:51:59.945744 3689 namespace.go:142] [tsf-tas] updating namespace W0420 11:52:00.058138 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0420 11:52:00.058155 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0420 11:52:00.127947 3689 namespace.go:142] [tsf-tpa] updating namespace I0420 11:52:00.271132 3689 namespace.go:142] [cert-manager] updating namespace I0420 11:52:00.289347 3689 namespace.go:142] [cert-manager-operator] updating namespace I0420 11:52:00.306313 3689 namespace.go:142] [default] updating namespace I0420 11:52:00.321338 3689 namespace.go:142] [konflux-operator] updating namespace I0420 11:52:00.335193 3689 namespace.go:142] [konflux-ui] updating namespace I0420 11:52:00.350445 3689 namespace.go:142] [open-cluster-management-2pprst5nusibgq03117kjfn2t3hbbnqb] updating namespace I0420 11:52:00.362378 3689 namespace.go:142] [openshift] updating namespace I0420 11:52:00.379096 3689 namespace.go:142] [rhbk-operator] updating namespace I0420 11:52:00.392852 3689 namespace.go:142] [rhtpa-operator] updating namespace I0420 11:52:00.403386 3689 namespace.go:142] [trusted-artifact-signer] updating namespace I0420 11:52:00.416029 3689 namespace.go:142] [tsf] updating namespace I0420 11:52:00.428395 3689 namespace.go:142] [tsf-keycloak] updating namespace I0420 11:52:00.440310 3689 namespace.go:142] [tsf-tas] updating namespace I0420 11:52:00.451925 3689 namespace.go:142] [tsf-tpa] updating namespace I0420 11:52:00.507408 3689 namespace.go:142] [cert-manager] updating namespace I0420 11:52:00.611449 3689 namespace.go:142] [cert-manager-operator] updating namespace I0420 11:52:00.710688 3689 namespace.go:142] [default] updating namespace I0420 11:52:00.807541 3689 namespace.go:142] [konflux-operator] updating namespace I0420 11:52:00.908786 3689 namespace.go:142] [konflux-ui] updating namespace I0420 11:52:01.013305 3689 namespace.go:142] [open-cluster-management-2pprst5nusibgq03117kjfn2t3hbbnqb] updating namespace I0420 11:52:01.109510 3689 namespace.go:142] [openshift] updating namespace I0420 11:52:01.217549 3689 namespace.go:142] [rhbk-operator] updating namespace I0420 11:52:01.309118 3689 namespace.go:142] [rhtpa-operator] updating namespace I0420 11:52:01.408599 3689 namespace.go:142] [trusted-artifact-signer] updating namespace I0420 11:52:01.510824 3689 namespace.go:142] [tsf] updating namespace I0420 11:52:01.620016 3689 namespace.go:142] [tsf-keycloak] updating namespace I0420 11:52:01.714460 3689 namespace.go:142] [tsf-tas] updating namespace I0420 11:52:01.811511 3689 namespace.go:142] [tsf-tpa] updating namespace I0420 11:52:02.383402 3689 namespace.go:142] [openshift-pipelines] updating namespace I0420 11:52:06.511467 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0420 11:52:11.484373 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 W0420 11:52:15.058645 3689 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0420 11:52:15.058665 3689 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0420 11:52:15.058669 3689 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0420 11:52:15.084365 3689 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 11:52:15.084379 3689 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 11:52:15.084384 3689 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 11:52:15.084388 3689 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 11:52:16.157063 3689 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0420 11:52:16.157083 3689 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0420 11:52:16.157099 3689 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0420 11:52:22.079741 3689 cni.go:452] [tsf/test-tsf-subscriptions cb3bb667147916051f63d2d180271f148f347d991967f7fed5186338d7c42444 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-subscriptions cb3bb667147916051f63d2d180271f148f347d991967f7fed5186338d7c42444 network default NAD default NAD key ] I0420 11:52:22.168122 3689 cni.go:473] [tsf/test-tsf-subscriptions cb3bb667147916051f63d2d180271f148f347d991967f7fed5186338d7c42444 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-subscriptions cb3bb667147916051f63d2d180271f148f347d991967f7fed5186338d7c42444 network default NAD default NAD key default], result "{}", err I0420 11:52:22.224536 3689 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:52:22.224569 3689 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0420 11:52:22.225781 3689 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.128.0.30 I0420 11:52:22.225815 3689 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:52:22.253587 3689 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:52:22.253608 3689 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:52:22.258817 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0420 11:52:22.258842 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0420 11:52:23.594426 3689 base_network_controller_pods.go:487] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-1-126.ec2.internal I0420 11:52:23.594508 3689 kube.go:256] Updating pod tsf/test-tsf-infrastructure I0420 11:52:23.608983 3689 pod.go:62] [tsf/test-tsf-infrastructure] pod update took 14.49141ms I0420 11:52:23.609015 3689 base_network_controller_pods.go:951] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 14.522819ms I0420 11:52:23.610577 3689 pods.go:271] [tsf/test-tsf-infrastructure] addLogicalPort took 16.163631ms, libovsdb time 1.05472ms I0420 11:52:23.981727 3689 cni.go:452] [tsf/test-tsf-infrastructure d9cee7832f7dc3cb272967565950487b80caefd5675a78452aa5e251283f85ed network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-infrastructure d9cee7832f7dc3cb272967565950487b80caefd5675a78452aa5e251283f85ed network default NAD default NAD key ] I0420 11:52:23.990953 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: d9cee7832f7dc3c, network: default, NAD default, SandboxID: "d9cee7832f7dc3cb272967565950487b80caefd5675a78452aa5e251283f85ed", PCI device ID: , UID: "32d80143-bb7b-4657-acc5-1b743f2df60e", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0420 11:52:24.244766 3689 cni.go:473] [tsf/test-tsf-infrastructure d9cee7832f7dc3cb272967565950487b80caefd5675a78452aa5e251283f85ed network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-infrastructure d9cee7832f7dc3cb272967565950487b80caefd5675a78452aa5e251283f85ed network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:c9:9d:0c:86:95\",\"name\":\"d9cee7832f7dc3c\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dc2439d7-c319-4743-b8c9-e4c86d7a34f3\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:52:26.480705 3689 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-nsbnj] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-nsbnj for pod on switch ip-10-0-1-126.ec2.internal I0420 11:52:26.480823 3689 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-nsbnj I0420 11:52:26.496276 3689 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-nsbnj] pod update took 15.48494ms I0420 11:52:26.496336 3689 base_network_controller_pods.go:951] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-nsbnj] addLogicalPort annotation time took 15.549665ms I0420 11:52:26.496883 3689 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-dn7bd] creating logical port tsf-keycloak_keycloak-pgsql-bee-744f6fc4d9-dn7bd for pod on switch ip-10-0-1-126.ec2.internal I0420 11:52:26.496975 3689 kube.go:256] Updating pod tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-dn7bd I0420 11:52:26.497774 3689 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-nsbnj] addLogicalPort took 17.092985ms, libovsdb time 993.646µs I0420 11:52:26.509981 3689 pod.go:62] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-dn7bd] pod update took 13.029376ms I0420 11:52:26.510103 3689 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-dn7bd] addLogicalPort annotation time took 13.106722ms I0420 11:52:26.511699 3689 pods.go:271] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-dn7bd] addLogicalPort took 14.831042ms, libovsdb time 1.147745ms W0420 11:52:37.168315 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0420 11:52:37.362800 3689 cni.go:452] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-nsbnj 4e91908ee7e580a07f85bd48a35a407f0b4e66b8cd1fbc202daf810783c84503 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-nsbnj 4e91908ee7e580a07f85bd48a35a407f0b4e66b8cd1fbc202daf810783c84503 network default NAD default NAD key ] I0420 11:52:37.373808 3689 cni.go:452] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-dn7bd 6d1443bec4ec02b3e41b9dd63e133455bf966315b1f8b8b7e8736c484293d0ae network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-dn7bd 6d1443bec4ec02b3e41b9dd63e133455bf966315b1f8b8b7e8736c484293d0ae network default NAD default NAD key ] I0420 11:52:37.374663 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-nsbnj, hostIfaceName: 4e91908ee7e580a, network: default, NAD default, SandboxID: "4e91908ee7e580a07f85bd48a35a407f0b4e66b8cd1fbc202daf810783c84503", PCI device ID: , UID: "1e72735a-f0b2-4b7b-a437-ae52a343438f", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0420 11:52:37.396935 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-pgsql-bee-744f6fc4d9-dn7bd, hostIfaceName: 6d1443bec4ec02b, network: default, NAD default, SandboxID: "6d1443bec4ec02b3e41b9dd63e133455bf966315b1f8b8b7e8736c484293d0ae", PCI device ID: , UID: "320c0734-001d-4dc7-99fd-9bcd0fe9ded6", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0420 11:52:37.431584 3689 cni.go:473] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-nsbnj 4e91908ee7e580a07f85bd48a35a407f0b4e66b8cd1fbc202daf810783c84503 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-nsbnj 4e91908ee7e580a07f85bd48a35a407f0b4e66b8cd1fbc202daf810783c84503 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:4a:17:b2:ee:7e\",\"name\":\"4e91908ee7e580a\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/902ce1cd-212f-429d-aeb9-26aaf8a2a3fd\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:52:37.453276 3689 cni.go:473] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-dn7bd 6d1443bec4ec02b3e41b9dd63e133455bf966315b1f8b8b7e8736c484293d0ae network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-dn7bd 6d1443bec4ec02b3e41b9dd63e133455bf966315b1f8b8b7e8736c484293d0ae network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:93:61:8c:af:b7\",\"name\":\"6d1443bec4ec02b\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6b5c98af-75f4-411b-b5ac-6e9b68a426a0\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:52:38.171626 3689 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0420 11:52:38.196625 3689 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0420 11:52:50.032120 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-l4vmv] creating logical port openshift-pipelines_tekton-chains-controller-5fb999ffff-l4vmv for pod on switch ip-10-0-1-126.ec2.internal I0420 11:52:50.032250 3689 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-5fb999ffff-l4vmv I0420 11:52:50.050061 3689 pod.go:62] [openshift-pipelines/tekton-chains-controller-5fb999ffff-l4vmv] pod update took 17.835034ms I0420 11:52:50.050085 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-l4vmv] addLogicalPort annotation time took 17.863275ms I0420 11:52:50.051421 3689 pods.go:271] [openshift-pipelines/tekton-chains-controller-5fb999ffff-l4vmv] addLogicalPort took 19.322893ms, libovsdb time 929.553µs I0420 11:52:50.416439 3689 cni.go:452] [openshift-pipelines/tekton-chains-controller-5fb999ffff-l4vmv e35010f595a245677894fde21893d35974beb9306404195c85ca56baea5c3fff network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-l4vmv e35010f595a245677894fde21893d35974beb9306404195c85ca56baea5c3fff network default NAD default NAD key ] I0420 11:52:50.425693 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-5fb999ffff-l4vmv, hostIfaceName: e35010f595a2456, network: default, NAD default, SandboxID: "e35010f595a245677894fde21893d35974beb9306404195c85ca56baea5c3fff", PCI device ID: , UID: "9f1768c8-9281-429b-afa7-1b4863b1d08b", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0420 11:52:50.476097 3689 cni.go:473] [openshift-pipelines/tekton-chains-controller-5fb999ffff-l4vmv e35010f595a245677894fde21893d35974beb9306404195c85ca56baea5c3fff network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-l4vmv e35010f595a245677894fde21893d35974beb9306404195c85ca56baea5c3fff network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:ee:5b:8b:51:85\",\"name\":\"e35010f595a2456\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/37497fa2-d42c-40f4-8806-a0a21f85029e\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:52:57.238162 3689 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0420 11:52:59.190454 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0420 11:52:59.207222 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 5432: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0420 11:53:01.310114 3689 cni.go:452] [tsf/test-tsf-infrastructure d9cee7832f7dc3cb272967565950487b80caefd5675a78452aa5e251283f85ed network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-infrastructure d9cee7832f7dc3cb272967565950487b80caefd5675a78452aa5e251283f85ed network default NAD default NAD key ] I0420 11:53:01.383589 3689 cni.go:473] [tsf/test-tsf-infrastructure d9cee7832f7dc3cb272967565950487b80caefd5675a78452aa5e251283f85ed network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-infrastructure d9cee7832f7dc3cb272967565950487b80caefd5675a78452aa5e251283f85ed network default NAD default NAD key default], result "{}", err I0420 11:53:01.407738 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=262 I0420 11:53:01.435524 3689 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:53:01.435559 3689 pods.go:176] Deleting pod: tsf/test-tsf-infrastructure I0420 11:53:01.437000 3689 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.128.0.37 I0420 11:53:01.437037 3689 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:53:01.481033 3689 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:53:01.481057 3689 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:53:01.485351 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0420 11:53:01.485376 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0420 11:53:02.601916 3689 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-1-126.ec2.internal I0420 11:53:02.602004 3689 kube.go:256] Updating pod tsf/test-tsf-iam I0420 11:53:02.617551 3689 pod.go:62] [tsf/test-tsf-iam] pod update took 15.570671ms I0420 11:53:02.617571 3689 base_network_controller_pods.go:951] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 15.593193ms I0420 11:53:02.618796 3689 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 16.91588ms, libovsdb time 885.128µs I0420 11:53:02.963094 3689 cni.go:452] [tsf/test-tsf-iam e17d7ae7d71748cab0b5d920bc9515c5b89dd5941ce9310f6edc45752d13887c network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-iam e17d7ae7d71748cab0b5d920bc9515c5b89dd5941ce9310f6edc45752d13887c network default NAD default NAD key ] I0420 11:53:02.972285 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: e17d7ae7d71748c, network: default, NAD default, SandboxID: "e17d7ae7d71748cab0b5d920bc9515c5b89dd5941ce9310f6edc45752d13887c", PCI device ID: , UID: "f5027aaa-0cef-44fa-b9c4-761c8178642a", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0420 11:53:03.022230 3689 cni.go:473] [tsf/test-tsf-iam e17d7ae7d71748cab0b5d920bc9515c5b89dd5941ce9310f6edc45752d13887c network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-iam e17d7ae7d71748cab0b5d920bc9515c5b89dd5941ce9310f6edc45752d13887c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:73:33:fa:8c:c6\",\"name\":\"e17d7ae7d71748c\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/447d4840-fcfe-478c-8983-28f7ea226a57\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:53:03.578145 3689 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0420 11:53:03.578947 3689 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0420 11:53:03.578996 3689 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0420 11:53:03.581316 3689 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0420 11:53:03.949359 3689 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:53:03.949481 3689 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0420 11:53:03.965441 3689 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 15.998745ms I0420 11:53:03.965466 3689 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 16.028223ms I0420 11:53:03.966822 3689 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 17.483352ms, libovsdb time 840.094µs I0420 11:53:04.310455 3689 cni.go:452] [tsf-keycloak/keycloak-0 508bb5f36b3bffb9a1cf39a31ceeb761787cc2f5b93aa859a8f32e7a32b6ef21 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-0 508bb5f36b3bffb9a1cf39a31ceeb761787cc2f5b93aa859a8f32e7a32b6ef21 network default NAD default NAD key ] I0420 11:53:04.320206 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 508bb5f36b3bffb, network: default, NAD default, SandboxID: "508bb5f36b3bffb9a1cf39a31ceeb761787cc2f5b93aa859a8f32e7a32b6ef21", PCI device ID: , UID: "a6a65ba7-5c23-4a01-b0c1-b24c92bc8bcc", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0420 11:53:04.374015 3689 cni.go:473] [tsf-keycloak/keycloak-0 508bb5f36b3bffb9a1cf39a31ceeb761787cc2f5b93aa859a8f32e7a32b6ef21 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-0 508bb5f36b3bffb9a1cf39a31ceeb761787cc2f5b93aa859a8f32e7a32b6ef21 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:b3:62:fa:99:e6\",\"name\":\"508bb5f36b3bffb\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67729bb9-ffa4-4785-afb5-dd0a641859e1\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:53:09.374762 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0420 11:53:09.374783 3689 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0420 11:53:09.377875 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0420 11:53:09.377977 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0420 11:53:22.432542 3689 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0420 11:53:22.432560 3689 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0420 11:53:22.432565 3689 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0420 11:53:32.573755 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0420 11:53:32.573775 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0420 11:53:32.576274 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0420 11:53:32.576296 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9000: empty IP address endpoints for service tsf-keycloak/keycloak-service I0420 11:53:33.588333 3689 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-cn9wh] creating logical port tsf-keycloak_tsf-iam-cn9wh for pod on switch ip-10-0-1-126.ec2.internal I0420 11:53:33.588438 3689 kube.go:256] Updating pod tsf-keycloak/tsf-iam-cn9wh I0420 11:53:33.609277 3689 pod.go:62] [tsf-keycloak/tsf-iam-cn9wh] pod update took 20.868397ms I0420 11:53:33.609298 3689 base_network_controller_pods.go:951] [default/tsf-keycloak/tsf-iam-cn9wh] addLogicalPort annotation time took 20.892385ms I0420 11:53:33.610527 3689 pods.go:271] [tsf-keycloak/tsf-iam-cn9wh] addLogicalPort took 22.214036ms, libovsdb time 831.763µs I0420 11:53:33.947970 3689 cni.go:452] [tsf-keycloak/tsf-iam-cn9wh 882b064c9e43bd4bd92341f96b7098b49da34fa3e52ff59ff7e10b36b6a3a2f6 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/tsf-iam-cn9wh 882b064c9e43bd4bd92341f96b7098b49da34fa3e52ff59ff7e10b36b6a3a2f6 network default NAD default NAD key ] I0420 11:53:33.956929 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-cn9wh, hostIfaceName: 882b064c9e43bd4, network: default, NAD default, SandboxID: "882b064c9e43bd4bd92341f96b7098b49da34fa3e52ff59ff7e10b36b6a3a2f6", PCI device ID: , UID: "d2a8fda0-93fd-41ff-94f2-71dec64c0f99", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0420 11:53:34.005852 3689 cni.go:473] [tsf-keycloak/tsf-iam-cn9wh 882b064c9e43bd4bd92341f96b7098b49da34fa3e52ff59ff7e10b36b6a3a2f6 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/tsf-iam-cn9wh 882b064c9e43bd4bd92341f96b7098b49da34fa3e52ff59ff7e10b36b6a3a2f6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:47:f5:0c:e1:5c\",\"name\":\"882b064c9e43bd4\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5af8fdc2-6993-4821-a46c-2ec92f6460ec\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:53:35.555210 3689 cni.go:452] [tsf/test-tsf-iam e17d7ae7d71748cab0b5d920bc9515c5b89dd5941ce9310f6edc45752d13887c network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-iam e17d7ae7d71748cab0b5d920bc9515c5b89dd5941ce9310f6edc45752d13887c network default NAD default NAD key ] I0420 11:53:35.646582 3689 cni.go:473] [tsf/test-tsf-iam e17d7ae7d71748cab0b5d920bc9515c5b89dd5941ce9310f6edc45752d13887c network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-iam e17d7ae7d71748cab0b5d920bc9515c5b89dd5941ce9310f6edc45752d13887c network default NAD default NAD key default], result "{}", err I0420 11:53:35.703785 3689 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:53:35.703828 3689 pods.go:176] Deleting pod: tsf/test-tsf-iam I0420 11:53:35.705154 3689 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.128.0.41 I0420 11:53:35.705185 3689 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:53:35.741139 3689 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:53:35.741166 3689 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:53:35.741220 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0420 11:53:35.741248 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0420 11:53:36.576330 3689 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-1-126.ec2.internal I0420 11:53:36.576446 3689 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0420 11:53:36.591397 3689 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 14.985506ms I0420 11:53:36.591423 3689 base_network_controller_pods.go:951] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 15.018607ms I0420 11:53:36.592676 3689 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 16.374985ms, libovsdb time 894.144µs I0420 11:53:36.937022 3689 cni.go:452] [tsf-tas/test-tsf-tas 850fb5a228e253971cf4c0698ca7c91f49b5bc25877421c627e7a35e880285ee network default NAD default NAD key ] ADD starting CNI request [tsf-tas/test-tsf-tas 850fb5a228e253971cf4c0698ca7c91f49b5bc25877421c627e7a35e880285ee network default NAD default NAD key ] I0420 11:53:36.949750 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: 850fb5a228e2539, network: default, NAD default, SandboxID: "850fb5a228e253971cf4c0698ca7c91f49b5bc25877421c627e7a35e880285ee", PCI device ID: , UID: "e3c011e4-f846-4ce1-81c5-3a02aedced34", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0420 11:53:37.002402 3689 cni.go:473] [tsf-tas/test-tsf-tas 850fb5a228e253971cf4c0698ca7c91f49b5bc25877421c627e7a35e880285ee network default NAD default NAD key default] ADD finished CNI request [tsf-tas/test-tsf-tas 850fb5a228e253971cf4c0698ca7c91f49b5bc25877421c627e7a35e880285ee network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:80:3b:01:d5:7d\",\"name\":\"850fb5a228e2539\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fa5338d3-cca0-4fb8-8a2f-da57b441f175\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:53:46.956768 3689 namespace.go:142] [openshift-pipelines] updating namespace W0420 11:53:48.547097 3689 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0420 11:53:51.671143 3689 cni.go:452] [tsf-keycloak/tsf-iam-cn9wh 882b064c9e43bd4bd92341f96b7098b49da34fa3e52ff59ff7e10b36b6a3a2f6 network default NAD default NAD key ] DEL starting CNI request [tsf-keycloak/tsf-iam-cn9wh 882b064c9e43bd4bd92341f96b7098b49da34fa3e52ff59ff7e10b36b6a3a2f6 network default NAD default NAD key ] I0420 11:53:51.757321 3689 cni.go:473] [tsf-keycloak/tsf-iam-cn9wh 882b064c9e43bd4bd92341f96b7098b49da34fa3e52ff59ff7e10b36b6a3a2f6 network default NAD default NAD key default] DEL finished CNI request [tsf-keycloak/tsf-iam-cn9wh 882b064c9e43bd4bd92341f96b7098b49da34fa3e52ff59ff7e10b36b6a3a2f6 network default NAD default NAD key default], result "{}", err I0420 11:53:51.808828 3689 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-cn9wh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:53:51.808873 3689 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-cn9wh I0420 11:53:51.810217 3689 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-cn9wh, ips: 10.128.0.43 I0420 11:53:51.810244 3689 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-cn9wh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:53:52.701642 3689 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-cn9wh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:53:52.701684 3689 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-cn9wh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0420 11:53:55.607017 3689 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0420 11:53:58.591127 3689 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0420 11:54:05.781430 3689 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-h5v7r-spphc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:54:05.782037 3689 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-h5v7r-spphc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:54:06.812832 3689 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-h5v7r-spphc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:06.812858 3689 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-h5v7r-spphc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:09.315062 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bhwdd] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-bhwdd for pod on switch ip-10-0-1-126.ec2.internal I0420 11:54:09.315152 3689 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bhwdd I0420 11:54:09.329685 3689 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bhwdd] pod update took 14.556969ms I0420 11:54:09.329710 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bhwdd] addLogicalPort annotation time took 14.587298ms I0420 11:54:09.330958 3689 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bhwdd] addLogicalPort took 15.914114ms, libovsdb time 842.024µs I0420 11:54:09.408554 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-45mvw] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-45mvw for pod on switch ip-10-0-1-126.ec2.internal I0420 11:54:09.408666 3689 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-45mvw I0420 11:54:09.422695 3689 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-45mvw] pod update took 14.061348ms I0420 11:54:09.422722 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-45mvw] addLogicalPort annotation time took 14.092576ms I0420 11:54:09.424015 3689 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-45mvw] addLogicalPort took 15.477194ms, libovsdb time 852.643µs I0420 11:54:09.528357 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-kkbcp] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-kkbcp for pod on switch ip-10-0-1-126.ec2.internal I0420 11:54:09.528499 3689 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-kkbcp I0420 11:54:09.542568 3689 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-kkbcp] pod update took 14.100734ms I0420 11:54:09.542593 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-kkbcp] addLogicalPort annotation time took 14.130437ms I0420 11:54:09.544042 3689 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-kkbcp] addLogicalPort took 15.705914ms, libovsdb time 1.008384ms I0420 11:54:09.676479 3689 cni.go:452] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bhwdd 1de7eedaed30b089294ec1a90f18463fd1e2388831798df581333c30f1fbe041 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bhwdd 1de7eedaed30b089294ec1a90f18463fd1e2388831798df581333c30f1fbe041 network default NAD default NAD key ] I0420 11:54:09.685884 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-bhwdd, hostIfaceName: 1de7eedaed30b08, network: default, NAD default, SandboxID: "1de7eedaed30b089294ec1a90f18463fd1e2388831798df581333c30f1fbe041", PCI device ID: , UID: "f9e8b3db-7e3f-43c9-8387-1cda70a35322", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0420 11:54:09.738377 3689 cni.go:473] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bhwdd 1de7eedaed30b089294ec1a90f18463fd1e2388831798df581333c30f1fbe041 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bhwdd 1de7eedaed30b089294ec1a90f18463fd1e2388831798df581333c30f1fbe041 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:ea:29:5e:a6:7d\",\"name\":\"1de7eedaed30b08\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/78e94ec0-e7ea-40fd-b954-54386c796555\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:54:09.772553 3689 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-45mvw 3b54d6b01f86e24cde12b540ed57988954d0e0e823037cfc43168c8542cd45e6 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-45mvw 3b54d6b01f86e24cde12b540ed57988954d0e0e823037cfc43168c8542cd45e6 network default NAD default NAD key ] I0420 11:54:09.781802 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-45mvw, hostIfaceName: 3b54d6b01f86e24, network: default, NAD default, SandboxID: "3b54d6b01f86e24cde12b540ed57988954d0e0e823037cfc43168c8542cd45e6", PCI device ID: , UID: "90d4d069-0e77-4e32-89b9-46e49a6e279c", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0420 11:54:09.830893 3689 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-45mvw 3b54d6b01f86e24cde12b540ed57988954d0e0e823037cfc43168c8542cd45e6 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-45mvw 3b54d6b01f86e24cde12b540ed57988954d0e0e823037cfc43168c8542cd45e6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:6d:a4:0c:3f:a5\",\"name\":\"3b54d6b01f86e24\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e4a85329-44d9-4297-af92-b1083b5468fc\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:54:09.893065 3689 cni.go:452] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-kkbcp b8be735b9b182763afc9937d5947945f35748f8c2d24a45f8bb6c422a2ad69b5 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-kkbcp b8be735b9b182763afc9937d5947945f35748f8c2d24a45f8bb6c422a2ad69b5 network default NAD default NAD key ] I0420 11:54:09.902534 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-kkbcp, hostIfaceName: b8be735b9b18276, network: default, NAD default, SandboxID: "b8be735b9b182763afc9937d5947945f35748f8c2d24a45f8bb6c422a2ad69b5", PCI device ID: , UID: "86b8c3c7-5368-4508-9106-3e5c44d52984", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0420 11:54:09.950391 3689 cni.go:473] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-kkbcp b8be735b9b182763afc9937d5947945f35748f8c2d24a45f8bb6c422a2ad69b5 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-kkbcp b8be735b9b182763afc9937d5947945f35748f8c2d24a45f8bb6c422a2ad69b5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:49:03:e5:26:09\",\"name\":\"b8be735b9b18276\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a1cdf980-19c5-47ec-b32b-e8f5fe3c5dd8\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:54:13.522297 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 W0420 11:54:13.798378 3689 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0420 11:54:14.487642 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 W0420 11:54:24.810088 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0420 11:54:30.900381 3689 cni.go:452] [tsf-tas/test-tsf-tas 850fb5a228e253971cf4c0698ca7c91f49b5bc25877421c627e7a35e880285ee network default NAD default NAD key ] DEL starting CNI request [tsf-tas/test-tsf-tas 850fb5a228e253971cf4c0698ca7c91f49b5bc25877421c627e7a35e880285ee network default NAD default NAD key ] I0420 11:54:30.980109 3689 cni.go:473] [tsf-tas/test-tsf-tas 850fb5a228e253971cf4c0698ca7c91f49b5bc25877421c627e7a35e880285ee network default NAD default NAD key default] DEL finished CNI request [tsf-tas/test-tsf-tas 850fb5a228e253971cf4c0698ca7c91f49b5bc25877421c627e7a35e880285ee network default NAD default NAD key default], result "{}", err I0420 11:54:31.031892 3689 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:54:31.031944 3689 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0420 11:54:31.033264 3689 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.128.0.44 I0420 11:54:31.033297 3689 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:54:31.056549 3689 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:31.056566 3689 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:31.065861 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0420 11:54:31.065890 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0420 11:54:42.998182 3689 obj_retry.go:498] Detected object tsf/patch-tekton-config-ccrnf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:54:42.998729 3689 obj_retry.go:498] Detected object tsf/patch-tekton-config-ccrnf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:54:43.883043 3689 obj_retry.go:492] Detected object tsf/patch-tekton-config-ccrnf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:43.883065 3689 obj_retry.go:492] Detected object tsf/patch-tekton-config-ccrnf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:43.991819 3689 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-tqqb6] creating logical port tsf_tsf-tekton-configuration-tqqb6 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:54:43.991991 3689 kube.go:256] Updating pod tsf/tsf-tekton-configuration-tqqb6 I0420 11:54:44.004995 3689 pod.go:62] [tsf/tsf-tekton-configuration-tqqb6] pod update took 13.032436ms I0420 11:54:44.005017 3689 base_network_controller_pods.go:951] [default/tsf/tsf-tekton-configuration-tqqb6] addLogicalPort annotation time took 13.06446ms I0420 11:54:44.006365 3689 pods.go:271] [tsf/tsf-tekton-configuration-tqqb6] addLogicalPort took 14.566454ms, libovsdb time 949.786µs I0420 11:54:44.015269 3689 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-l4mw9-qbz8d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:54:44.015755 3689 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-l4mw9-qbz8d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:54:44.351177 3689 cni.go:452] [tsf/tsf-tekton-configuration-tqqb6 8381d2baa8f8cc0dd67051d98adb893ee2be73923058701cb4305920d49a0b69 network default NAD default NAD key ] ADD starting CNI request [tsf/tsf-tekton-configuration-tqqb6 8381d2baa8f8cc0dd67051d98adb893ee2be73923058701cb4305920d49a0b69 network default NAD default NAD key ] I0420 11:54:44.360342 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-tqqb6, hostIfaceName: 8381d2baa8f8cc0, network: default, NAD default, SandboxID: "8381d2baa8f8cc0dd67051d98adb893ee2be73923058701cb4305920d49a0b69", PCI device ID: , UID: "db4e4e11-5897-4afb-8c39-32afdf5f4c51", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0420 11:54:44.410874 3689 cni.go:473] [tsf/tsf-tekton-configuration-tqqb6 8381d2baa8f8cc0dd67051d98adb893ee2be73923058701cb4305920d49a0b69 network default NAD default NAD key default] ADD finished CNI request [tsf/tsf-tekton-configuration-tqqb6 8381d2baa8f8cc0dd67051d98adb893ee2be73923058701cb4305920d49a0b69 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:69:0a:5b:91:4f\",\"name\":\"8381d2baa8f8cc0\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/60591420-7b91-4ace-92e5-eb34be2f1a6e\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:54:44.890977 3689 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-l4mw9-qbz8d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:44.891005 3689 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-l4mw9-qbz8d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:46.312051 3689 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-88fc5cd64-fk769] creating logical port tsf-tas_rekor-redis-88fc5cd64-fk769 for pod on switch ip-10-0-1-126.ec2.internal I0420 11:54:46.312178 3689 kube.go:256] Updating pod tsf-tas/rekor-redis-88fc5cd64-fk769 I0420 11:54:46.326499 3689 pod.go:62] [tsf-tas/rekor-redis-88fc5cd64-fk769] pod update took 14.355802ms I0420 11:54:46.326540 3689 base_network_controller_pods.go:951] [default/tsf-tas/rekor-redis-88fc5cd64-fk769] addLogicalPort annotation time took 14.397242ms I0420 11:54:46.328490 3689 pods.go:271] [tsf-tas/rekor-redis-88fc5cd64-fk769] addLogicalPort took 16.498604ms, libovsdb time 1.213577ms I0420 11:54:48.178071 3689 cni.go:452] [tsf-tas/rekor-redis-88fc5cd64-fk769 b748631a840951bdb224772268f0453ed11bc73da2192766b35c5bf085cb0606 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-redis-88fc5cd64-fk769 b748631a840951bdb224772268f0453ed11bc73da2192766b35c5bf085cb0606 network default NAD default NAD key ] I0420 11:54:48.187405 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-88fc5cd64-fk769, hostIfaceName: b748631a840951b, network: default, NAD default, SandboxID: "b748631a840951bdb224772268f0453ed11bc73da2192766b35c5bf085cb0606", PCI device ID: , UID: "32845c9c-1c0a-4734-a2ce-988cc2e8ca10", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0420 11:54:48.240395 3689 cni.go:473] [tsf-tas/rekor-redis-88fc5cd64-fk769 b748631a840951bdb224772268f0453ed11bc73da2192766b35c5bf085cb0606 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-redis-88fc5cd64-fk769 b748631a840951bdb224772268f0453ed11bc73da2192766b35c5bf085cb0606 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:d2:5b:43:f0:60\",\"name\":\"b748631a840951b\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1e6c5710-ab91-4eb0-abea-a1aa4a48b542\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:54:49.864436 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0420 11:54:50.019628 3689 cni.go:452] [tsf/tsf-tekton-configuration-tqqb6 8381d2baa8f8cc0dd67051d98adb893ee2be73923058701cb4305920d49a0b69 network default NAD default NAD key ] DEL starting CNI request [tsf/tsf-tekton-configuration-tqqb6 8381d2baa8f8cc0dd67051d98adb893ee2be73923058701cb4305920d49a0b69 network default NAD default NAD key ] I0420 11:54:50.110047 3689 cni.go:473] [tsf/tsf-tekton-configuration-tqqb6 8381d2baa8f8cc0dd67051d98adb893ee2be73923058701cb4305920d49a0b69 network default NAD default NAD key default] DEL finished CNI request [tsf/tsf-tekton-configuration-tqqb6 8381d2baa8f8cc0dd67051d98adb893ee2be73923058701cb4305920d49a0b69 network default NAD default NAD key default], result "{}", err I0420 11:54:50.162360 3689 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-tqqb6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:54:50.162389 3689 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-tqqb6 I0420 11:54:50.163810 3689 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-tqqb6, ips: 10.128.0.48 I0420 11:54:50.163841 3689 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-tqqb6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:54:51.186483 3689 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-tqqb6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:51.186503 3689 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-tqqb6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:51.232428 3689 obj_retry.go:492] Detected object tsf/patch-tekton-config-ccrnf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:51.232451 3689 obj_retry.go:492] Detected object tsf/patch-tekton-config-ccrnf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:51.252271 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-ccrnf I0420 11:54:51.252300 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-ccrnf I0420 11:54:51.835978 3689 base_network_controller_pods.go:487] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-1-126.ec2.internal I0420 11:54:51.836084 3689 kube.go:256] Updating pod tsf-tpa/pre-install-tsf-tpa I0420 11:54:51.850470 3689 pod.go:62] [tsf-tpa/pre-install-tsf-tpa] pod update took 14.424318ms I0420 11:54:51.850497 3689 base_network_controller_pods.go:951] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 14.452797ms I0420 11:54:51.851849 3689 pods.go:271] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 15.887367ms, libovsdb time 953.076µs I0420 11:54:52.748306 3689 cni.go:452] [tsf-tpa/pre-install-tsf-tpa 72db490e3e456be4419485d698d55d65af560d2da002774201347daee4950c00 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa 72db490e3e456be4419485d698d55d65af560d2da002774201347daee4950c00 network default NAD default NAD key ] I0420 11:54:52.757561 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: 72db490e3e456be, network: default, NAD default, SandboxID: "72db490e3e456be4419485d698d55d65af560d2da002774201347daee4950c00", PCI device ID: , UID: "8b47e8f6-a96c-4f06-9285-bc4e8374fd3e", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0420 11:54:52.809299 3689 cni.go:473] [tsf-tpa/pre-install-tsf-tpa 72db490e3e456be4419485d698d55d65af560d2da002774201347daee4950c00 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa 72db490e3e456be4419485d698d55d65af560d2da002774201347daee4950c00 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:3a:56:9a:d2:45\",\"name\":\"72db490e3e456be\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/87abe39f-1eeb-4b86-9de0-ccdc2d83b66b\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:54:53.043613 3689 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0420 11:54:56.064679 3689 cni.go:452] [tsf-tpa/pre-install-tsf-tpa 72db490e3e456be4419485d698d55d65af560d2da002774201347daee4950c00 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa 72db490e3e456be4419485d698d55d65af560d2da002774201347daee4950c00 network default NAD default NAD key ] I0420 11:54:56.149835 3689 cni.go:473] [tsf-tpa/pre-install-tsf-tpa 72db490e3e456be4419485d698d55d65af560d2da002774201347daee4950c00 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa 72db490e3e456be4419485d698d55d65af560d2da002774201347daee4950c00 network default NAD default NAD key default], result "{}", err I0420 11:54:56.204809 3689 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:54:56.204843 3689 pods.go:176] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0420 11:54:56.206146 3689 pods.go:236] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.128.0.50 I0420 11:54:56.206175 3689 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:54:56.227562 3689 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:56.227585 3689 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:54:56.237725 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0420 11:54:56.237751 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0420 11:54:56.485400 3689 base_network_controller_pods.go:487] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-1-126.ec2.internal I0420 11:54:56.485496 3689 kube.go:256] Updating pod tsf-tpa/test-tsf-tpa I0420 11:54:56.500688 3689 pod.go:62] [tsf-tpa/test-tsf-tpa] pod update took 15.209543ms I0420 11:54:56.500710 3689 base_network_controller_pods.go:951] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 15.241332ms I0420 11:54:56.501884 3689 pods.go:271] [tsf-tpa/test-tsf-tpa] addLogicalPort took 16.498418ms, libovsdb time 757.172µs I0420 11:54:56.845119 3689 cni.go:452] [tsf-tpa/test-tsf-tpa 2bb7bcd40e06e40a33458cd757e3365d6ef735b71c188baeb0eb47823b75f203 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/test-tsf-tpa 2bb7bcd40e06e40a33458cd757e3365d6ef735b71c188baeb0eb47823b75f203 network default NAD default NAD key ] I0420 11:54:56.854422 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: 2bb7bcd40e06e40, network: default, NAD default, SandboxID: "2bb7bcd40e06e40a33458cd757e3365d6ef735b71c188baeb0eb47823b75f203", PCI device ID: , UID: "20714db1-f468-4db8-b656-49670dc59b57", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0420 11:54:56.904155 3689 cni.go:473] [tsf-tpa/test-tsf-tpa 2bb7bcd40e06e40a33458cd757e3365d6ef735b71c188baeb0eb47823b75f203 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/test-tsf-tpa 2bb7bcd40e06e40a33458cd757e3365d6ef735b71c188baeb0eb47823b75f203 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:b3:9c:ba:52:c2\",\"name\":\"2bb7bcd40e06e40\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6565fd86-b2b2-4c24-8568-fe60dde3bbb3\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:54:57.611893 3689 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-wbkjp] creating logical port tsf-tpa_create-db-wbkjp for pod on switch ip-10-0-1-126.ec2.internal I0420 11:54:57.612014 3689 kube.go:256] Updating pod tsf-tpa/create-db-wbkjp I0420 11:54:57.625218 3689 pod.go:62] [tsf-tpa/create-db-wbkjp] pod update took 13.235281ms I0420 11:54:57.625242 3689 base_network_controller_pods.go:951] [default/tsf-tpa/create-db-wbkjp] addLogicalPort annotation time took 13.263015ms I0420 11:54:57.626603 3689 pods.go:271] [tsf-tpa/create-db-wbkjp] addLogicalPort took 14.728625ms, libovsdb time 960.211µs W0420 11:54:57.945310 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0420 11:54:57.976302 3689 cni.go:452] [tsf-tpa/create-db-wbkjp 44d99a6909fb63dab0fda72a26afbe1107883a715231363c9e5d7501b2674e81 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-db-wbkjp 44d99a6909fb63dab0fda72a26afbe1107883a715231363c9e5d7501b2674e81 network default NAD default NAD key ] I0420 11:54:57.997957 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-db-wbkjp, hostIfaceName: 44d99a6909fb63d, network: default, NAD default, SandboxID: "44d99a6909fb63dab0fda72a26afbe1107883a715231363c9e5d7501b2674e81", PCI device ID: , UID: "494054ec-dc37-4e99-9a11-39eebdba7e3c", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0420 11:54:58.048717 3689 cni.go:473] [tsf-tpa/create-db-wbkjp 44d99a6909fb63dab0fda72a26afbe1107883a715231363c9e5d7501b2674e81 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-db-wbkjp 44d99a6909fb63dab0fda72a26afbe1107883a715231363c9e5d7501b2674e81 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:c3:9c:9a:e6:2b\",\"name\":\"44d99a6909fb63d\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a4ab674c-7efc-4874-8d3c-587aaedbdba4\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:55:02.484851 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=11 I0420 11:55:03.116171 3689 cni.go:452] [tsf-tpa/create-db-wbkjp 44d99a6909fb63dab0fda72a26afbe1107883a715231363c9e5d7501b2674e81 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-db-wbkjp 44d99a6909fb63dab0fda72a26afbe1107883a715231363c9e5d7501b2674e81 network default NAD default NAD key ] I0420 11:55:03.187267 3689 cni.go:473] [tsf-tpa/create-db-wbkjp 44d99a6909fb63dab0fda72a26afbe1107883a715231363c9e5d7501b2674e81 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-db-wbkjp 44d99a6909fb63dab0fda72a26afbe1107883a715231363c9e5d7501b2674e81 network default NAD default NAD key default], result "{}", err I0420 11:55:03.242050 3689 obj_retry.go:498] Detected object tsf-tpa/create-db-wbkjp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:55:03.242085 3689 pods.go:176] Deleting pod: tsf-tpa/create-db-wbkjp I0420 11:55:03.243547 3689 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-wbkjp, ips: 10.128.0.52 I0420 11:55:03.243575 3689 obj_retry.go:498] Detected object tsf-tpa/create-db-wbkjp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:55:04.269441 3689 obj_retry.go:492] Detected object tsf-tpa/create-db-wbkjp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:55:04.269472 3689 obj_retry.go:492] Detected object tsf-tpa/create-db-wbkjp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0420 11:55:05.085510 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis I0420 11:55:10.403197 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=66 I0420 11:55:11.257948 3689 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-tm7r8-lcvqj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:55:11.258480 3689 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-tm7r8-lcvqj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:55:12.029222 3689 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-tm7r8-lcvqj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:55:12.029246 3689 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-tm7r8-lcvqj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:55:14.331348 3689 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-fb874484-87r7x] creating logical port tsf-tas_ctlog-fb874484-87r7x for pod on switch ip-10-0-1-126.ec2.internal I0420 11:55:14.331465 3689 kube.go:256] Updating pod tsf-tas/ctlog-fb874484-87r7x I0420 11:55:14.353012 3689 pod.go:62] [tsf-tas/ctlog-fb874484-87r7x] pod update took 21.567056ms I0420 11:55:14.353040 3689 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-fb874484-87r7x] addLogicalPort annotation time took 21.598913ms I0420 11:55:14.354566 3689 pods.go:271] [tsf-tas/ctlog-fb874484-87r7x] addLogicalPort took 23.23755ms, libovsdb time 996.324µs I0420 11:55:15.293841 3689 cni.go:452] [tsf-tas/ctlog-fb874484-87r7x b57e43ec10043dc2fc36dc779478ee604d96c8806a4203e0237da730e7d59451 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-fb874484-87r7x b57e43ec10043dc2fc36dc779478ee604d96c8806a4203e0237da730e7d59451 network default NAD default NAD key ] I0420 11:55:15.302923 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-fb874484-87r7x, hostIfaceName: b57e43ec10043dc, network: default, NAD default, SandboxID: "b57e43ec10043dc2fc36dc779478ee604d96c8806a4203e0237da730e7d59451", PCI device ID: , UID: "1af47395-e774-4c05-9be5-d8ffec092ec2", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0420 11:55:15.355305 3689 cni.go:473] [tsf-tas/ctlog-fb874484-87r7x b57e43ec10043dc2fc36dc779478ee604d96c8806a4203e0237da730e7d59451 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-fb874484-87r7x b57e43ec10043dc2fc36dc779478ee604d96c8806a4203e0237da730e7d59451 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:83:d2:4a:4d:73\",\"name\":\"b57e43ec10043dc\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/43a75d84-e53a-40e8-a3ba-9e9f63c60dbe\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:55:19.174588 3689 obj_retry.go:498] Detected object tsf-tpa/migrate-db-r7zqp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:55:19.175180 3689 obj_retry.go:498] Detected object tsf-tpa/migrate-db-r7zqp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0420 11:55:19.214553 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0420 11:55:20.042990 3689 obj_retry.go:492] Detected object tsf-tpa/migrate-db-r7zqp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:55:20.043016 3689 obj_retry.go:492] Detected object tsf-tpa/migrate-db-r7zqp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:55:22.194345 3689 obj_retry.go:498] Detected object tsf-tpa/create-importers-7dqlw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:55:22.194942 3689 obj_retry.go:498] Detected object tsf-tpa/create-importers-7dqlw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:55:23.214220 3689 base_network_controller_pods.go:487] [default/tsf-tpa/server-76bc5d7c69-6v4gn] creating logical port tsf-tpa_server-76bc5d7c69-6v4gn for pod on switch ip-10-0-1-126.ec2.internal I0420 11:55:23.214337 3689 kube.go:256] Updating pod tsf-tpa/server-76bc5d7c69-6v4gn I0420 11:55:23.219872 3689 obj_retry.go:492] Detected object tsf-tpa/create-importers-7dqlw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:55:23.219894 3689 obj_retry.go:492] Detected object tsf-tpa/create-importers-7dqlw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:55:23.234402 3689 pod.go:62] [tsf-tpa/server-76bc5d7c69-6v4gn] pod update took 20.100526ms I0420 11:55:23.234425 3689 base_network_controller_pods.go:951] [default/tsf-tpa/server-76bc5d7c69-6v4gn] addLogicalPort annotation time took 20.127371ms I0420 11:55:23.235788 3689 pods.go:271] [tsf-tpa/server-76bc5d7c69-6v4gn] addLogicalPort took 21.592112ms, libovsdb time 823.633µs I0420 11:55:24.211006 3689 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-wv57z-d2mm7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:55:24.211574 3689 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-wv57z-d2mm7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:55:25.031514 3689 base_network_controller_pods.go:487] [default/tsf-tpa/importer-5b5b6dd8fb-2kw6p] creating logical port tsf-tpa_importer-5b5b6dd8fb-2kw6p for pod on switch ip-10-0-1-126.ec2.internal I0420 11:55:25.031616 3689 kube.go:256] Updating pod tsf-tpa/importer-5b5b6dd8fb-2kw6p I0420 11:55:25.049857 3689 pod.go:62] [tsf-tpa/importer-5b5b6dd8fb-2kw6p] pod update took 18.267582ms I0420 11:55:25.049882 3689 base_network_controller_pods.go:951] [default/tsf-tpa/importer-5b5b6dd8fb-2kw6p] addLogicalPort annotation time took 18.298827ms I0420 11:55:25.051263 3689 pods.go:271] [tsf-tpa/importer-5b5b6dd8fb-2kw6p] addLogicalPort took 19.767458ms, libovsdb time 901.323µs I0420 11:55:25.104561 3689 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-wv57z-d2mm7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:55:25.104582 3689 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-wv57z-d2mm7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0420 11:55:30.228094 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog I0420 11:55:30.850505 3689 cni.go:452] [tsf-tpa/server-76bc5d7c69-6v4gn 1814703ae6c3c62e51709dedbd147e931ae4eb03a1661fb38912cae8d67b08b7 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-76bc5d7c69-6v4gn 1814703ae6c3c62e51709dedbd147e931ae4eb03a1661fb38912cae8d67b08b7 network default NAD default NAD key ] I0420 11:55:30.860174 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: server-76bc5d7c69-6v4gn, hostIfaceName: 1814703ae6c3c62, network: default, NAD default, SandboxID: "1814703ae6c3c62e51709dedbd147e931ae4eb03a1661fb38912cae8d67b08b7", PCI device ID: , UID: "56507b1c-fbc5-4c00-88bf-2d7b40170eb1", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0420 11:55:30.897890 3689 cni.go:452] [tsf-tpa/importer-5b5b6dd8fb-2kw6p 6412dcdc936e8524567a518f5fcbdb6f7f63834fcb1cdb6ddb4027682892fbc8 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/importer-5b5b6dd8fb-2kw6p 6412dcdc936e8524567a518f5fcbdb6f7f63834fcb1cdb6ddb4027682892fbc8 network default NAD default NAD key ] I0420 11:55:30.909706 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: importer-5b5b6dd8fb-2kw6p, hostIfaceName: 6412dcdc936e852, network: default, NAD default, SandboxID: "6412dcdc936e8524567a518f5fcbdb6f7f63834fcb1cdb6ddb4027682892fbc8", PCI device ID: , UID: "94f9c9a8-5f0b-4707-8711-dcd04cc76678", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0420 11:55:30.913799 3689 cni.go:473] [tsf-tpa/server-76bc5d7c69-6v4gn 1814703ae6c3c62e51709dedbd147e931ae4eb03a1661fb38912cae8d67b08b7 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-76bc5d7c69-6v4gn 1814703ae6c3c62e51709dedbd147e931ae4eb03a1661fb38912cae8d67b08b7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:8f:69:f1:bd:a1\",\"name\":\"1814703ae6c3c62\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1bd765e1-74a9-493f-9bf1-64ff03f44aba\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 11:55:30.961996 3689 cni.go:473] [tsf-tpa/importer-5b5b6dd8fb-2kw6p 6412dcdc936e8524567a518f5fcbdb6f7f63834fcb1cdb6ddb4027682892fbc8 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/importer-5b5b6dd8fb-2kw6p 6412dcdc936e8524567a518f5fcbdb6f7f63834fcb1cdb6ddb4027682892fbc8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:d3:be:6b:51:43\",\"name\":\"6412dcdc936e852\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e92229ea-5e26-488c-acb2-d03a37d5644e\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 11:55:31.310861 3689 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0420 11:55:32.115988 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0420 11:55:42.332562 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0420 11:55:48.518254 3689 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0420 11:56:04.490617 3689 cni.go:452] [tsf-tpa/test-tsf-tpa 2bb7bcd40e06e40a33458cd757e3365d6ef735b71c188baeb0eb47823b75f203 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/test-tsf-tpa 2bb7bcd40e06e40a33458cd757e3365d6ef735b71c188baeb0eb47823b75f203 network default NAD default NAD key ] I0420 11:56:04.561962 3689 cni.go:473] [tsf-tpa/test-tsf-tpa 2bb7bcd40e06e40a33458cd757e3365d6ef735b71c188baeb0eb47823b75f203 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/test-tsf-tpa 2bb7bcd40e06e40a33458cd757e3365d6ef735b71c188baeb0eb47823b75f203 network default NAD default NAD key default], result "{}", err I0420 11:56:04.619242 3689 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:56:04.619281 3689 pods.go:176] Deleting pod: tsf-tpa/test-tsf-tpa I0420 11:56:04.620741 3689 pods.go:236] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.128.0.51 I0420 11:56:04.620774 3689 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:56:04.675045 3689 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:56:04.675129 3689 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:56:04.684238 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0420 11:56:04.684264 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0420 11:56:05.525840 3689 namespace.go:98] [build-service] adding namespace I0420 11:56:05.525922 3689 namespace.go:98] [default-tenant] adding namespace I0420 11:56:05.526964 3689 namespace.go:102] [build-service] adding namespace took 1.102142ms I0420 11:56:05.527683 3689 namespace.go:102] [default-tenant] adding namespace took 1.747394ms I0420 11:56:05.529123 3689 namespace.go:98] [integration-service] adding namespace I0420 11:56:05.529188 3689 namespace.go:98] [image-controller] adding namespace I0420 11:56:05.529732 3689 namespace.go:102] [integration-service] adding namespace took 595.598µs I0420 11:56:05.530264 3689 namespace.go:102] [image-controller] adding namespace took 1.059557ms I0420 11:56:05.576460 3689 namespace.go:142] [build-service] updating namespace I0420 11:56:05.618349 3689 namespace.go:142] [build-service] updating namespace I0420 11:56:05.620082 3689 namespace.go:142] [default-tenant] updating namespace I0420 11:56:05.657208 3689 namespace.go:142] [default-tenant] updating namespace I0420 11:56:05.686306 3689 namespace.go:142] [integration-service] updating namespace I0420 11:56:05.730334 3689 namespace.go:142] [image-controller] updating namespace I0420 11:56:05.735017 3689 namespace.go:142] [integration-service] updating namespace I0420 11:56:05.760390 3689 namespace.go:142] [image-controller] updating namespace I0420 11:56:05.837238 3689 namespace.go:142] [build-service] updating namespace I0420 11:56:05.855226 3689 namespace.go:142] [integration-service] updating namespace I0420 11:56:05.893132 3689 namespace.go:98] [release-service] adding namespace I0420 11:56:05.893949 3689 namespace.go:102] [release-service] adding namespace took 798.84µs I0420 11:56:05.939474 3689 namespace.go:142] [konflux-ui] updating namespace I0420 11:56:05.963485 3689 namespace.go:142] [release-service] updating namespace I0420 11:56:06.017136 3689 namespace.go:142] [release-service] updating namespace I0420 11:56:06.156261 3689 namespace.go:98] [konflux-info] adding namespace I0420 11:56:06.157081 3689 namespace.go:102] [konflux-info] adding namespace took 801.478µs I0420 11:56:06.243367 3689 namespace.go:142] [konflux-info] updating namespace I0420 11:56:06.262462 3689 namespace.go:98] [namespace-lister] adding namespace I0420 11:56:06.263246 3689 namespace.go:102] [namespace-lister] adding namespace took 767.204µs I0420 11:56:06.290831 3689 namespace.go:142] [konflux-info] updating namespace I0420 11:56:06.290875 3689 namespace.go:98] [enterprise-contract-service] adding namespace I0420 11:56:06.291578 3689 namespace.go:102] [enterprise-contract-service] adding namespace took 691.084µs I0420 11:56:06.331189 3689 namespace.go:142] [image-controller] updating namespace I0420 11:56:06.346094 3689 namespace.go:142] [namespace-lister] updating namespace I0420 11:56:06.499890 3689 namespace.go:142] [enterprise-contract-service] updating namespace I0420 11:56:06.500105 3689 namespace.go:142] [namespace-lister] updating namespace I0420 11:56:06.572954 3689 namespace.go:142] [default-tenant] updating namespace I0420 11:56:06.716646 3689 namespace.go:98] [segment-bridge] adding namespace I0420 11:56:06.717487 3689 namespace.go:102] [segment-bridge] adding namespace took 820.838µs I0420 11:56:06.775150 3689 namespace.go:142] [enterprise-contract-service] updating namespace I0420 11:56:07.353175 3689 namespace.go:142] [segment-bridge] updating namespace I0420 11:56:07.645554 3689 namespace.go:142] [build-service] updating namespace I0420 11:56:07.862135 3689 namespace.go:142] [segment-bridge] updating namespace I0420 11:56:07.983549 3689 namespace.go:142] [default-tenant] updating namespace I0420 11:56:08.046425 3689 namespace.go:142] [image-controller] updating namespace I0420 11:56:08.128215 3689 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0420 11:56:08.128815 3689 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0420 11:56:08.128836 3689 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0420 11:56:08.131653 3689 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0420 11:56:08.227163 3689 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0420 11:56:08.227186 3689 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0420 11:56:08.227977 3689 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0420 11:56:08.345419 3689 namespace.go:142] [integration-service] updating namespace I0420 11:56:08.581509 3689 namespace.go:142] [release-service] updating namespace I0420 11:56:08.792469 3689 namespace.go:142] [build-service] updating namespace I0420 11:56:08.829014 3689 namespace.go:142] [default-tenant] updating namespace I0420 11:56:08.849334 3689 namespace.go:142] [image-controller] updating namespace I0420 11:56:08.884109 3689 namespace.go:142] [integration-service] updating namespace I0420 11:56:08.910299 3689 namespace.go:142] [release-service] updating namespace I0420 11:56:09.021057 3689 namespace.go:142] [build-service] updating namespace I0420 11:56:09.138629 3689 namespace.go:142] [default-tenant] updating namespace I0420 11:56:09.265487 3689 namespace.go:142] [image-controller] updating namespace I0420 11:56:09.361622 3689 namespace.go:142] [integration-service] updating namespace I0420 11:56:09.444884 3689 namespace.go:142] [release-service] updating namespace I0420 11:56:10.734707 3689 namespace.go:142] [enterprise-contract-service] updating namespace I0420 11:56:10.951791 3689 namespace.go:142] [konflux-info] updating namespace I0420 11:56:11.142299 3689 namespace.go:142] [namespace-lister] updating namespace I0420 11:56:11.336521 3689 namespace.go:142] [segment-bridge] updating namespace I0420 11:56:11.451733 3689 namespace.go:142] [enterprise-contract-service] updating namespace I0420 11:56:11.473408 3689 namespace.go:142] [konflux-info] updating namespace I0420 11:56:11.494221 3689 namespace.go:142] [namespace-lister] updating namespace I0420 11:56:11.515726 3689 namespace.go:142] [segment-bridge] updating namespace I0420 11:56:11.599461 3689 namespace.go:142] [enterprise-contract-service] updating namespace I0420 11:56:11.696090 3689 namespace.go:142] [konflux-info] updating namespace I0420 11:56:11.788167 3689 namespace.go:142] [namespace-lister] updating namespace I0420 11:56:11.848599 3689 namespace.go:142] [segment-bridge] updating namespace W0420 11:56:12.323383 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0420 11:56:12.329531 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0420 11:56:13.401837 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=159 W0420 11:56:15.337950 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0420 11:56:19.369341 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0420 11:56:19.370171 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0420 11:56:20.371891 3689 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0420 11:56:23.407441 3689 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0420 11:56:23.407465 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0420 11:56:23.432670 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0420 11:56:23.433969 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0420 11:56:23.437563 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0420 11:56:23.437576 3689 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0420 11:56:25.417791 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0420 11:56:25.417853 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service I0420 11:56:30.711357 3689 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-zcthk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:56:30.711893 3689 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-zcthk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:56:31.455702 3689 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-zcthk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:56:31.455726 3689 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-zcthk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:56:31.465440 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-zcthk I0420 11:56:31.465460 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-zcthk I0420 11:56:37.400007 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=82 I0420 11:57:02.524958 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0420 11:57:20.506194 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=11 I0420 11:57:33.477574 3689 obj_retry.go:498] Detected object konflux-ui/proxy-9f69774dc-2wjlc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:57:33.478159 3689 obj_retry.go:498] Detected object konflux-ui/proxy-9f69774dc-2wjlc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:57:33.776514 3689 obj_retry.go:492] Detected object konflux-ui/proxy-9f69774dc-2wjlc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:57:33.776543 3689 obj_retry.go:492] Detected object konflux-ui/proxy-9f69774dc-2wjlc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:57:33.786373 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-9f69774dc-2wjlc I0420 11:57:33.786394 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-9f69774dc-2wjlc I0420 11:57:45.042602 3689 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 11:57:45.043213 3689 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 11:57:45.064017 3689 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:57:45.064039 3689 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 11:57:45.068618 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0420 11:57:45.068642 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0420 11:58:18.225775 3689 namespace.go:98] [default-managed-tenant-n1jbg] adding namespace I0420 11:58:18.226634 3689 namespace.go:102] [default-managed-tenant-n1jbg] adding namespace took 838.367µs I0420 11:58:18.253402 3689 namespace.go:142] [default-managed-tenant-n1jbg] updating namespace I0420 11:58:18.273644 3689 namespace.go:142] [default-managed-tenant-n1jbg] updating namespace I0420 11:58:19.245829 3689 namespace.go:142] [default-managed-tenant-n1jbg] updating namespace I0420 11:58:19.389981 3689 namespace.go:142] [default-managed-tenant-n1jbg] updating namespace I0420 11:58:19.473118 3689 namespace.go:142] [default-managed-tenant-n1jbg] updating namespace I0420 11:58:59.401644 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=155 I0420 11:59:13.405036 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=13 I0420 11:59:13.487294 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0420 11:59:30.525477 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0420 12:00:02.799376 3689 obj_retry.go:498] Detected object integration-service/integration-service-snapshot-garbage-collector-29611440-9bjrn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:00:02.799984 3689 obj_retry.go:498] Detected object integration-service/integration-service-snapshot-garbage-collector-29611440-9bjrn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:00:03.826016 3689 obj_retry.go:492] Detected object integration-service/integration-service-snapshot-garbage-collector-29611440-9bjrn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:00:03.826039 3689 obj_retry.go:492] Detected object integration-service/integration-service-snapshot-garbage-collector-29611440-9bjrn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:00:04.572854 3689 obj_retry.go:498] Detected object kube-system/ecr-credential-refresh-29611440-xw8qq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:00:04.573353 3689 obj_retry.go:498] Detected object kube-system/ecr-credential-refresh-29611440-xw8qq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:00:04.600139 3689 obj_retry.go:492] Detected object kube-system/ecr-credential-refresh-29611440-xw8qq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:00:04.600157 3689 obj_retry.go:492] Detected object kube-system/ecr-credential-refresh-29611440-xw8qq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:00:04.694305 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611440-8bnr8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:00:04.694839 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611440-8bnr8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:00:05.718343 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611440-8bnr8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:00:05.718370 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611440-8bnr8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:00:31.497314 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=11 I0420 12:00:58.490316 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0420 12:00:59.502157 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0420 12:01:13.514806 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0420 12:01:47.405649 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=22 I0420 12:01:53.410645 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=418 I0420 12:02:39.402999 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=31 I0420 12:02:55.487066 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0420 12:02:56.404823 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0420 12:03:47.520436 3689 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0420 12:04:59.407590 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=5 I0420 12:05:01.509371 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0420 12:05:04.062232 3689 obj_retry.go:492] Detected object tsf-tpa/create-db-wbkjp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:05:04.062284 3689 obj_retry.go:492] Detected object tsf-tpa/create-db-wbkjp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:05:04.069836 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-wbkjp I0420 12:05:04.069879 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-wbkjp I0420 12:05:20.042141 3689 obj_retry.go:492] Detected object tsf-tpa/migrate-db-r7zqp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:05:20.042186 3689 obj_retry.go:492] Detected object tsf-tpa/migrate-db-r7zqp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:05:20.045995 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-r7zqp I0420 12:05:20.046016 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-r7zqp I0420 12:05:23.052825 3689 obj_retry.go:492] Detected object tsf-tpa/create-importers-7dqlw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:05:23.052884 3689 obj_retry.go:492] Detected object tsf-tpa/create-importers-7dqlw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:05:23.060526 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-7dqlw I0420 12:05:23.060559 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-7dqlw I0420 12:06:26.489770 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0420 12:06:41.527403 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11 I0420 12:06:45.403712 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=8 I0420 12:07:10.202577 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-pull-request-l9mzc-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:07:10.203210 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-pull-request-l9mzc-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:07:10.850461 3689 obj_retry.go:492] Detected object default-tenant/tsf-demo-comp-on-pull-request-l9mzc-init-pod of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:07:10.850486 3689 obj_retry.go:492] Detected object default-tenant/tsf-demo-comp-on-pull-request-l9mzc-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:07:10.866544 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod default-tenant/tsf-demo-comp-on-pull-request-l9mzc-init-pod I0420 12:07:10.866570 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod default-tenant/tsf-demo-comp-on-pull-request-l9mzc-init-pod I0420 12:07:12.009165 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:07:12.009755 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:07:25.532321 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-2zntn for pod on switch ip-10-0-1-126.ec2.internal I0420 12:07:25.532402 3689 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn I0420 12:07:25.550654 3689 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn] pod update took 18.274043ms I0420 12:07:25.550680 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn] addLogicalPort annotation time took 18.306157ms I0420 12:07:25.551980 3689 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn] addLogicalPort took 19.675748ms, libovsdb time 861.933µs I0420 12:07:25.893584 3689 cni.go:452] [openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn 11c9a19ea9960475e81b0d6fa77776cde314223520acc1e5c63409ea4534f12a network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn 11c9a19ea9960475e81b0d6fa77776cde314223520acc1e5c63409ea4534f12a network default NAD default NAD key ] I0420 12:07:25.903223 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-2zntn, hostIfaceName: 11c9a19ea996047, network: default, NAD default, SandboxID: "11c9a19ea9960475e81b0d6fa77776cde314223520acc1e5c63409ea4534f12a", PCI device ID: , UID: "9891c463-c607-45b3-96b5-9b502e6922bf", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0420 12:07:25.959972 3689 cni.go:473] [openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn 11c9a19ea9960475e81b0d6fa77776cde314223520acc1e5c63409ea4534f12a network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn 11c9a19ea9960475e81b0d6fa77776cde314223520acc1e5c63409ea4534f12a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:6c:2c:5e:51:f7\",\"name\":\"11c9a19ea996047\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8adcd887-4080-4ffe-9df3-65f4a47db946\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 12:07:27.122360 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:07:27.122981 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0420 12:07:39.964822 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 12:07:39.964844 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 12:07:39.964849 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 12:07:39.964855 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 12:07:39.964869 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 12:07:39.964875 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 12:07:39.964880 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 12:07:39.964884 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 12:07:40.511488 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 12:07:40.511515 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 12:07:40.511522 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 12:07:40.511527 3689 util.go:959] Failed to build node endpoints for node ip-10-0-1-126.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0420 12:07:47.406123 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=32 I0420 12:07:57.504231 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0420 12:08:08.428937 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:08:08.429477 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:08:19.407554 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=5 I0420 12:08:33.489270 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0420 12:08:35.527922 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0420 12:09:00.500692 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0420 12:09:14.522596 3689 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0420 12:09:52.413425 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=93 I0420 12:09:57.493103 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0420 12:10:02.408186 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=10 I0420 12:10:04.035301 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611450-ppt5s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:10:04.035887 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611450-ppt5s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:10:04.906385 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611450-ppt5s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:10:04.906412 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611450-ppt5s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:10:04.945109 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611420-jjt9x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:10:04.945138 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611420-jjt9x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:10:04.949588 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29611420-jjt9x I0420 12:10:04.949615 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29611420-jjt9x I0420 12:11:02.517358 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0420 12:11:09.640752 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:11:09.641344 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:11:40.672554 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:11:40.673214 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:12:45.512355 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0420 12:12:57.271520 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:12:57.272172 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:12:59.587665 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:12:59.588292 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:12:59.621981 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:12:59.622545 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:12:59.664455 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:12:59.665041 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:13:11.427843 3689 obj_retry.go:498] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-997nj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:13:11.428473 3689 obj_retry.go:498] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-997nj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:13:12.176638 3689 obj_retry.go:492] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-997nj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:13:12.176666 3689 obj_retry.go:492] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-997nj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:13:12.187467 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-997nj I0420 12:13:12.187496 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-997nj I0420 12:13:20.405986 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=14 I0420 12:13:38.796438 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-tpa-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:13:38.797081 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-tpa-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:13:41.222546 3689 cni.go:452] [openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn 11c9a19ea9960475e81b0d6fa77776cde314223520acc1e5c63409ea4534f12a network default NAD default NAD key ] DEL starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn 11c9a19ea9960475e81b0d6fa77776cde314223520acc1e5c63409ea4534f12a network default NAD default NAD key ] I0420 12:13:41.315822 3689 cni.go:473] [openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn 11c9a19ea9960475e81b0d6fa77776cde314223520acc1e5c63409ea4534f12a network default NAD default NAD key default] DEL finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn 11c9a19ea9960475e81b0d6fa77776cde314223520acc1e5c63409ea4534f12a network default NAD default NAD key default], result "{}", err I0420 12:13:41.376666 3689 obj_retry.go:498] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:13:41.376699 3689 pods.go:176] Deleting pod: openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn I0420 12:13:41.378396 3689 pods.go:236] Attempting to release IPs for pod: openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn, ips: 10.128.0.56 I0420 12:13:41.378423 3689 obj_retry.go:498] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:13:41.655489 3689 obj_retry.go:492] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:13:41.655517 3689 obj_retry.go:492] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 12:13:41.664110 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn I0420 12:13:41.664144 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-2zntn I0420 12:13:56.491564 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0420 12:13:59.491804 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0420 12:14:21.530245 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0420 12:14:38.410174 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0420 12:15:06.496209 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0420 12:15:32.239410 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:15:32.240061 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:16:00.731415 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:16:00.732040 3689 obj_retry.go:498] Detected object default-tenant/tsf-demo-comp-on-push-8zc7p-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:16:18.900706 3689 obj_retry.go:498] Detected object default-tenant/my-integration-test-bixa-g8fbc-collect-keyless-params-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:16:18.901333 3689 obj_retry.go:498] Detected object default-tenant/my-integration-test-bixa-g8fbc-collect-keyless-params-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:16:34.036303 3689 obj_retry.go:498] Detected object default-tenant/my-integration-test-bixa-g8fbc-verify-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 12:16:34.036876 3689 obj_retry.go:498] Detected object default-tenant/my-integration-test-bixa-g8fbc-verify-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 12:16:45.408499 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=46