+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-140.ec2.internal ++ [[ -n ip-10-0-2-140.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-140.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0428 11:08:33.501194431 - disable conntrack on geneve port' I0428 11:08:33.501194431 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0428 11:08:33.530153781 - starting ovnkube-node + echo 'I0428 11:08:33.530153781 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-140.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-140.ec2.internal --init-node ip-10-0-2-140.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-140.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0428 11:08:33.572768 3665 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0428 11:08:33.572825 3665 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-a33fc7489a.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0428 11:08:33.574270 3665 kube.go:431] Waiting for certificate I0428 11:08:33.574294 3665 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0428 11:08:33.574331 3665 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0428 11:08:33.575147 3665 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0428 11:08:33.605752 3665 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 11:08:33.605975 3665 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 11:08:33.608948 3665 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 11:08:33.609059 3665 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-bjq89" I0428 11:08:33.615555 3665 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-bjq89" I0428 11:08:33.615652 3665 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 11:08:34.574418 3665 kube.go:438] Certificate found I0428 11:08:34.575126 3665 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0428 11:08:34.576132 3665 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0428 11:08:34.634815 3665 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-29 11:03:33 +0000 UTC" deadline="2026-04-29 06:13:12.787136851 +0000 UTC" I0428 11:08:34.634858 3665 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h4m38.152295508s" I0428 11:08:34.652074 3665 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0428 11:08:34.661964 3665 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0428 11:08:34.662034 3665 server.go:101] MetricServer registers OVS metrics I0428 11:08:34.662366 3665 server.go:105] MetricServer registers OVN DB metrics I0428 11:08:34.662782 3665 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0428 11:08:34.662857 3665 factory.go:561] Starting watch factory I0428 11:08:34.662964 3665 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.662976 3665 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.662983 3665 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.662992 3665 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.663007 3665 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.663038 3665 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.663055 3665 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.663074 3665 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.663129 3665 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.663551 3665 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.663926 3665 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.663947 3665 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.665489 3665 controller_manager.go:370] Starting the ovnkube controller I0428 11:08:34.665503 3665 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-140.ec2.internal I0428 11:08:34.665555 3665 controller_manager.go:395] NBDB zone sync took: 45.27µs I0428 11:08:34.665568 3665 factory.go:561] Starting watch factory I0428 11:08:34.666999 3665 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0428 11:08:34.667020 3665 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0428 11:08:34.681000 3665 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0428 11:08:34.681042 3665 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0428 11:08:34.681805 3665 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.685082 3665 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.685435 3665 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0428 11:08:34.685474 3665 server.go:109] MetricServer registers OVN Controller metrics I0428 11:08:34.685550 3665 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.686013 3665 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.689154 3665 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.691159 3665 server.go:113] MetricServer registers OVN Northd metrics I0428 11:08:34.695851 3665 metrics.go:519] OVN Metrics Server starts to run ... I0428 11:08:34.699905 3665 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:08:34.769113 3665 factory.go:1977] *v1.Service informer cache synced successfully I0428 11:08:34.769137 3665 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0428 11:08:34.769142 3665 factory.go:1977] *v1.Pod informer cache synced successfully I0428 11:08:34.769145 3665 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0428 11:08:34.769148 3665 factory.go:1977] *v1.Namespace informer cache synced successfully I0428 11:08:34.769151 3665 factory.go:1977] *v1.Node informer cache synced successfully I0428 11:08:34.769262 3665 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 11:08:34.769271 3665 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 11:08:34.769277 3665 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 11:08:34.769294 3665 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 11:08:34.770753 3665 factory.go:1977] *v1.Namespace informer cache synced successfully I0428 11:08:34.770769 3665 factory.go:1977] *v1.Node informer cache synced successfully I0428 11:08:34.770774 3665 factory.go:1977] *v1.Service informer cache synced successfully I0428 11:08:34.770779 3665 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0428 11:08:34.770784 3665 factory.go:1977] *v1.Pod informer cache synced successfully I0428 11:08:34.770787 3665 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0428 11:08:34.773222 3665 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 11:08:34.773245 3665 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 11:08:34.775116 3665 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0428 11:08:34.775128 3665 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0428 11:08:34.775189 3665 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.775203 3665 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.775851 3665 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0428 11:08:34.775863 3665 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0428 11:08:34.778288 3665 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.780214 3665 factory.go:1977] *v1.EgressIP informer cache synced successfully I0428 11:08:34.780284 3665 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.780303 3665 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.781180 3665 factory.go:1977] *v1.EgressIP informer cache synced successfully I0428 11:08:34.783907 3665 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.785459 3665 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0428 11:08:34.785516 3665 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.785536 3665 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.786731 3665 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0428 11:08:34.788695 3665 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.790698 3665 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0428 11:08:34.790769 3665 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.790782 3665 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.792470 3665 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0428 11:08:34.793674 3665 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.796516 3665 factory.go:1977] *v1.EgressService informer cache synced successfully I0428 11:08:34.796572 3665 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.796583 3665 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.797620 3665 factory.go:1977] *v1.EgressService informer cache synced successfully I0428 11:08:34.799861 3665 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.801787 3665 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0428 11:08:34.801969 3665 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 11:08:34.801985 3665 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 11:08:34.802669 3665 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0428 11:08:34.805408 3665 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 11:08:34.807398 3665 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0428 11:08:34.807473 3665 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.807489 3665 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.807527 3665 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.807542 3665 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.808090 3665 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0428 11:08:34.810902 3665 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.810971 3665 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 11:08:34.812901 3665 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0428 11:08:34.812912 3665 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0428 11:08:34.812919 3665 factory.go:680] Watch Factory start up complete, took: 150.067271ms I0428 11:08:34.813010 3665 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0428 11:08:34.813166 3665 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0428 11:08:34.813179 3665 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0428 11:08:34.813190 3665 factory.go:680] Watch Factory start up complete, took: 147.619289ms I0428 11:08:34.813197 3665 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-140.ec2.internal" zone I0428 11:08:34.865850 3665 default_node_network_controller.go:710] Initializing the default node network controller I0428 11:08:34.942257 3665 udn_isolation.go:101] Starting UDN host isolation manager I0428 11:08:34.948386 3665 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0428 11:08:35.014983 3665 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0428 11:08:35.015072 3665 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0428 11:08:35.015099 3665 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0428 11:08:35.052119 3665 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0428 11:08:35.052185 3665 default_node_network_controller.go:809] Node ip-10-0-2-140.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0428 11:08:35.059246 3665 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.140"] k8s.ovn.org/zone-name:ip-10-0-2-140.ec2.internal] on node ip-10-0-2-140.ec2.internal I0428 11:08:35.080191 3665 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0428 11:08:35.088009 3665 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0428 11:08:35.088152 3665 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0428 11:08:35.088291 3665 gateway_init.go:263] Preparing Gateway I0428 11:08:35.088303 3665 gateway_shared_intf.go:1687] Creating new gateway I0428 11:08:35.208567 3665 iptables.go:108] Creating table: filter chain: FORWARD I0428 11:08:35.215618 3665 iptables.go:108] Creating table: filter chain: OUTPUT I0428 11:08:35.222510 3665 gateway_shared_intf.go:1816] Gateway Creation Complete I0428 11:08:35.222529 3665 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-140.ec2.internal","mac-address":"0e:be:38:f7:d0:f9","ip-addresses":["10.0.2.140/24"],"ip-address":"10.0.2.140/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:0b84a12b-335a-47d7-b59a-108faa74bb35 k8s.ovn.org/node-encap-ips:["10.0.2.140"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.140/24"} k8s.ovn.org/zone-name:ip-10-0-2-140.ec2.internal] on node ip-10-0-2-140.ec2.internal I0428 11:08:35.243360 3665 default_node_network_controller.go:519] Node connection status = connected I0428 11:08:35.313478 3665 controller_manager.go:429] Waiting for node in zone sync took: 500.265998ms I0428 11:08:35.318638 3665 controller_manager.go:611] Switching to transit router for layer2 networks I0428 11:08:35.318653 3665 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-140.ec2.internal I0428 11:08:35.335019 3665 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0428 11:08:35.342181 3665 controller_manager.go:290] SCTP support detected in OVN I0428 11:08:35.342747 3665 services_controller.go:69] Creating services controller for network=default I0428 11:08:35.342843 3665 udn_enabled_svc.go:80] Waiting for service informer to sync I0428 11:08:35.342929 3665 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0428 11:08:35.342968 3665 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0428 11:08:35.342982 3665 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0428 11:08:35.342990 3665 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0428 11:08:35.343184 3665 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0428 11:08:35.343222 3665 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0428 11:08:35.343298 3665 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 78.57µs I0428 11:08:35.343321 3665 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 91.8µs I0428 11:08:35.343328 3665 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 105.981µs I0428 11:08:35.343334 3665 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0428 11:08:35.343340 3665 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0428 11:08:35.343343 3665 udn_enabled_svc.go:95] Performing full resync I0428 11:08:35.343350 3665 default_network_controller.go:325] Starting the default network controller I0428 11:08:35.343451 3665 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 15.661µs I0428 11:08:35.343658 3665 udn_enabled_svc.go:99] Waiting for handler to sync I0428 11:08:35.343669 3665 udn_enabled_svc.go:104] Starting worker I0428 11:08:35.352990 3665 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0428 11:08:35.353058 3665 repair.go:33] Syncing exgw routes took 53.925µs I0428 11:08:35.353080 3665 default_network_controller.go:414] Starting all the Watchers... I0428 11:08:35.353296 3665 namespace.go:98] [openshift-cluster-version] adding namespace I0428 11:08:35.353307 3665 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0428 11:08:35.353315 3665 namespace.go:98] [openshift-network-console] adding namespace I0428 11:08:35.353329 3665 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0428 11:08:35.353333 3665 namespace.go:98] [openshift-host-network] adding namespace I0428 11:08:35.353341 3665 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0428 11:08:35.353352 3665 namespace.go:98] [openshift-backplane-srep] adding namespace I0428 11:08:35.353352 3665 namespace.go:98] [openshift-machine-config-operator] adding namespace I0428 11:08:35.353357 3665 namespace.go:98] [openshift-backplane-csm] adding namespace I0428 11:08:35.353358 3665 namespace.go:98] [openshift-network-operator] adding namespace I0428 11:08:35.353361 3665 namespace.go:98] [openshift-apiserver] adding namespace I0428 11:08:35.353365 3665 namespace.go:98] [openshift-dns-operator] adding namespace I0428 11:08:35.353367 3665 namespace.go:98] [openshift-backplane-tam] adding namespace I0428 11:08:35.353372 3665 namespace.go:98] [openshift-service-ca-operator] adding namespace I0428 11:08:35.353296 3665 namespace.go:98] [openshift-customer-monitoring] adding namespace I0428 11:08:35.353858 3665 namespace.go:102] [openshift-cluster-version] adding namespace took 549.294µs I0428 11:08:35.353874 3665 namespace.go:98] [openshift-config] adding namespace I0428 11:08:35.354266 3665 namespace.go:102] [openshift-config] adding namespace took 386.895µs I0428 11:08:35.354279 3665 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0428 11:08:35.354734 3665 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 450.574µs I0428 11:08:35.354752 3665 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0428 11:08:35.355311 3665 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 1.99334ms I0428 11:08:35.355328 3665 namespace.go:98] [openshift-kube-scheduler] adding namespace I0428 11:08:35.355817 3665 namespace.go:102] [openshift-network-console] adding namespace took 2.491192ms I0428 11:08:35.355829 3665 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0428 11:08:35.356251 3665 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 2.91398ms I0428 11:08:35.356268 3665 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0428 11:08:35.356828 3665 namespace.go:102] [openshift-host-network] adding namespace took 3.489113ms I0428 11:08:35.356846 3665 namespace.go:98] [openshift-dns] adding namespace I0428 11:08:35.357298 3665 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 3.947164ms I0428 11:08:35.357315 3665 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0428 11:08:35.357741 3665 namespace.go:102] [openshift-backplane-srep] adding namespace took 4.382569ms I0428 11:08:35.357754 3665 namespace.go:98] [openshift-image-registry] adding namespace I0428 11:08:35.358250 3665 namespace.go:102] [openshift-backplane-csm] adding namespace took 4.87358ms I0428 11:08:35.358268 3665 namespace.go:98] [openshift-operators-redhat] adding namespace I0428 11:08:35.358818 3665 namespace.go:102] [openshift-machine-config-operator] adding namespace took 5.455482ms I0428 11:08:35.358832 3665 namespace.go:98] [openshift-insights] adding namespace I0428 11:08:35.359347 3665 namespace.go:102] [openshift-network-operator] adding namespace took 5.977818ms I0428 11:08:35.359366 3665 namespace.go:98] [openshift-logging] adding namespace I0428 11:08:35.359852 3665 namespace.go:102] [openshift-apiserver] adding namespace took 6.463394ms I0428 11:08:35.359864 3665 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0428 11:08:35.360279 3665 namespace.go:102] [openshift-dns-operator] adding namespace took 6.908797ms I0428 11:08:35.360289 3665 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0428 11:08:35.360795 3665 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.421609ms I0428 11:08:35.360805 3665 namespace.go:98] [openshift-config-managed] adding namespace I0428 11:08:35.361375 3665 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.994863ms I0428 11:08:35.361395 3665 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0428 11:08:35.361901 3665 namespace.go:102] [openshift-customer-monitoring] adding namespace took 8.478463ms I0428 11:08:35.361916 3665 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0428 11:08:35.362695 3665 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.93639ms I0428 11:08:35.362724 3665 namespace.go:98] [openshift-node] adding namespace I0428 11:08:35.363118 3665 namespace.go:102] [openshift-kube-scheduler] adding namespace took 7.784458ms I0428 11:08:35.363130 3665 namespace.go:98] [openshift-must-gather-operator] adding namespace I0428 11:08:35.363521 3665 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 7.687156ms I0428 11:08:35.363532 3665 namespace.go:98] [kube-public] adding namespace I0428 11:08:35.363999 3665 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 7.724899ms I0428 11:08:35.364010 3665 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0428 11:08:35.364416 3665 namespace.go:102] [openshift-dns] adding namespace took 7.564455ms I0428 11:08:35.364429 3665 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0428 11:08:35.364816 3665 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.493143ms I0428 11:08:35.364843 3665 namespace.go:98] [openshift-monitoring] adding namespace I0428 11:08:35.365397 3665 namespace.go:102] [openshift-image-registry] adding namespace took 7.637295ms I0428 11:08:35.365410 3665 namespace.go:98] [openshift-ingress] adding namespace I0428 11:08:35.365839 3665 namespace.go:102] [openshift-operators-redhat] adding namespace took 7.564059ms I0428 11:08:35.365852 3665 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0428 11:08:35.366262 3665 namespace.go:102] [openshift-insights] adding namespace took 7.425339ms I0428 11:08:35.366275 3665 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0428 11:08:35.366674 3665 namespace.go:102] [openshift-logging] adding namespace took 7.302439ms I0428 11:08:35.366695 3665 namespace.go:98] [open-cluster-management-2pv477t4c9srti67smm59tsf841pmq8b] adding namespace I0428 11:08:35.367141 3665 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.272012ms I0428 11:08:35.367153 3665 namespace.go:98] [openshift-backplane-cee] adding namespace I0428 11:08:35.367600 3665 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.306338ms I0428 11:08:35.367610 3665 namespace.go:98] [openshift-network-diagnostics] adding namespace I0428 11:08:35.368037 3665 namespace.go:102] [openshift-config-managed] adding namespace took 7.227646ms I0428 11:08:35.368048 3665 namespace.go:98] [kube-system] adding namespace I0428 11:08:35.368428 3665 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.027373ms I0428 11:08:35.368442 3665 namespace.go:98] [openshift-backplane-cse] adding namespace I0428 11:08:35.368856 3665 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.934271ms I0428 11:08:35.368869 3665 namespace.go:98] [openshift] adding namespace I0428 11:08:35.369235 3665 namespace.go:102] [openshift-node] adding namespace took 6.505855ms I0428 11:08:35.369680 3665 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.543436ms I0428 11:08:35.369693 3665 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0428 11:08:35.370098 3665 namespace.go:102] [kube-public] adding namespace took 6.558031ms I0428 11:08:35.370119 3665 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0428 11:08:35.370537 3665 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.522157ms I0428 11:08:35.370549 3665 namespace.go:98] [openshift-backplane-mobb] adding namespace I0428 11:08:35.370913 3665 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.478019ms I0428 11:08:35.370928 3665 namespace.go:98] [kube-node-lease] adding namespace I0428 11:08:35.371375 3665 namespace.go:102] [openshift-monitoring] adding namespace took 6.523966ms I0428 11:08:35.371391 3665 namespace.go:98] [dedicated-admin] adding namespace I0428 11:08:35.371737 3665 namespace.go:102] [openshift-ingress] adding namespace took 6.322206ms I0428 11:08:35.371751 3665 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0428 11:08:35.372162 3665 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.303944ms I0428 11:08:35.372176 3665 namespace.go:98] [openshift-operators] adding namespace I0428 11:08:35.372642 3665 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.361373ms I0428 11:08:35.372656 3665 namespace.go:98] [openshift-config-operator] adding namespace I0428 11:08:35.373065 3665 namespace.go:102] [open-cluster-management-2pv477t4c9srti67smm59tsf841pmq8b] adding namespace took 6.365279ms I0428 11:08:35.373074 3665 namespace.go:98] [openshift-authentication-operator] adding namespace W0428 11:08:35.373523 3665 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-2dtl4: pod openshift-network-diagnostics/network-check-target-2dtl4: no pod IPs found W0428 11:08:35.373545 3665 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-p4xdk: pod openshift-network-diagnostics/network-check-target-p4xdk: no pod IPs found W0428 11:08:35.373552 3665 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-w9w75: pod openshift-network-diagnostics/network-check-target-w9w75: no pod IPs found I0428 11:08:35.373564 3665 namespace.go:102] [openshift-backplane-cee] adding namespace took 6.407009ms I0428 11:08:35.373573 3665 namespace.go:98] [default] adding namespace I0428 11:08:35.373998 3665 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.384024ms I0428 11:08:35.374008 3665 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0428 11:08:35.374487 3665 namespace.go:102] [kube-system] adding namespace took 6.434855ms I0428 11:08:35.374501 3665 namespace.go:98] [openshift-apiserver-operator] adding namespace I0428 11:08:35.374869 3665 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.423149ms I0428 11:08:35.374878 3665 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0428 11:08:35.375350 3665 namespace.go:102] [openshift] adding namespace took 6.47495ms I0428 11:08:35.375363 3665 namespace.go:98] [openshift-console-operator] adding namespace I0428 11:08:35.375735 3665 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.036757ms I0428 11:08:35.375746 3665 namespace.go:98] [openshift-backplane] adding namespace I0428 11:08:35.376221 3665 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.096528ms I0428 11:08:35.376233 3665 namespace.go:98] [openshift-console] adding namespace I0428 11:08:35.376574 3665 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.020175ms I0428 11:08:35.376587 3665 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0428 11:08:35.377073 3665 namespace.go:102] [kube-node-lease] adding namespace took 6.141114ms I0428 11:08:35.377085 3665 namespace.go:98] [openshift-package-operator] adding namespace I0428 11:08:35.377441 3665 namespace.go:102] [dedicated-admin] adding namespace took 6.044846ms I0428 11:08:35.377451 3665 namespace.go:98] [openshift-marketplace] adding namespace I0428 11:08:35.377945 3665 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.188184ms I0428 11:08:35.377958 3665 namespace.go:98] [openshift-ingress-canary] adding namespace I0428 11:08:35.378309 3665 namespace.go:102] [openshift-operators] adding namespace took 6.12789ms I0428 11:08:35.378318 3665 namespace.go:98] [openshift-route-controller-manager] adding namespace I0428 11:08:35.378853 3665 namespace.go:102] [openshift-config-operator] adding namespace took 6.193385ms I0428 11:08:35.378862 3665 namespace.go:98] [openshift-ingress-operator] adding namespace I0428 11:08:35.379239 3665 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.160598ms I0428 11:08:35.379247 3665 namespace.go:98] [openshift-authentication] adding namespace I0428 11:08:35.379745 3665 namespace.go:102] [default] adding namespace took 6.16615ms I0428 11:08:35.379754 3665 namespace.go:98] [openshift-console-user-settings] adding namespace I0428 11:08:35.380172 3665 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.155955ms I0428 11:08:35.380187 3665 namespace.go:98] [openshift-network-node-identity] adding namespace I0428 11:08:35.380698 3665 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.192324ms I0428 11:08:35.380727 3665 namespace.go:98] [openshift-controller-manager] adding namespace I0428 11:08:35.381172 3665 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.289514ms I0428 11:08:35.381183 3665 namespace.go:98] [openshift-infra] adding namespace I0428 11:08:35.381626 3665 namespace.go:102] [openshift-console-operator] adding namespace took 6.258418ms I0428 11:08:35.381638 3665 namespace.go:98] [openshift-multus] adding namespace I0428 11:08:35.382026 3665 namespace.go:102] [openshift-backplane] adding namespace took 6.276275ms I0428 11:08:35.382926 3665 namespace.go:102] [openshift-console] adding namespace took 6.685874ms I0428 11:08:35.382945 3665 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0428 11:08:35.383361 3665 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.767118ms I0428 11:08:35.383805 3665 namespace.go:102] [openshift-package-operator] adding namespace took 6.715754ms I0428 11:08:35.384212 3665 namespace.go:102] [openshift-marketplace] adding namespace took 6.755857ms I0428 11:08:35.384629 3665 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.666884ms I0428 11:08:35.385138 3665 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.814275ms I0428 11:08:35.385517 3665 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.649028ms I0428 11:08:35.386083 3665 namespace.go:102] [openshift-authentication] adding namespace took 6.831389ms I0428 11:08:35.386095 3665 namespace.go:98] [openshift-machine-api] adding namespace I0428 11:08:35.386456 3665 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.696934ms I0428 11:08:35.386469 3665 namespace.go:98] [openshift-etcd] adding namespace I0428 11:08:35.386951 3665 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.759008ms I0428 11:08:35.387368 3665 namespace.go:102] [openshift-controller-manager] adding namespace took 6.635152ms W0428 11:08:35.387776 3665 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-b5djk: pod openshift-multus/network-metrics-daemon-b5djk: no pod IPs found W0428 11:08:35.387787 3665 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-rzlqs: pod openshift-multus/network-metrics-daemon-rzlqs: no pod IPs found W0428 11:08:35.387794 3665 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-5hzxg: pod openshift-multus/network-metrics-daemon-5hzxg: no pod IPs found I0428 11:08:35.387830 3665 namespace.go:102] [openshift-infra] adding namespace took 6.642209ms I0428 11:08:35.387847 3665 namespace.go:98] [openshift-kube-apiserver] adding namespace I0428 11:08:35.388295 3665 namespace.go:102] [openshift-multus] adding namespace took 6.652843ms I0428 11:08:35.388753 3665 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 5.803407ms I0428 11:08:35.389186 3665 namespace.go:102] [openshift-machine-api] adding namespace took 3.085071ms I0428 11:08:35.389622 3665 namespace.go:102] [openshift-etcd] adding namespace took 3.146736ms I0428 11:08:35.390025 3665 namespace.go:102] [openshift-kube-apiserver] adding namespace took 2.173142ms W0428 11:08:35.390119 3665 master.go:471] Unable to parse remote node ip-10-0-0-109.ec2.internal chassis-id annotation. Chassis may be removed during sync W0428 11:08:35.390129 3665 master.go:471] Unable to parse remote node ip-10-0-1-135.ec2.internal chassis-id annotation. Chassis may be removed during sync I0428 11:08:35.390257 3665 master.go:548] Adding or Updating local node "ip-10-0-2-140.ec2.internal" for network "default" I0428 11:08:35.390319 3665 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-109.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-109.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-109.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-109.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-109.ec2.internal I0428 11:08:35.390367 3665 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-135.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-135.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-135.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-135.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-135.ec2.internal W0428 11:08:35.397820 3665 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-p4xdk: pod openshift-network-diagnostics/network-check-target-p4xdk: no pod IPs found W0428 11:08:35.397843 3665 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-rzlqs: pod openshift-multus/network-metrics-daemon-rzlqs: no pod IPs found I0428 11:08:35.398836 3665 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-140.ec2.internal for the network default I0428 11:08:35.399132 3665 zone_ic_handler.go:195] Time taken to create transit switch: 282.174µs I0428 11:08:35.399907 3665 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0428 11:08:35.399935 3665 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0428 11:08:35.399953 3665 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0428 11:08:35.399960 3665 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-109.ec2.internal in network "default" I0428 11:08:35.400006 3665 node_tracker.go:235] Node ip-10-0-0-109.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-109.ec2.internal" I0428 11:08:35.400022 3665 node_tracker.go:169] Node ip-10-0-0-109.ec2.internal switch + router changed, syncing services in network "default" I0428 11:08:35.400041 3665 services_controller.go:574] Full service sync requested for network=default I0428 11:08:35.400054 3665 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-135.ec2.internal in network "default" I0428 11:08:35.400068 3665 node_tracker.go:235] Node ip-10-0-1-135.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-135.ec2.internal" I0428 11:08:35.400077 3665 node_tracker.go:169] Node ip-10-0-1-135.ec2.internal switch + router changed, syncing services in network "default" I0428 11:08:35.400083 3665 services_controller.go:574] Full service sync requested for network=default I0428 11:08:35.400088 3665 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-140.ec2.internal in network "default" I0428 11:08:35.400116 3665 node_tracker.go:169] Node ip-10-0-2-140.ec2.internal switch + router changed, syncing services in network "default" I0428 11:08:35.400126 3665 services_controller.go:574] Full service sync requested for network=default I0428 11:08:35.469313 3665 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-109.ec2.internal in network "default" I0428 11:08:35.469341 3665 node_tracker.go:235] Node ip-10-0-0-109.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-109.ec2.internal" I0428 11:08:35.469343 3665 default_network_controller.go:1025] Node "ip-10-0-0-109.ec2.internal" in remote zone "ip-10-0-0-109.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 11:08:35.469357 3665 node_tracker.go:169] Node ip-10-0-0-109.ec2.internal switch + router changed, syncing services in network "default" I0428 11:08:35.469373 3665 services_controller.go:574] Full service sync requested for network=default I0428 11:08:35.487474 3665 default_network_controller.go:1025] Node "ip-10-0-0-109.ec2.internal" in remote zone "ip-10-0-0-109.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 11:08:35.500383 3665 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0428 11:08:35.500402 3665 services_controller.go:208] Setting up event handlers for services for network=default I0428 11:08:35.500471 3665 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0428 11:08:35.500509 3665 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0428 11:08:35.500522 3665 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0428 11:08:35.601638 3665 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0428 11:08:35.601681 3665 repair.go:60] Starting repairing loop for services I0428 11:08:35.601811 3665 repair.go:131] Deleted 0 stale service LBs I0428 11:08:35.601831 3665 repair.go:137] Deleted 0 stale Chassis Template Vars I0428 11:08:35.601852 3665 repair.go:62] Finished repairing loop for services: 174.239µs I0428 11:08:35.601875 3665 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0428 11:08:35.601885 3665 services_controller.go:254] Starting workers for network=default I0428 11:08:35.602414 3665 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-b5djk, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-b5djk: suppressed error logged: pod openshift-multus/network-metrics-daemon-b5djk: no pod IPs found I0428 11:08:35.602446 3665 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-w9w75, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-w9w75: suppressed error logged: pod openshift-network-diagnostics/network-check-target-w9w75: no pod IPs found I0428 11:08:35.602454 3665 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-2dtl4, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-2dtl4: suppressed error logged: pod openshift-network-diagnostics/network-check-target-2dtl4: no pod IPs found I0428 11:08:35.602655 3665 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-rzlqs] creating logical port openshift-multus_network-metrics-daemon-rzlqs for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:35.602750 3665 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-5hzxg, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-5hzxg: suppressed error logged: pod openshift-multus/network-metrics-daemon-5hzxg: no pod IPs found I0428 11:08:35.602910 3665 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-p4xdk] creating logical port openshift-network-diagnostics_network-check-target-p4xdk for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:35.602916 3665 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-rzlqs I0428 11:08:35.603045 3665 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-p4xdk I0428 11:08:35.619627 3665 pod.go:62] [openshift-network-diagnostics/network-check-target-p4xdk] pod update took 16.615367ms I0428 11:08:35.619649 3665 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-p4xdk] addLogicalPort annotation time took 16.6441ms I0428 11:08:35.620629 3665 pods.go:271] [openshift-network-diagnostics/network-check-target-p4xdk] addLogicalPort took 17.723614ms, libovsdb time 732.6µs I0428 11:08:35.620915 3665 pod.go:62] [openshift-multus/network-metrics-daemon-rzlqs] pod update took 18.165763ms I0428 11:08:35.620932 3665 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-rzlqs] addLogicalPort annotation time took 18.186403ms I0428 11:08:35.621845 3665 pods.go:271] [openshift-multus/network-metrics-daemon-rzlqs] addLogicalPort took 19.200711ms, libovsdb time 633.667µs I0428 11:08:35.622821 3665 admin_network_policy_controller.go:227] Starting controller default-network-controller I0428 11:08:35.622842 3665 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0428 11:08:35.622857 3665 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0428 11:08:35.622874 3665 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0428 11:08:35.623100 3665 repair.go:29] Repairing admin network policies took 218.992µs I0428 11:08:35.623231 3665 repair.go:92] Repairing baseline admin network policies took 121.708µs I0428 11:08:35.623640 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0428 11:08:35.624141 3665 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0428 11:08:35.624156 3665 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0428 11:08:35.626081 3665 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0428 11:08:35.626100 3665 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0428 11:08:35.626108 3665 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0428 11:08:35.628378 3665 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0428 11:08:35.628399 3665 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0428 11:08:35.628406 3665 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0428 11:08:35.629732 3665 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0428 11:08:35.629754 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0428 11:08:35.629763 3665 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0428 11:08:35.631270 3665 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0428 11:08:35.631288 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0428 11:08:35.631294 3665 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0428 11:08:35.631639 3665 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0428 11:08:35.631657 3665 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0428 11:08:35.631670 3665 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0428 11:08:35.633019 3665 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0428 11:08:35.633039 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0428 11:08:35.633047 3665 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0428 11:08:35.633702 3665 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0428 11:08:35.633742 3665 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0428 11:08:35.633750 3665 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0428 11:08:35.634251 3665 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0428 11:08:35.634265 3665 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0428 11:08:35.634274 3665 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0428 11:08:35.635170 3665 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0428 11:08:35.635190 3665 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0428 11:08:35.635201 3665 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0428 11:08:35.636275 3665 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0428 11:08:35.636300 3665 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0428 11:08:35.636311 3665 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0428 11:08:35.637538 3665 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0428 11:08:35.637555 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0428 11:08:35.637563 3665 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0428 11:08:35.638210 3665 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-109.ec2.internal in network "default" I0428 11:08:35.638269 3665 node_tracker.go:169] Node ip-10-0-0-109.ec2.internal switch + router changed, syncing services in network "default" I0428 11:08:35.638264 3665 default_network_controller.go:1025] Node "ip-10-0-0-109.ec2.internal" in remote zone "ip-10-0-0-109.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 11:08:35.638285 3665 services_controller.go:574] Full service sync requested for network=default I0428 11:08:35.639106 3665 zone_ic_handler.go:195] Time taken to create transit switch: 187.96µs I0428 11:08:35.639120 3665 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-109.ec2.internal for the network default I0428 11:08:35.639308 3665 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0428 11:08:35.639328 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0428 11:08:35.639337 3665 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0428 11:08:35.640004 3665 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0428 11:08:35.640023 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0428 11:08:35.640045 3665 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0428 11:08:35.640264 3665 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-109.ec2.internal" on network "default" took: 1.353583ms I0428 11:08:35.640665 3665 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0428 11:08:35.640683 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0428 11:08:35.640695 3665 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0428 11:08:35.641330 3665 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0428 11:08:35.641351 3665 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0428 11:08:35.641358 3665 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0428 11:08:35.642828 3665 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0428 11:08:35.642846 3665 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0428 11:08:35.642854 3665 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0428 11:08:35.644595 3665 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0428 11:08:35.644615 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0428 11:08:35.644622 3665 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0428 11:08:35.644938 3665 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0428 11:08:35.644957 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0428 11:08:35.644966 3665 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0428 11:08:35.645471 3665 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0428 11:08:35.645490 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0428 11:08:35.645507 3665 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0428 11:08:35.646048 3665 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0428 11:08:35.646064 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0428 11:08:35.646091 3665 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0428 11:08:35.646829 3665 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0428 11:08:35.646853 3665 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0428 11:08:35.646861 3665 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0428 11:08:35.647428 3665 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0428 11:08:35.647449 3665 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0428 11:08:35.647462 3665 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0428 11:08:35.648614 3665 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0428 11:08:35.648634 3665 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0428 11:08:35.648642 3665 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0428 11:08:35.649561 3665 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0428 11:08:35.649583 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0428 11:08:35.649592 3665 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0428 11:08:35.650138 3665 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0428 11:08:35.650158 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0428 11:08:35.650166 3665 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0428 11:08:35.650678 3665 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0428 11:08:35.650695 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0428 11:08:35.650703 3665 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0428 11:08:35.651200 3665 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0428 11:08:35.651216 3665 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0428 11:08:35.651221 3665 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0428 11:08:35.652476 3665 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0428 11:08:35.652499 3665 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0428 11:08:35.652513 3665 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0428 11:08:35.653606 3665 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0428 11:08:35.659052 3665 egressfirewall.go:374] Starting EgressFirewall controller I0428 11:08:35.659085 3665 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0428 11:08:35.659089 3665 controller.go:133] Adding controller egress-firewall-controller event handlers I0428 11:08:35.659120 3665 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0428 11:08:35.659137 3665 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0428 11:08:35.659121 3665 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0428 11:08:35.659148 3665 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0428 11:08:35.659441 3665 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0428 11:08:35.659457 3665 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0428 11:08:35.659464 3665 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0428 11:08:35.659476 3665 egressqos.go:193] Setting up event handlers for EgressQoS I0428 11:08:35.659484 3665 egressfirewall.go:723] Syncing node "ip-10-0-0-109.ec2.internal" for egress firewall I0428 11:08:35.659494 3665 egressfirewall.go:723] Syncing node "ip-10-0-1-135.ec2.internal" for egress firewall I0428 11:08:35.659499 3665 egressfirewall.go:723] Syncing node "ip-10-0-2-140.ec2.internal" for egress firewall I0428 11:08:35.659623 3665 egressqos.go:245] Starting EgressQoS Controller I0428 11:08:35.659653 3665 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0428 11:08:35.659664 3665 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0428 11:08:35.659669 3665 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0428 11:08:35.659673 3665 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0428 11:08:35.659678 3665 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0428 11:08:35.659682 3665 shared_informer.go:356] "Caches are synced" controller="egressqos" I0428 11:08:35.659686 3665 egressqos.go:259] Repairing EgressQoSes I0428 11:08:35.659690 3665 egressqos.go:399] Starting repairing loop for egressqos I0428 11:08:35.659807 3665 egressqos.go:401] Finished repairing loop for egressqos: 116.828µs I0428 11:08:35.659842 3665 egressservice_zone.go:132] Setting up event handlers for Egress Services I0428 11:08:35.659860 3665 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-140.ec2.internal I0428 11:08:35.659917 3665 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-140.ec2.internal : 59.284µs I0428 11:08:35.660025 3665 egressservice_zone.go:210] Starting Egress Services Controller I0428 11:08:35.660047 3665 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0428 11:08:35.660058 3665 shared_informer.go:356] "Caches are synced" controller="egressservices" I0428 11:08:35.660067 3665 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0428 11:08:35.660073 3665 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0428 11:08:35.660081 3665 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0428 11:08:35.660085 3665 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0428 11:08:35.660098 3665 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0428 11:08:35.660108 3665 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0428 11:08:35.660113 3665 egressservice_zone.go:228] Repairing Egress Services I0428 11:08:35.660421 3665 master_controller.go:88] Starting Admin Policy Based Route Controller I0428 11:08:35.660432 3665 external_controller.go:278] Starting Admin Policy Based Route Controller I0428 11:08:35.660505 3665 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-135.ec2.internal I0428 11:08:35.660521 3665 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-135.ec2.internal: 18.46µs I0428 11:08:35.660533 3665 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-140.ec2.internal I0428 11:08:35.660539 3665 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-140.ec2.internal: 8.878µs I0428 11:08:35.660544 3665 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-109.ec2.internal I0428 11:08:35.660548 3665 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-109.ec2.internal: 4.726µs I0428 11:08:35.660607 3665 default_network_controller.go:585] Completing all the Watchers took 307.517631ms I0428 11:08:35.660625 3665 default_network_controller.go:589] Starting unidling controllers I0428 11:08:35.660641 3665 unidle.go:46] Registering OVN SB ControllerEvent handler I0428 11:08:35.660652 3665 unidle.go:63] Populating Initial ContollerEvent events I0428 11:08:35.660672 3665 unidle.go:79] Setting up event handlers for services I0428 11:08:35.660761 3665 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0428 11:08:35.743377 3665 default_node_network_controller.go:519] Node connection status = connected I0428 11:08:36.244915 3665 default_node_network_controller.go:519] Node connection status = connected I0428 11:08:36.744702 3665 default_node_network_controller.go:519] Node connection status = connected I0428 11:08:36.871409 3665 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0428 11:08:37.243894 3665 default_node_network_controller.go:519] Node connection status = connected I0428 11:08:37.263193 3665 bridgeconfig.go:537] Gateway is ready I0428 11:08:37.263215 3665 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0428 11:08:37.276153 3665 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.140. Updating OVN encap IP. I0428 11:08:37.281930 3665 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.140 - it is already configured I0428 11:08:37.281944 3665 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.140/24:{} 172.20.0.1/32:{}]. Updating annotations. I0428 11:08:37.282195 3665 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.140/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-140.ec2.internal","mac-address":"0e:be:38:f7:d0:f9","ip-addresses":["10.0.2.140/24"],"ip-address":"10.0.2.140/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:0b84a12b-335a-47d7-b59a-108faa74bb35 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.140/24"}] on node ip-10-0-2-140.ec2.internal I0428 11:08:37.297373 3665 egressfirewall.go:723] Syncing node "ip-10-0-2-140.ec2.internal" for egress firewall I0428 11:08:37.297410 3665 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-140.ec2.internal I0428 11:08:37.297429 3665 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-140.ec2.internal: 26.736µs I0428 11:08:37.297436 3665 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-140.ec2.internal in network "default" I0428 11:08:37.297482 3665 master.go:548] Adding or Updating local node "ip-10-0-2-140.ec2.internal" for network "default" I0428 11:08:37.297512 3665 node_tracker.go:169] Node ip-10-0-2-140.ec2.internal switch + router changed, syncing services in network "default" I0428 11:08:37.297531 3665 services_controller.go:574] Full service sync requested for network=default I0428 11:08:37.298769 3665 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.140 I0428 11:08:37.298878 3665 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-140.ec2.internal I0428 11:08:37.302472 3665 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-140.ec2.internal. Updating no re-route policies I0428 11:08:37.302547 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-rzlqs I0428 11:08:37.302566 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-5wfw2 I0428 11:08:37.302572 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-rzlqs I0428 11:08:37.302572 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-c8bnn I0428 11:08:37.302580 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-5wfw2 I0428 11:08:37.302585 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-c8bnn I0428 11:08:37.302587 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-p4xdk I0428 11:08:37.302599 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-p4xdk I0428 11:08:37.302601 3665 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-c8bnn after 0 failed attempt(s) I0428 11:08:37.302602 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-9dg2c I0428 11:08:37.302602 3665 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-dl77t I0428 11:08:37.302568 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-5hzxg I0428 11:08:37.302614 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-w9w75 I0428 11:08:37.302618 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-n9d26 I0428 11:08:37.302624 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-5hzxg I0428 11:08:37.302621 3665 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-140.ec2.internal I0428 11:08:37.302628 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-w9w75 I0428 11:08:37.302629 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-n9d26 I0428 11:08:37.302586 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-140.ec2.internal I0428 11:08:37.302634 3665 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-140.ec2.internal I0428 11:08:37.302647 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-140.ec2.internal I0428 11:08:37.302653 3665 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-n9d26 after 0 failed attempt(s) I0428 11:08:37.302659 3665 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-140.ec2.internal after 0 failed attempt(s) I0428 11:08:37.302599 3665 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-5wfw2 after 0 failed attempt(s) I0428 11:08:37.302672 3665 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-140.ec2.internal after 0 failed attempt(s) I0428 11:08:37.302610 3665 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-rzlqs] creating logical port openshift-multus_network-metrics-daemon-rzlqs for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:37.302667 3665 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-5hzxg, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-5hzxg: suppressed error logged: pod openshift-multus/network-metrics-daemon-5hzxg: no pod IPs found I0428 11:08:37.302606 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-jqnhw I0428 11:08:37.302723 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-jqnhw I0428 11:08:37.302733 3665 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-p4xdk] creating logical port openshift-network-diagnostics_network-check-target-p4xdk for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:37.302742 3665 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-jqnhw after 0 failed attempt(s) I0428 11:08:37.302744 3665 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-w9w75, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-w9w75: suppressed error logged: pod openshift-network-diagnostics/network-check-target-w9w75: no pod IPs found I0428 11:08:37.302579 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-2rwnn I0428 11:08:37.302754 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-2rwnn I0428 11:08:37.302765 3665 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-2rwnn after 0 failed attempt(s) I0428 11:08:37.302615 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-9dg2c I0428 11:08:37.302551 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-2f7qb I0428 11:08:37.302793 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-2f7qb I0428 11:08:37.302625 3665 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-sxvnk I0428 11:08:37.302806 3665 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-2f7qb after 0 failed attempt(s) I0428 11:08:37.302795 3665 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-9dg2c after 0 failed attempt(s) I0428 11:08:37.302809 3665 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-sxvnk I0428 11:08:37.302621 3665 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-dl77t I0428 11:08:37.302826 3665 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-sxvnk after 0 failed attempt(s) I0428 11:08:37.302839 3665 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-dl77t after 0 failed attempt(s) I0428 11:08:37.303350 3665 pods.go:271] [openshift-multus/network-metrics-daemon-rzlqs] addLogicalPort took 747µs, libovsdb time 237.606µs I0428 11:08:37.303368 3665 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-rzlqs after 0 failed attempt(s) I0428 11:08:37.303427 3665 pods.go:271] [openshift-network-diagnostics/network-check-target-p4xdk] addLogicalPort took 701.739µs, libovsdb time 230.882µs I0428 11:08:37.303439 3665 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-p4xdk after 0 failed attempt(s) I0428 11:08:37.315181 3665 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0428 11:08:37.328338 3665 iptables.go:108] Creating table: mangle chain: OUTPUT I0428 11:08:37.332008 3665 iptables.go:108] Creating table: nat chain: OUTPUT I0428 11:08:37.375186 3665 iptables.go:108] Creating table: nat chain: PREROUTING I0428 11:08:37.425558 3665 iptables.go:108] Creating table: filter chain: FORWARD I0428 11:08:37.444313 3665 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0428 11:08:37.444444 3665 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0428 11:08:37.444492 3665 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0428 11:08:37.444570 3665 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0428 11:08:37.444626 3665 gateway_init.go:325] Gateway and management port readiness took 2.20585179s I0428 11:08:37.444653 3665 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0428 11:08:37.444729 3665 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0428 11:08:37.444741 3665 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0428 11:08:37.444750 3665 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0428 11:08:37.444811 3665 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0428 11:08:37.444896 3665 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 54.499µs I0428 11:08:37.445185 3665 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0428 11:08:37.445210 3665 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 15.236µs I0428 11:08:37.445218 3665 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 33.59µs I0428 11:08:37.445225 3665 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0428 11:08:37.445233 3665 nad_controller.go:379] [node-nad-controller NAD controller]: started I0428 11:08:37.445238 3665 default_node_network_controller.go:899] Starting the default node network controller I0428 11:08:37.445249 3665 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0428 11:08:37.445323 3665 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 13.666µs I0428 11:08:37.445432 3665 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0428 11:08:37.445440 3665 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0428 11:08:37.445465 3665 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c01640 0xc0003401c0 0xc000340310 0xc000340380 0xc0003403f0 0xc000340460 0xc0003404d0 0xc000c15a90 0xc0003405b0 0xc000340620 0xc0006e2280 0xc0065f5340 0xc000340690 map[0x2ff9080:0xc002696780 0x2ff95c0:0xc002696a80 0x2ffa2e0:0xc0026966c0 0x2ffa580:0xc002696720 0x2ffad60:0xc0026967e0 0x2ffb000:0xc0026969c0 0x2ffb2a0:0xc002696a20 0x2ffba80:0xc002696ae0 0x2ffbd20:0xc002696b40 0x3026980:0xc0007cf800 0x3026d20:0xc0024835c0 0x3027800:0xc000c85b00 0x3029160:0xc001c3c000 0x30298a0:0xc0022a2fc0 0x302a720:0xc001086840] 0xc000c32b60 0} I0428 11:08:37.445517 3665 gateway.go:151] Starting gateway service sync I0428 11:08:37.446012 3665 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0428 11:08:37.448486 3665 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0428 11:08:37.473043 3665 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0428 11:08:37.474419 3665 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0428 11:08:37.475690 3665 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0428 11:08:37.511449 3665 gateway.go:168] Gateway service sync done. Time taken: 65.919535ms I0428 11:08:37.511926 3665 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30949 I0428 11:08:37.512204 3665 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c01640 0xc0003401c0 0xc000340310 0xc000340380 0xc0003403f0 0xc000340460 0xc0003404d0 0xc000c15a90 0xc0003405b0 0xc000340620 0xc0006e2280 0xc0065f5340 0xc000340690 map[0x2ff9080:0xc002696780 0x2ff95c0:0xc002696a80 0x2ffa2e0:0xc0026966c0 0x2ffa580:0xc002696720 0x2ffad60:0xc0026967e0 0x2ffb000:0xc0026969c0 0x2ffb2a0:0xc002696a20 0x2ffba80:0xc002696ae0 0x2ffbd20:0xc002696b40 0x3026980:0xc0007cf800 0x3026d20:0xc0024835c0 0x3027800:0xc000c85b00 0x3029160:0xc001c3c000 0x30298a0:0xc0022a2fc0 0x302a720:0xc001086840] 0xc000c32b60 0} I0428 11:08:37.512629 3665 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c01640 0xc0003401c0 0xc000340310 0xc000340380 0xc0003403f0 0xc000340460 0xc0003404d0 0xc000c15a90 0xc0003405b0 0xc000340620 0xc0006e2280 0xc0065f5340 0xc000340690 map[0x2ff9080:0xc002696780 0x2ff95c0:0xc002696a80 0x2ffa2e0:0xc0026966c0 0x2ffa580:0xc002696720 0x2ffad60:0xc0026967e0 0x2ffb000:0xc0026969c0 0x2ffb2a0:0xc002696a20 0x2ffba80:0xc002696ae0 0x2ffbd20:0xc002696b40 0x3026980:0xc0007cf800 0x3026d20:0xc0024835c0 0x3027800:0xc000c85b00 0x3029160:0xc001c3c000 0x30298a0:0xc0022a2fc0 0x302a720:0xc001086840] 0xc000c32b60 0} I0428 11:08:37.512665 3665 gateway.go:532] Reconciling gateway with updates I0428 11:08:37.518640 3665 gateway.go:339] Spawning Conntrack Rule Check Thread I0428 11:08:37.518749 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 11:08:37.518761 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 11:08:37.518774 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 11:08:37.518773 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 11:08:37.518788 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 11:08:37.518789 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 11:08:37.518794 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 11:08:37.518816 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 11:08:37.518778 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 11:08:37.518835 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 11:08:37.518838 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0428 11:08:37.518839 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 11:08:37.518848 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0428 11:08:37.518845 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0428 11:08:37.518857 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 11:08:37.518856 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0428 11:08:37.518864 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 11:08:37.518864 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0428 11:08:37.518759 3665 default_node_network_controller.go:944] Gateway and management port readiness took 73.308234ms I0428 11:08:37.518869 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 11:08:37.518874 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 11:08:37.518879 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 11:08:37.518884 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 11:08:37.518883 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0428 11:08:37.518890 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 11:08:37.518889 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 11:08:37.518892 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0428 11:08:37.518892 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0428 11:08:37.518900 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0428 11:08:37.518889 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0428 11:08:37.518904 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 11:08:37.518914 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0428 11:08:37.518915 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 11:08:37.518919 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 11:08:37.518875 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 11:08:37.518917 3665 port_claim.go:71] Svc openshift-ingress/router-default: port 30374 is already open, no action needed I0428 11:08:37.518925 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 11:08:37.518929 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 11:08:37.518932 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 11:08:37.518935 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 11:08:37.518934 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0428 11:08:37.518938 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0428 11:08:37.518874 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0428 11:08:37.518947 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0428 11:08:37.518951 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0428 11:08:37.518954 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0428 11:08:37.518956 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 11:08:37.518954 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0428 11:08:37.518955 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0428 11:08:37.518960 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0428 11:08:37.518970 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0428 11:08:37.518969 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0428 11:08:37.518975 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0428 11:08:37.518980 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0428 11:08:37.518983 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 11:08:37.518965 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 11:08:37.518992 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 11:08:37.518990 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 11:08:37.518920 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 11:08:37.519016 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 11:08:37.518998 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0428 11:08:37.518927 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0428 11:08:37.518924 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 11:08:37.518960 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0428 11:08:37.519017 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 11:08:37.519051 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0428 11:08:37.519053 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 11:08:37.518940 3665 port_claim.go:71] Svc openshift-ingress/router-default: port 32063 is already open, no action needed I0428 11:08:37.519066 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 11:08:37.519076 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0428 11:08:37.518968 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 11:08:37.519087 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 11:08:37.519089 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0428 11:08:37.519051 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0428 11:08:37.518847 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 11:08:37.519135 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 11:08:37.519144 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0428 11:08:37.519172 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0428 11:08:37.518974 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0428 11:08:37.519199 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 11:08:37.519204 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 11:08:37.519209 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 11:08:37.519217 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 11:08:37.519220 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 11:08:37.519223 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0428 11:08:37.519226 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 11:08:37.519230 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 11:08:37.519232 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 11:08:37.519239 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0428 11:08:37.519234 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0428 11:08:37.519238 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 11:08:37.519246 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 11:08:37.519253 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0428 11:08:37.519204 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 11:08:37.519222 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 11:08:37.519311 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 11:08:37.519189 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 11:08:37.519323 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 11:08:37.519218 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 11:08:37.519210 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 11:08:37.519358 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0428 11:08:37.519190 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0428 11:08:37.519211 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0428 11:08:37.519215 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 11:08:37.519406 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0428 11:08:37.519414 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 11:08:37.519423 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0428 11:08:37.519363 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 11:08:37.519430 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0428 11:08:37.519223 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 11:08:37.519450 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 11:08:37.519614 3665 node_ip_handler_linux.go:160] Node IP manager is running W0428 11:08:37.558275 3665 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0428 11:08:37.558403 3665 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0428 11:08:37.558555 3665 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0428 11:08:37.566864 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0428 11:08:37.566913 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0428 11:08:37.566922 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0428 11:08:37.608143 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0428 11:08:37.608174 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0428 11:08:37.608180 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0428 11:08:37.608186 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0428 11:08:37.608193 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0428 11:08:37.650009 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0428 11:08:37.650026 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0428 11:08:37.650015 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0428 11:08:37.650041 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0428 11:08:37.650047 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0428 11:08:37.650058 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0428 11:08:37.650063 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0428 11:08:37.650069 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0428 11:08:37.650089 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0428 11:08:37.650109 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0428 11:08:37.650119 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0428 11:08:37.650129 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0428 11:08:37.650138 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0428 11:08:37.650147 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0428 11:08:37.650158 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0428 11:08:37.650172 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0428 11:08:38.192792 3665 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-109.ec2.internal I0428 11:08:38.192812 3665 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-109.ec2.internal in network "default" I0428 11:08:38.192847 3665 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-109.ec2.internal. Updating no re-route policies I0428 11:08:38.192865 3665 egressfirewall.go:723] Syncing node "ip-10-0-0-109.ec2.internal" for egress firewall I0428 11:08:38.192885 3665 node_tracker.go:169] Node ip-10-0-0-109.ec2.internal switch + router changed, syncing services in network "default" I0428 11:08:38.192903 3665 services_controller.go:574] Full service sync requested for network=default I0428 11:08:38.192816 3665 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-109.ec2.internal: 29.543µs W0428 11:08:39.317525 3665 egressip_healthcheck.go:76] Health checking using insecure connection I0428 11:08:39.317586 3665 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0428 11:08:39.317594 3665 default_node_network_controller.go:1380] Starting node controller node sync I0428 11:08:39.329899 3665 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 12.286192ms I0428 11:08:39.330074 3665 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-109.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 11:08:39.330087 3665 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-109.ec2.internal", IP: 10.0.0.109 to PMTUD blocking rules I0428 11:08:39.336143 3665 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0428 11:08:39.336415 3665 egressservice_node.go:93] Setting up event handlers for Egress Services I0428 11:08:39.336527 3665 egressservice_node.go:183] Starting Egress Services Controller I0428 11:08:39.336548 3665 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0428 11:08:39.336558 3665 shared_informer.go:356] "Caches are synced" controller="egressservices" I0428 11:08:39.336564 3665 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0428 11:08:39.336569 3665 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0428 11:08:39.336575 3665 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0428 11:08:39.336580 3665 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0428 11:08:39.336583 3665 egressservice_node.go:197] Repairing Egress Services I0428 11:08:39.347491 3665 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0428 11:08:39.347506 3665 external_controller.go:278] Starting Admin Policy Based Route Controller I0428 11:08:39.347653 3665 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0428 11:08:39.347668 3665 link_network_manager.go:119] Link manager is running I0428 11:08:39.347684 3665 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0428 11:08:39.347868 3665 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0428 11:08:39.348310 3665 vrf_manager.go:128] VRF manager is running I0428 11:08:39.348540 3665 node_controller_manager.go:437] Removing flows to drop GARP I0428 11:08:39.348553 3665 gateway.go:532] Reconciling gateway with updates I0428 11:08:39.372035 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0428 11:08:39.372056 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0428 11:08:39.372057 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 11:08:39.372067 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0428 11:08:39.372069 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 11:08:39.372071 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0428 11:08:39.372076 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 11:08:39.372077 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0428 11:08:39.372085 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0428 11:08:39.372093 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 11:08:39.372101 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0428 11:08:39.372104 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 11:08:39.372108 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 11:08:39.372115 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 11:08:39.372120 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 11:08:39.372121 3665 port_claim.go:71] Svc openshift-ingress/router-default: port 30374 is already open, no action needed I0428 11:08:39.372126 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 11:08:39.372102 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 11:08:39.372134 3665 port_claim.go:71] Svc openshift-ingress/router-default: port 32063 is already open, no action needed I0428 11:08:39.372134 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 11:08:39.372138 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 11:08:39.372144 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 11:08:39.372148 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 11:08:39.372151 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0428 11:08:39.372161 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 11:08:39.372162 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 11:08:39.372172 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0428 11:08:39.372170 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0428 11:08:39.372043 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 11:08:39.372183 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 11:08:39.372187 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0428 11:08:39.372060 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 11:08:39.372093 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0428 11:08:39.372198 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 11:08:39.372206 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 11:08:39.372208 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0428 11:08:39.372208 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0428 11:08:39.372218 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 11:08:39.372166 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0428 11:08:39.372133 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 11:08:39.372233 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0428 11:08:39.372236 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0428 11:08:39.372234 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0428 11:08:39.372220 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0428 11:08:39.372250 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 11:08:39.372259 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0428 11:08:39.372245 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0428 11:08:39.372207 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 11:08:39.372270 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0428 11:08:39.372273 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 11:08:39.372035 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 11:08:39.372281 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 11:08:39.372285 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 11:08:39.372288 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 11:08:39.372294 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 11:08:39.372288 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 11:08:39.372297 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 11:08:39.372302 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 11:08:39.372305 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 11:08:39.372310 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0428 11:08:39.372314 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 11:08:39.372335 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0428 11:08:39.372278 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 11:08:39.372352 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0428 11:08:39.372361 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 11:08:39.372384 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 11:08:39.372260 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0428 11:08:39.372400 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 11:08:39.372415 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 11:08:39.372338 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0428 11:08:39.372413 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 11:08:39.372421 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0428 11:08:39.372427 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 11:08:39.372135 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 11:08:39.372435 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 11:08:39.372151 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0428 11:08:39.372080 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 11:08:39.372159 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0428 11:08:39.372512 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0428 11:08:39.372183 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 11:08:39.372536 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0428 11:08:39.372181 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 11:08:39.372553 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 11:08:39.372193 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 11:08:39.372602 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0428 11:08:39.372207 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 11:08:39.372612 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 11:08:39.372205 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 11:08:39.372637 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 11:08:39.372226 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0428 11:08:39.372693 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0428 11:08:39.372227 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 11:08:39.372701 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 11:08:39.372737 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0428 11:08:39.372236 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 11:08:39.372242 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 11:08:39.372245 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0428 11:08:39.372797 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0428 11:08:39.372264 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 11:08:39.372804 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0428 11:08:39.372278 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0428 11:08:39.372811 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0428 11:08:39.372294 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 11:08:39.372300 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 11:08:39.372313 3665 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 11:08:39.372891 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 11:08:39.372318 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 11:08:39.372322 3665 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0428 11:08:39.413993 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0428 11:08:39.414013 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0428 11:08:39.414020 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0428 11:08:39.414047 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0428 11:08:39.414055 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0428 11:08:39.414060 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0428 11:08:39.414066 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0428 11:08:39.460683 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0428 11:08:39.460741 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0428 11:08:39.511166 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0428 11:08:39.511198 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0428 11:08:39.511206 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0428 11:08:39.511215 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0428 11:08:39.511224 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0428 11:08:39.511230 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0428 11:08:39.511236 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0428 11:08:39.511242 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0428 11:08:39.511248 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0428 11:08:39.511254 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0428 11:08:39.511261 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0428 11:08:39.511266 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0428 11:08:39.511275 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0428 11:08:39.511280 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0428 11:08:39.511285 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0428 11:08:39.511291 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0428 11:08:39.511299 3665 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0428 11:08:40.815114 3665 egressfirewall.go:723] Syncing node "ip-10-0-1-135.ec2.internal" for egress firewall I0428 11:08:40.815204 3665 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-135.ec2.internal I0428 11:08:40.815226 3665 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-135.ec2.internal: 27.738µs I0428 11:08:40.815229 3665 default_network_controller.go:1025] Node "ip-10-0-1-135.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 11:08:41.421613 3665 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-140.ec2.internal I0428 11:08:41.421675 3665 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-140.ec2.internal: 70.106µs I0428 11:08:41.452637 3665 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-dnbv5] creating logical port openshift-service-ca-operator_service-ca-operator-bc9564db4-dnbv5 for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.452787 3665 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-bc9564db4-dnbv5 I0428 11:08:41.459045 3665 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-677c7cc65b-qsgvf] creating logical port openshift-image-registry_image-registry-677c7cc65b-qsgvf for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.459191 3665 kube.go:256] Updating pod openshift-image-registry/image-registry-677c7cc65b-qsgvf I0428 11:08:41.459173 3665 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-wnsqk] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-wnsqk for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.459394 3665 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-wnsqk I0428 11:08:41.460871 3665 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-d6tqp] creating logical port openshift-insights_insights-operator-6dbb575d6c-d6tqp for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.460991 3665 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mk62j] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67f4d568c7-mk62j for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.461062 3665 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-d6tqp I0428 11:08:41.461075 3665 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mk62j I0428 11:08:41.464205 3665 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-6xn42] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-6xn42 for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.464307 3665 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-6xn42 I0428 11:08:41.477044 3665 pod.go:62] [openshift-service-ca-operator/service-ca-operator-bc9564db4-dnbv5] pod update took 24.298138ms I0428 11:08:41.477069 3665 base_network_controller_pods.go:951] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-dnbv5] addLogicalPort annotation time took 24.325909ms I0428 11:08:41.478412 3665 pods.go:271] [openshift-service-ca-operator/service-ca-operator-bc9564db4-dnbv5] addLogicalPort took 25.790069ms, libovsdb time 1.029651ms I0428 11:08:41.478481 3665 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-f98sw] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-f98sw for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.478550 3665 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-f98sw I0428 11:08:41.488212 3665 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-h4fmf] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-h4fmf for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.488372 3665 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-h4fmf I0428 11:08:41.492460 3665 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-wnsqk] pod update took 33.095168ms I0428 11:08:41.492484 3665 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-wnsqk] addLogicalPort annotation time took 33.123823ms I0428 11:08:41.493707 3665 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mk62j] pod update took 32.657247ms I0428 11:08:41.493705 3665 pod.go:62] [openshift-image-registry/image-registry-677c7cc65b-qsgvf] pod update took 34.554057ms I0428 11:08:41.493749 3665 base_network_controller_pods.go:951] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mk62j] addLogicalPort annotation time took 32.701322ms I0428 11:08:41.493752 3665 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-677c7cc65b-qsgvf] addLogicalPort annotation time took 34.60458ms I0428 11:08:41.493912 3665 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-wnsqk] addLogicalPort took 34.751359ms, libovsdb time 1.103291ms I0428 11:08:41.494030 3665 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-6xn42] pod update took 29.750895ms I0428 11:08:41.494043 3665 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5bd84bbbf9-b6fhm] creating logical port openshift-ingress_router-default-5bd84bbbf9-b6fhm for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.494044 3665 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-d6tqp] pod update took 33.017984ms I0428 11:08:41.494050 3665 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-6xn42] addLogicalPort annotation time took 29.771344ms I0428 11:08:41.494060 3665 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-d6tqp] addLogicalPort annotation time took 33.033332ms I0428 11:08:41.494139 3665 kube.go:256] Updating pod openshift-ingress/router-default-5bd84bbbf9-b6fhm I0428 11:08:41.495277 3665 pods.go:271] [openshift-image-registry/image-registry-677c7cc65b-qsgvf] addLogicalPort took 36.251896ms, libovsdb time 1.143614ms I0428 11:08:41.495366 3665 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-zh5kn] creating logical port openshift-console-operator_console-operator-66dbbb6468-zh5kn for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.495441 3665 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-zh5kn I0428 11:08:41.495542 3665 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-d6tqp] addLogicalPort took 34.670058ms, libovsdb time 1.226464ms I0428 11:08:41.495589 3665 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-c45c8f597-g4js8] creating logical port openshift-ingress_router-default-c45c8f597-g4js8 for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.495690 3665 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mk62j] addLogicalPort took 34.712789ms, libovsdb time 1.525866ms I0428 11:08:41.495753 3665 kube.go:256] Updating pod openshift-ingress/router-default-c45c8f597-g4js8 I0428 11:08:41.495775 3665 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lkkdq] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-6944588dcc-lkkdq for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.495855 3665 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lkkdq I0428 11:08:41.496026 3665 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-6xn42] addLogicalPort took 31.834091ms, libovsdb time 1.742324ms I0428 11:08:41.517265 3665 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-109.ec2.internal I0428 11:08:41.517297 3665 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-109.ec2.internal: 105.508µs I0428 11:08:41.517357 3665 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-h4fmf] pod update took 29.009847ms I0428 11:08:41.517381 3665 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-h4fmf] addLogicalPort annotation time took 29.044673ms I0428 11:08:41.518617 3665 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-h4fmf] addLogicalPort took 30.454048ms, libovsdb time 936.249µs I0428 11:08:41.520760 3665 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-82bbk] creating logical port openshift-dns_dns-default-82bbk for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.521119 3665 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-zh5kn] pod update took 25.698806ms I0428 11:08:41.521139 3665 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-zh5kn] addLogicalPort annotation time took 25.723872ms I0428 11:08:41.521326 3665 kube.go:256] Updating pod openshift-dns/dns-default-82bbk I0428 11:08:41.522414 3665 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-zh5kn] addLogicalPort took 27.055941ms, libovsdb time 821.495µs I0428 11:08:41.526920 3665 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-f98sw] pod update took 48.389976ms I0428 11:08:41.526922 3665 pod.go:62] [openshift-ingress/router-default-5bd84bbbf9-b6fhm] pod update took 32.824557ms I0428 11:08:41.526942 3665 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-f98sw] addLogicalPort annotation time took 48.413454ms I0428 11:08:41.526980 3665 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-5bd84bbbf9-b6fhm] addLogicalPort annotation time took 32.857126ms I0428 11:08:41.527170 3665 pod.go:62] [openshift-ingress/router-default-c45c8f597-g4js8] pod update took 31.461696ms I0428 11:08:41.527194 3665 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-c45c8f597-g4js8] addLogicalPort annotation time took 31.486651ms I0428 11:08:41.529191 3665 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-f98sw] addLogicalPort took 50.717188ms, libovsdb time 1.893463ms I0428 11:08:41.533476 3665 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lkkdq] pod update took 37.640099ms I0428 11:08:41.533548 3665 base_network_controller_pods.go:951] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lkkdq] addLogicalPort annotation time took 37.724844ms I0428 11:08:41.533918 3665 pods.go:271] [openshift-ingress/router-default-c45c8f597-g4js8] addLogicalPort took 38.335072ms, libovsdb time 4.530974ms I0428 11:08:41.534072 3665 pods.go:271] [openshift-ingress/router-default-5bd84bbbf9-b6fhm] addLogicalPort took 40.098155ms, libovsdb time 4.664344ms I0428 11:08:41.536582 3665 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lkkdq] addLogicalPort took 40.816828ms, libovsdb time 2.087868ms I0428 11:08:41.555892 3665 pod.go:62] [openshift-dns/dns-default-82bbk] pod update took 35.026448ms I0428 11:08:41.555913 3665 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-82bbk] addLogicalPort annotation time took 35.051535ms I0428 11:08:41.557029 3665 pods.go:271] [openshift-dns/dns-default-82bbk] addLogicalPort took 36.285741ms, libovsdb time 839.8µs I0428 11:08:41.568243 3665 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-rd5sg] creating logical port openshift-ingress-canary_ingress-canary-rd5sg for pod on switch ip-10-0-2-140.ec2.internal I0428 11:08:41.568353 3665 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-rd5sg I0428 11:08:41.600507 3665 pod.go:62] [openshift-ingress-canary/ingress-canary-rd5sg] pod update took 32.17976ms I0428 11:08:41.600534 3665 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-rd5sg] addLogicalPort annotation time took 32.211293ms I0428 11:08:41.602007 3665 pods.go:271] [openshift-ingress-canary/ingress-canary-rd5sg] addLogicalPort took 33.771246ms, libovsdb time 1.06977ms I0428 11:08:42.127898 3665 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-d6tqp 4f1e3d4e33d65c90eee87db4338f264e6df3c66e1a3f407a570545e50ad16e3e network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-d6tqp 4f1e3d4e33d65c90eee87db4338f264e6df3c66e1a3f407a570545e50ad16e3e network default NAD default NAD key ] I0428 11:08:42.131134 3665 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-zh5kn a5809d0b65e877d974cc4e202bcb6df53bf4aca159f01e1e930395a1895223c1 network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-zh5kn a5809d0b65e877d974cc4e202bcb6df53bf4aca159f01e1e930395a1895223c1 network default NAD default NAD key ] I0428 11:08:42.131851 3665 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-f98sw 3a6e26e64e313563d3cc08f6b87c458e0caac63e7147d21a7800ad73da38188c network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-f98sw 3a6e26e64e313563d3cc08f6b87c458e0caac63e7147d21a7800ad73da38188c network default NAD default NAD key ] I0428 11:08:42.131895 3665 cni.go:452] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lkkdq 9cc9bb4ed95c4027727cc50b769063601f6b7120cf80ef206848627c317788b7 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lkkdq 9cc9bb4ed95c4027727cc50b769063601f6b7120cf80ef206848627c317788b7 network default NAD default NAD key ] I0428 11:08:42.132750 3665 cni.go:452] [openshift-ingress-canary/ingress-canary-rd5sg fee518c8233502a1fd86c30f6d41f831b5786d02e1612f1626ef768f8f4fafd2 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-rd5sg fee518c8233502a1fd86c30f6d41f831b5786d02e1612f1626ef768f8f4fafd2 network default NAD default NAD key ] I0428 11:08:42.134022 3665 cni.go:452] [openshift-service-ca-operator/service-ca-operator-bc9564db4-dnbv5 c6af2f3332d501cfc93e9953a9789f910b67edc26f6acd35552bef4a17cd1068 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-dnbv5 c6af2f3332d501cfc93e9953a9789f910b67edc26f6acd35552bef4a17cd1068 network default NAD default NAD key ] I0428 11:08:42.173389 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-6944588dcc-lkkdq, hostIfaceName: 9cc9bb4ed95c402, network: default, NAD default, SandboxID: "9cc9bb4ed95c4027727cc50b769063601f6b7120cf80ef206848627c317788b7", PCI device ID: , UID: "b1586241-0cee-46ec-8984-0deaa5695624", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0428 11:08:42.184353 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-f98sw, hostIfaceName: 3a6e26e64e31356, network: default, NAD default, SandboxID: "3a6e26e64e313563d3cc08f6b87c458e0caac63e7147d21a7800ad73da38188c", PCI device ID: , UID: "4fd0a17e-5938-46da-a610-376a7932d2b4", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0428 11:08:42.186844 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-zh5kn, hostIfaceName: a5809d0b65e877d, network: default, NAD default, SandboxID: "a5809d0b65e877d974cc4e202bcb6df53bf4aca159f01e1e930395a1895223c1", PCI device ID: , UID: "4df9c73d-3233-43a3-889e-8e7c12e256ca", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0428 11:08:42.189262 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-rd5sg, hostIfaceName: fee518c8233502a, network: default, NAD default, SandboxID: "fee518c8233502a1fd86c30f6d41f831b5786d02e1612f1626ef768f8f4fafd2", PCI device ID: , UID: "f2654a42-2889-4e0f-8e1f-f5835ad4cf3a", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0428 11:08:42.190222 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-d6tqp, hostIfaceName: 4f1e3d4e33d65c9, network: default, NAD default, SandboxID: "4f1e3d4e33d65c90eee87db4338f264e6df3c66e1a3f407a570545e50ad16e3e", PCI device ID: , UID: "38c2f9c1-4f87-42d6-86ef-c16846a35af7", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0428 11:08:42.190949 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-bc9564db4-dnbv5, hostIfaceName: c6af2f3332d501c, network: default, NAD default, SandboxID: "c6af2f3332d501cfc93e9953a9789f910b67edc26f6acd35552bef4a17cd1068", PCI device ID: , UID: "3c7a9f8e-bef8-49ea-88bb-710860e15aa7", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0428 11:08:42.240268 3665 cni.go:473] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lkkdq 9cc9bb4ed95c4027727cc50b769063601f6b7120cf80ef206848627c317788b7 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lkkdq 9cc9bb4ed95c4027727cc50b769063601f6b7120cf80ef206848627c317788b7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:b5:bf:89:b3:40\",\"name\":\"9cc9bb4ed95c402\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/91a63043-64d7-40f6-9688-4ece0d8803fe\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:08:42.249905 3665 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-f98sw 3a6e26e64e313563d3cc08f6b87c458e0caac63e7147d21a7800ad73da38188c network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-f98sw 3a6e26e64e313563d3cc08f6b87c458e0caac63e7147d21a7800ad73da38188c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:0b:c0:2c:2c:3e\",\"name\":\"3a6e26e64e31356\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7fd5657c-b515-47ce-b33b-99d1ad94b365\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:08:42.254900 3665 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-zh5kn a5809d0b65e877d974cc4e202bcb6df53bf4aca159f01e1e930395a1895223c1 network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-zh5kn a5809d0b65e877d974cc4e202bcb6df53bf4aca159f01e1e930395a1895223c1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:28:20:13:1b:5f\",\"name\":\"a5809d0b65e877d\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/00e30610-cc26-487e-8638-e7ab55aa547f\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:08:42.260128 3665 cni.go:473] [openshift-service-ca-operator/service-ca-operator-bc9564db4-dnbv5 c6af2f3332d501cfc93e9953a9789f910b67edc26f6acd35552bef4a17cd1068 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-dnbv5 c6af2f3332d501cfc93e9953a9789f910b67edc26f6acd35552bef4a17cd1068 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:c3:f6:01:1b:2b\",\"name\":\"c6af2f3332d501c\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dc2daa5f-e8e0-45fc-9639-9a4cd7fd4e7e\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:08:42.260208 3665 cni.go:473] [openshift-ingress-canary/ingress-canary-rd5sg fee518c8233502a1fd86c30f6d41f831b5786d02e1612f1626ef768f8f4fafd2 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-rd5sg fee518c8233502a1fd86c30f6d41f831b5786d02e1612f1626ef768f8f4fafd2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:8d:8c:d7:4b:93\",\"name\":\"fee518c8233502a\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a9d76342-7a48-4650-9a94-f775676b1540\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:08:42.260230 3665 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-d6tqp 4f1e3d4e33d65c90eee87db4338f264e6df3c66e1a3f407a570545e50ad16e3e network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-d6tqp 4f1e3d4e33d65c90eee87db4338f264e6df3c66e1a3f407a570545e50ad16e3e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:07:1a:18:66:b8\",\"name\":\"4f1e3d4e33d65c9\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fb861bc0-75ec-43ad-a377-a461a24ae8c8\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:08:46.943288 3665 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-135.ec2.internal in network "default" I0428 11:08:46.943320 3665 default_network_controller.go:1025] Node "ip-10-0-1-135.ec2.internal" in remote zone "ip-10-0-1-135.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 11:08:46.943335 3665 node_tracker.go:235] Node ip-10-0-1-135.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-135.ec2.internal" I0428 11:08:46.943350 3665 node_tracker.go:169] Node ip-10-0-1-135.ec2.internal switch + router changed, syncing services in network "default" I0428 11:08:46.943366 3665 services_controller.go:574] Full service sync requested for network=default I0428 11:08:47.075982 3665 default_network_controller.go:1025] Node "ip-10-0-1-135.ec2.internal" in remote zone "ip-10-0-1-135.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 11:08:47.076018 3665 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-135.ec2.internal in network "default" I0428 11:08:47.076070 3665 node_tracker.go:169] Node ip-10-0-1-135.ec2.internal switch + router changed, syncing services in network "default" I0428 11:08:47.076084 3665 services_controller.go:574] Full service sync requested for network=default I0428 11:08:47.077016 3665 zone_ic_handler.go:195] Time taken to create transit switch: 328.026µs I0428 11:08:47.077039 3665 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-135.ec2.internal for the network default I0428 11:08:47.078046 3665 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-135.ec2.internal" on network "default" took: 1.373476ms I0428 11:08:47.749209 3665 cni.go:452] [openshift-network-diagnostics/network-check-target-p4xdk e953b93be3a8bad9e0385f79dc75f56b817b74edd43e0f93fdce689d7afbf689 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-p4xdk e953b93be3a8bad9e0385f79dc75f56b817b74edd43e0f93fdce689d7afbf689 network default NAD default NAD key ] I0428 11:08:47.759414 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-p4xdk, hostIfaceName: e953b93be3a8bad, network: default, NAD default, SandboxID: "e953b93be3a8bad9e0385f79dc75f56b817b74edd43e0f93fdce689d7afbf689", PCI device ID: , UID: "d0766aa3-2a65-4752-ba62-4840198bfb34", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0428 11:08:47.815187 3665 cni.go:473] [openshift-network-diagnostics/network-check-target-p4xdk e953b93be3a8bad9e0385f79dc75f56b817b74edd43e0f93fdce689d7afbf689 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-p4xdk e953b93be3a8bad9e0385f79dc75f56b817b74edd43e0f93fdce689d7afbf689 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:22:ad:8e:da:a0\",\"name\":\"e953b93be3a8bad\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/517808c8-1e58-42f0-9dd4-881c2f44c50e\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:08:48.337881 3665 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0428 11:08:48.736061 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0428 11:08:49.271486 3665 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0428 11:08:49.272585 3665 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 1.068629ms I0428 11:08:49.296530 3665 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0428 11:08:49.630707 3665 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-135.ec2.internal I0428 11:08:49.630748 3665 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-135.ec2.internal: 46.328µs I0428 11:08:49.630760 3665 egressfirewall.go:723] Syncing node "ip-10-0-1-135.ec2.internal" for egress firewall I0428 11:08:49.630782 3665 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-135.ec2.internal in network "default" I0428 11:08:49.630837 3665 node_tracker.go:169] Node ip-10-0-1-135.ec2.internal switch + router changed, syncing services in network "default" I0428 11:08:49.630857 3665 services_controller.go:574] Full service sync requested for network=default I0428 11:08:49.630855 3665 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-135.ec2.internal", IP: 10.0.1.135 to PMTUD blocking rules I0428 11:08:49.630874 3665 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-135.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0428 11:08:49.631010 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0428 11:08:49.631564 3665 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 11:08:49.637187 3665 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-135.ec2.internal. Updating no re-route policies I0428 11:08:49.648067 3665 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-135.ec2.internal", IP: 10.0.1.135 to PMTUD blocking rules I0428 11:08:49.648085 3665 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-135.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 11:08:50.159609 3665 namespace.go:98] [openshift-service-ca] adding namespace I0428 11:08:50.160622 3665 namespace.go:102] [openshift-service-ca] adding namespace took 991.522µs I0428 11:08:50.180741 3665 namespace.go:142] [openshift-service-ca] updating namespace W0428 11:08:51.714733 3665 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 11:08:53.224792 3665 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-135.ec2.internal I0428 11:08:53.224822 3665 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-135.ec2.internal: 36.218µs I0428 11:08:57.525862 3665 cni.go:452] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mk62j a6b84ed4f202c9755884c940a0396813fcbb09b042dcfe505ac703edeb607e17 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mk62j a6b84ed4f202c9755884c940a0396813fcbb09b042dcfe505ac703edeb607e17 network default NAD default NAD key ] I0428 11:08:57.537119 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67f4d568c7-mk62j, hostIfaceName: a6b84ed4f202c97, network: default, NAD default, SandboxID: "a6b84ed4f202c9755884c940a0396813fcbb09b042dcfe505ac703edeb607e17", PCI device ID: , UID: "61e6dd50-41d0-44d2-bad1-c9606b5a5498", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0428 11:08:57.587903 3665 cni.go:473] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mk62j a6b84ed4f202c9755884c940a0396813fcbb09b042dcfe505ac703edeb607e17 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-mk62j a6b84ed4f202c9755884c940a0396813fcbb09b042dcfe505ac703edeb607e17 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:10:12:05:8a:e9\",\"name\":\"a6b84ed4f202c97\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/aad860c6-6dc9-4591-8508-0925e5c5e38f\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:08:58.461703 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0428 11:09:13.664850 3665 cni.go:452] [openshift-image-registry/image-registry-677c7cc65b-qsgvf 42e3690655a6c395817b7e88c26089a86e1ac657dfd7c68a0cc80817bd3ceecf network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-677c7cc65b-qsgvf 42e3690655a6c395817b7e88c26089a86e1ac657dfd7c68a0cc80817bd3ceecf network default NAD default NAD key ] I0428 11:09:13.672867 3665 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-wnsqk 4ff4a0d86cb8251a7b23bba31eb4100b15bf46a51260a6cf97330d538524f335 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-wnsqk 4ff4a0d86cb8251a7b23bba31eb4100b15bf46a51260a6cf97330d538524f335 network default NAD default NAD key ] I0428 11:09:13.676373 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-677c7cc65b-qsgvf, hostIfaceName: 42e3690655a6c39, network: default, NAD default, SandboxID: "42e3690655a6c395817b7e88c26089a86e1ac657dfd7c68a0cc80817bd3ceecf", PCI device ID: , UID: "26952bac-fc40-4c36-83fa-4e2b1cb4ba75", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0428 11:09:13.701038 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-wnsqk, hostIfaceName: 4ff4a0d86cb8251, network: default, NAD default, SandboxID: "4ff4a0d86cb8251a7b23bba31eb4100b15bf46a51260a6cf97330d538524f335", PCI device ID: , UID: "831875b0-8b55-433a-a8b4-1a1dad6fd0b1", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0428 11:09:13.743506 3665 cni.go:473] [openshift-image-registry/image-registry-677c7cc65b-qsgvf 42e3690655a6c395817b7e88c26089a86e1ac657dfd7c68a0cc80817bd3ceecf network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-677c7cc65b-qsgvf 42e3690655a6c395817b7e88c26089a86e1ac657dfd7c68a0cc80817bd3ceecf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:1d:27:01:92:62\",\"name\":\"42e3690655a6c39\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ba821a65-b99d-4145-82d2-9bb43c37e8fe\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:09:13.744036 3665 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-6xn42 fd8ca99fad765d58fbd1112e88b6cee95d97eb017e664070379200364a50ce8e network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-6xn42 fd8ca99fad765d58fbd1112e88b6cee95d97eb017e664070379200364a50ce8e network default NAD default NAD key ] I0428 11:09:13.753415 3665 cni.go:452] [openshift-ingress/router-default-c45c8f597-g4js8 e78ef727a09a8120533947cdbc98e98b30fa448c6b92997506cbe7b7f1e1a778 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-c45c8f597-g4js8 e78ef727a09a8120533947cdbc98e98b30fa448c6b92997506cbe7b7f1e1a778 network default NAD default NAD key ] I0428 11:09:13.753666 3665 cni.go:452] [openshift-ingress/router-default-5bd84bbbf9-b6fhm f8dadc2ea5ad670291f4195295ba81e248519079734d27074fe92fcbd161a165 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-5bd84bbbf9-b6fhm f8dadc2ea5ad670291f4195295ba81e248519079734d27074fe92fcbd161a165 network default NAD default NAD key ] I0428 11:09:13.757599 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-6xn42, hostIfaceName: fd8ca99fad765d5, network: default, NAD default, SandboxID: "fd8ca99fad765d58fbd1112e88b6cee95d97eb017e664070379200364a50ce8e", PCI device ID: , UID: "c36b9dab-a3e8-4412-9884-2ce567bca045", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0428 11:09:13.764938 3665 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-wnsqk 4ff4a0d86cb8251a7b23bba31eb4100b15bf46a51260a6cf97330d538524f335 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-wnsqk 4ff4a0d86cb8251a7b23bba31eb4100b15bf46a51260a6cf97330d538524f335 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:04:1a:1a:5f:2d\",\"name\":\"4ff4a0d86cb8251\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b9d699df-0db3-4087-910f-463a639ac701\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:09:13.765850 3665 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-h4fmf 097f84135e0a743b0d60e7a2ff83f8ce349629122d6a52d343023ec84776418d network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-h4fmf 097f84135e0a743b0d60e7a2ff83f8ce349629122d6a52d343023ec84776418d network default NAD default NAD key ] I0428 11:09:13.779175 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-c45c8f597-g4js8, hostIfaceName: e78ef727a09a812, network: default, NAD default, SandboxID: "e78ef727a09a8120533947cdbc98e98b30fa448c6b92997506cbe7b7f1e1a778", PCI device ID: , UID: "15de84aa-dd44-49c3-a92d-dc246210a12f", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0428 11:09:13.779369 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5bd84bbbf9-b6fhm, hostIfaceName: f8dadc2ea5ad670, network: default, NAD default, SandboxID: "f8dadc2ea5ad670291f4195295ba81e248519079734d27074fe92fcbd161a165", PCI device ID: , UID: "c6334445-f595-4de0-850f-ab1dd0337903", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0428 11:09:13.779537 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-h4fmf, hostIfaceName: 097f84135e0a743, network: default, NAD default, SandboxID: "097f84135e0a743b0d60e7a2ff83f8ce349629122d6a52d343023ec84776418d", PCI device ID: , UID: "fd1dae16-e474-4520-88b9-8cfe93922d0c", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0428 11:09:13.800326 3665 cni.go:452] [openshift-dns/dns-default-82bbk d933f9765927765ade1ea1aed8877e73dc474de6140e08807169844050e4145e network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-82bbk d933f9765927765ade1ea1aed8877e73dc474de6140e08807169844050e4145e network default NAD default NAD key ] I0428 11:09:13.816146 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-82bbk, hostIfaceName: d933f9765927765, network: default, NAD default, SandboxID: "d933f9765927765ade1ea1aed8877e73dc474de6140e08807169844050e4145e", PCI device ID: , UID: "b11570d7-63ee-4334-b8b9-db130081b7e4", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0428 11:09:13.824091 3665 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-6xn42 fd8ca99fad765d58fbd1112e88b6cee95d97eb017e664070379200364a50ce8e network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-6xn42 fd8ca99fad765d58fbd1112e88b6cee95d97eb017e664070379200364a50ce8e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:2d:6b:a6:23:b0\",\"name\":\"fd8ca99fad765d5\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/08e3572b-23cf-4a3f-9152-40633353b3e1\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:09:13.841924 3665 cni.go:473] [openshift-ingress/router-default-c45c8f597-g4js8 e78ef727a09a8120533947cdbc98e98b30fa448c6b92997506cbe7b7f1e1a778 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-c45c8f597-g4js8 e78ef727a09a8120533947cdbc98e98b30fa448c6b92997506cbe7b7f1e1a778 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:f7:5a:f2:bd:43\",\"name\":\"e78ef727a09a812\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fb2b87a4-be5c-4098-9fa7-c4a0287347e9\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:09:13.892855 3665 cni.go:473] [openshift-dns/dns-default-82bbk d933f9765927765ade1ea1aed8877e73dc474de6140e08807169844050e4145e network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-82bbk d933f9765927765ade1ea1aed8877e73dc474de6140e08807169844050e4145e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:dc:ec:1b:ea:e1\",\"name\":\"d933f9765927765\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d42fb71a-6c14-4dfe-bdf5-9186a946d3bf\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:09:14.066671 3665 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-h4fmf 097f84135e0a743b0d60e7a2ff83f8ce349629122d6a52d343023ec84776418d network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-h4fmf 097f84135e0a743b0d60e7a2ff83f8ce349629122d6a52d343023ec84776418d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:36:2f:6a:f9:86\",\"name\":\"097f84135e0a743\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f3a74b61-ac22-4d98-b285-06f9919a313b\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:09:14.068425 3665 cni.go:473] [openshift-ingress/router-default-5bd84bbbf9-b6fhm f8dadc2ea5ad670291f4195295ba81e248519079734d27074fe92fcbd161a165 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-5bd84bbbf9-b6fhm f8dadc2ea5ad670291f4195295ba81e248519079734d27074fe92fcbd161a165 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:16:e6:0b:79:ea\",\"name\":\"f8dadc2ea5ad670\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/af66fcab-2d4d-46ad-a3c5-6fff0e21343f\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:09:14.976806 3665 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:09:14.977010 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0428 11:09:15.009880 3665 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 11:09:15.039756 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 11:09:15.039781 3665 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 11:09:15.039786 3665 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 11:09:15.081060 3665 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0428 11:09:15.081089 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0428 11:09:15.083812 3665 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 11:09:15.117589 3665 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-kqrn2] creating logical port openshift-insights_insights-runtime-extractor-kqrn2 for pod on switch ip-10-0-2-140.ec2.internal I0428 11:09:15.117786 3665 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-kqrn2 I0428 11:09:15.134829 3665 pod.go:62] [openshift-insights/insights-runtime-extractor-kqrn2] pod update took 17.088217ms I0428 11:09:15.134861 3665 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-kqrn2] addLogicalPort annotation time took 17.146073ms I0428 11:09:15.136335 3665 pods.go:271] [openshift-insights/insights-runtime-extractor-kqrn2] addLogicalPort took 18.767745ms, libovsdb time 970.731µs W0428 11:09:15.154840 3665 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:09:15.182270 3665 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:09:15.447210 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0428 11:09:15.447238 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0428 11:09:15.447234 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 11:09:15.447254 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 11:09:15.447261 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 11:09:15.447281 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 11:09:15.447289 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 11:09:15.447296 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 11:09:15.447354 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0428 11:09:15.447370 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0428 11:09:15.447381 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0428 11:09:15.447387 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0428 11:09:15.623395 3665 cni.go:452] [openshift-insights/insights-runtime-extractor-kqrn2 33b2e50a0a7c24231486f74ac8342faf869a6ada97c4d6caa9f1418feb5cfa81 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-kqrn2 33b2e50a0a7c24231486f74ac8342faf869a6ada97c4d6caa9f1418feb5cfa81 network default NAD default NAD key ] I0428 11:09:15.635820 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-kqrn2, hostIfaceName: 33b2e50a0a7c242, network: default, NAD default, SandboxID: "33b2e50a0a7c24231486f74ac8342faf869a6ada97c4d6caa9f1418feb5cfa81", PCI device ID: , UID: "d7bf1f94-4705-47af-ac70-650117c213b6", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0428 11:09:15.693252 3665 cni.go:473] [openshift-insights/insights-runtime-extractor-kqrn2 33b2e50a0a7c24231486f74ac8342faf869a6ada97c4d6caa9f1418feb5cfa81 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-kqrn2 33b2e50a0a7c24231486f74ac8342faf869a6ada97c4d6caa9f1418feb5cfa81 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:05:24:f1:65:fe\",\"name\":\"33b2e50a0a7c242\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f9588656-99c6-47e3-a096-173118ee83d5\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:09:15.856697 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0428 11:09:15.856748 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0428 11:09:15.946661 3665 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0428 11:09:15.960774 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 11:09:15.960791 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 11:09:15.960797 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default I0428 11:09:16.085773 3665 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0428 11:09:16.730968 3665 cni.go:452] [openshift-multus/network-metrics-daemon-rzlqs f03183dcd0db83560ff3fb7e45a70b97897687a7122763477bfb7b31f403253d network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-rzlqs f03183dcd0db83560ff3fb7e45a70b97897687a7122763477bfb7b31f403253d network default NAD default NAD key ] I0428 11:09:16.741152 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-rzlqs, hostIfaceName: f03183dcd0db835, network: default, NAD default, SandboxID: "f03183dcd0db83560ff3fb7e45a70b97897687a7122763477bfb7b31f403253d", PCI device ID: , UID: "2e92b32d-a5e6-4356-a2fe-b15dee440997", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0428 11:09:16.793949 3665 cni.go:473] [openshift-multus/network-metrics-daemon-rzlqs f03183dcd0db83560ff3fb7e45a70b97897687a7122763477bfb7b31f403253d network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-rzlqs f03183dcd0db83560ff3fb7e45a70b97897687a7122763477bfb7b31f403253d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:fc:0f:ef:71:91\",\"name\":\"f03183dcd0db835\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/314d2a14-1c7c-48bb-9ee8-2c07cb7c42f1\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:09:20.045020 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0428 11:09:21.102063 3665 base_network_controller_pods.go:487] [default/openshift-console/console-65ff4cc5-j65g9] creating logical port openshift-console_console-65ff4cc5-j65g9 for pod on switch ip-10-0-2-140.ec2.internal I0428 11:09:21.102155 3665 kube.go:256] Updating pod openshift-console/console-65ff4cc5-j65g9 I0428 11:09:21.120604 3665 pod.go:62] [openshift-console/console-65ff4cc5-j65g9] pod update took 18.472378ms I0428 11:09:21.120635 3665 base_network_controller_pods.go:951] [default/openshift-console/console-65ff4cc5-j65g9] addLogicalPort annotation time took 18.509422ms I0428 11:09:21.121918 3665 pods.go:271] [openshift-console/console-65ff4cc5-j65g9] addLogicalPort took 19.875327ms, libovsdb time 902.471µs I0428 11:09:21.519927 3665 cni.go:452] [openshift-console/console-65ff4cc5-j65g9 2f24dc1ed5bd2c789acda8fb9953b438dbe17b318edd77d0e2e5ae5a6d41ab0d network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-65ff4cc5-j65g9 2f24dc1ed5bd2c789acda8fb9953b438dbe17b318edd77d0e2e5ae5a6d41ab0d network default NAD default NAD key ] I0428 11:09:21.529514 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-65ff4cc5-j65g9, hostIfaceName: 2f24dc1ed5bd2c7, network: default, NAD default, SandboxID: "2f24dc1ed5bd2c789acda8fb9953b438dbe17b318edd77d0e2e5ae5a6d41ab0d", PCI device ID: , UID: "7d3f90e0-703e-4ee1-801f-3318c68c1bc1", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0428 11:09:21.583262 3665 cni.go:473] [openshift-console/console-65ff4cc5-j65g9 2f24dc1ed5bd2c789acda8fb9953b438dbe17b318edd77d0e2e5ae5a6d41ab0d network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-65ff4cc5-j65g9 2f24dc1ed5bd2c789acda8fb9953b438dbe17b318edd77d0e2e5ae5a6d41ab0d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:4f:ca:4f:c7:a3\",\"name\":\"2f24dc1ed5bd2c7\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8d9fc0fb-a1c3-4c2a-9e52-d5bd5fad24d1\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:09:22.720449 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 11:09:23.005242 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:09:23.005268 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:09:25.073164 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 11:09:25.871239 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 11:09:27.462384 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:09:27.462411 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 11:09:27.462429 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:09:27.462447 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:09:27.462454 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 11:09:27.462460 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:09:27.726644 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 11:09:27.866550 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:09:27.866572 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 11:09:27.866580 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0428 11:09:31.564119 3665 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-76f5f99d79-pwxwc] creating logical port openshift-monitoring_thanos-querier-76f5f99d79-pwxwc for pod on switch ip-10-0-2-140.ec2.internal I0428 11:09:31.564247 3665 kube.go:256] Updating pod openshift-monitoring/thanos-querier-76f5f99d79-pwxwc I0428 11:09:31.585020 3665 pod.go:62] [openshift-monitoring/thanos-querier-76f5f99d79-pwxwc] pod update took 20.819071ms I0428 11:09:31.585042 3665 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-76f5f99d79-pwxwc] addLogicalPort annotation time took 20.848008ms I0428 11:09:31.586303 3665 pods.go:271] [openshift-monitoring/thanos-querier-76f5f99d79-pwxwc] addLogicalPort took 22.205257ms, libovsdb time 915.305µs I0428 11:09:31.924868 3665 cni.go:452] [openshift-monitoring/thanos-querier-76f5f99d79-pwxwc aca6dc37e2133e765a91b6a2f59307d32c7db6ce2547ebb22291bcb65a469c03 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-76f5f99d79-pwxwc aca6dc37e2133e765a91b6a2f59307d32c7db6ce2547ebb22291bcb65a469c03 network default NAD default NAD key ] W0428 11:09:31.934028 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 11:09:31.950503 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-76f5f99d79-pwxwc, hostIfaceName: aca6dc37e2133e7, network: default, NAD default, SandboxID: "aca6dc37e2133e765a91b6a2f59307d32c7db6ce2547ebb22291bcb65a469c03", PCI device ID: , UID: "7e7732c0-0204-4c84-b268-000c6506609a", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0428 11:09:32.005352 3665 cni.go:473] [openshift-monitoring/thanos-querier-76f5f99d79-pwxwc aca6dc37e2133e765a91b6a2f59307d32c7db6ce2547ebb22291bcb65a469c03 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-76f5f99d79-pwxwc aca6dc37e2133e765a91b6a2f59307d32c7db6ce2547ebb22291bcb65a469c03 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:00:e2:ae:de:a3\",\"name\":\"aca6dc37e2133e7\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/41661366-efd6-4f6d-8afb-a0b8d2de343f\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:09:33.891474 3665 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-5b9b4544b9-8gwvh IP address from the namespace address-set, err: pod openshift-console/console-5b9b4544b9-8gwvh: no pod IPs found W0428 11:09:36.966226 3665 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:09:36.966248 3665 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:09:36.966256 3665 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:09:36.966260 3665 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:09:40.221683 3665 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:09:40.221703 3665 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:09:40.221723 3665 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:09:40.285842 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 11:09:40.329958 3665 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 11:09:40.329976 3665 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 11:09:40.562970 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 11:09:40.609696 3665 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 11:09:40.609730 3665 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 11:09:40.698452 3665 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:09:40.698472 3665 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:09:40.698477 3665 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:09:41.597863 3665 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:09:41.597883 3665 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:09:41.597907 3665 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:09:41.597911 3665 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier I0428 11:09:41.959588 3665 obj_retry.go:498] Detected object openshift-image-registry/image-registry-66f8f49b55-fhcvp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:09:41.960146 3665 obj_retry.go:498] Detected object openshift-image-registry/image-registry-66f8f49b55-fhcvp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:09:41.961068 3665 cni.go:452] [openshift-image-registry/image-registry-677c7cc65b-qsgvf 42e3690655a6c395817b7e88c26089a86e1ac657dfd7c68a0cc80817bd3ceecf network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-677c7cc65b-qsgvf 42e3690655a6c395817b7e88c26089a86e1ac657dfd7c68a0cc80817bd3ceecf network default NAD default NAD key ] I0428 11:09:42.042126 3665 cni.go:473] [openshift-image-registry/image-registry-677c7cc65b-qsgvf 42e3690655a6c395817b7e88c26089a86e1ac657dfd7c68a0cc80817bd3ceecf network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-677c7cc65b-qsgvf 42e3690655a6c395817b7e88c26089a86e1ac657dfd7c68a0cc80817bd3ceecf network default NAD default NAD key default], result "{}", err I0428 11:09:42.089096 3665 obj_retry.go:498] Detected object openshift-image-registry/image-registry-677c7cc65b-qsgvf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:09:42.089135 3665 pods.go:176] Deleting pod: openshift-image-registry/image-registry-677c7cc65b-qsgvf I0428 11:09:42.090289 3665 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-677c7cc65b-qsgvf, ips: 10.129.0.6 I0428 11:09:42.090317 3665 obj_retry.go:498] Detected object openshift-image-registry/image-registry-677c7cc65b-qsgvf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:09:42.099973 3665 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-684757f64-9cqt5] creating logical port openshift-image-registry_image-registry-684757f64-9cqt5 for pod on switch ip-10-0-2-140.ec2.internal I0428 11:09:42.100084 3665 kube.go:256] Updating pod openshift-image-registry/image-registry-684757f64-9cqt5 I0428 11:09:42.117033 3665 pod.go:62] [openshift-image-registry/image-registry-684757f64-9cqt5] pod update took 16.986185ms I0428 11:09:42.117062 3665 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-684757f64-9cqt5] addLogicalPort annotation time took 17.020854ms I0428 11:09:42.118324 3665 pods.go:271] [openshift-image-registry/image-registry-684757f64-9cqt5] addLogicalPort took 18.367678ms, libovsdb time 927.555µs I0428 11:09:42.467579 3665 cni.go:452] [openshift-image-registry/image-registry-684757f64-9cqt5 b231526f012fc48b4b1e00f7ceef4d07072b2e7a5da045e9c1892f34d3f2a916 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-684757f64-9cqt5 b231526f012fc48b4b1e00f7ceef4d07072b2e7a5da045e9c1892f34d3f2a916 network default NAD default NAD key ] I0428 11:09:42.477181 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-684757f64-9cqt5, hostIfaceName: b231526f012fc48, network: default, NAD default, SandboxID: "b231526f012fc48b4b1e00f7ceef4d07072b2e7a5da045e9c1892f34d3f2a916", PCI device ID: , UID: "32f8eb1c-4ff9-4afc-b45a-09e6fa5279fb", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0428 11:09:42.530663 3665 cni.go:473] [openshift-image-registry/image-registry-684757f64-9cqt5 b231526f012fc48b4b1e00f7ceef4d07072b2e7a5da045e9c1892f34d3f2a916 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-684757f64-9cqt5 b231526f012fc48b4b1e00f7ceef4d07072b2e7a5da045e9c1892f34d3f2a916 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:b1:8d:39:ef:7e\",\"name\":\"b231526f012fc48\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/38f9d539-85c1-4285-b8b2-8ec665e700ce\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:09:42.681746 3665 obj_retry.go:492] Detected object openshift-image-registry/image-registry-66f8f49b55-fhcvp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:09:42.681773 3665 obj_retry.go:492] Detected object openshift-image-registry/image-registry-66f8f49b55-fhcvp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:09:42.687811 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-66f8f49b55-fhcvp I0428 11:09:42.687843 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-66f8f49b55-fhcvp W0428 11:09:42.953514 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:09:42.953536 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:09:42.953542 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:09:42.953547 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier I0428 11:09:42.960224 3665 obj_retry.go:492] Detected object openshift-image-registry/image-registry-677c7cc65b-qsgvf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:09:42.960253 3665 obj_retry.go:492] Detected object openshift-image-registry/image-registry-677c7cc65b-qsgvf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:09:42.967889 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-677c7cc65b-qsgvf I0428 11:09:42.967921 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-677c7cc65b-qsgvf I0428 11:09:45.211801 3665 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0428 11:09:51.843193 3665 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0428 11:09:52.995374 3665 obj_retry.go:498] Detected object openshift-console/console-65ff4cc5-8p6gm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:09:52.995918 3665 obj_retry.go:498] Detected object openshift-console/console-65ff4cc5-8p6gm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:09:53.243051 3665 obj_retry.go:492] Detected object openshift-console/console-65ff4cc5-8p6gm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:09:53.243082 3665 obj_retry.go:492] Detected object openshift-console/console-65ff4cc5-8p6gm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:09:53.248772 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-65ff4cc5-8p6gm I0428 11:09:53.248797 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-65ff4cc5-8p6gm I0428 11:10:02.430108 3665 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dc4z49 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:02.430623 3665 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dc4z49 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:02.430655 3665 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dc4z49 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:10:02.430668 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0428 11:10:02.430690 3665 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dc4z49 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:10:02.430699 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0428 11:10:02.841643 3665 obj_retry.go:498] Detected object openshift-console/console-5b9b4544b9-lkh5c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:02.842224 3665 obj_retry.go:498] Detected object openshift-console/console-5b9b4544b9-lkh5c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:03.456137 3665 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dc4z49 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:03.456165 3665 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dc4z49 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:03.456183 3665 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dc4z49 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:03.456195 3665 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dc4z49 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0428 11:10:03.603559 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:10:03.603594 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:10:03.642958 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:10:03.642985 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0428 11:10:03.707706 3665 obj_retry.go:492] Detected object openshift-console/console-5b9b4544b9-lkh5c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:03.707755 3665 obj_retry.go:492] Detected object openshift-console/console-5b9b4544b9-lkh5c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:03.710848 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5b9b4544b9-lkh5c I0428 11:10:03.710868 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5b9b4544b9-lkh5c I0428 11:10:03.778968 3665 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0428 11:10:03.789917 3665 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0428 11:10:03.972636 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0428 11:10:05.347485 3665 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-l9wpq] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-5d7684fb5d-l9wpq for pod on switch ip-10-0-2-140.ec2.internal I0428 11:10:05.347594 3665 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-l9wpq I0428 11:10:05.362324 3665 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-l9wpq] pod update took 14.758622ms I0428 11:10:05.362355 3665 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-l9wpq] addLogicalPort annotation time took 14.794587ms I0428 11:10:05.363764 3665 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-l9wpq] addLogicalPort took 16.301104ms, libovsdb time 918.356µs I0428 11:10:05.707163 3665 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-l9wpq e375245911897443ac31c26b30bf5d3bf6390ab4ebd5ec7e2ffd85f17be786fb network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-l9wpq e375245911897443ac31c26b30bf5d3bf6390ab4ebd5ec7e2ffd85f17be786fb network default NAD default NAD key ] I0428 11:10:05.716694 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-5d7684fb5d-l9wpq, hostIfaceName: e37524591189744, network: default, NAD default, SandboxID: "e375245911897443ac31c26b30bf5d3bf6390ab4ebd5ec7e2ffd85f17be786fb", PCI device ID: , UID: "77c07035-83db-450e-a82e-4b0b3f5e5691", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0428 11:10:05.771299 3665 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-l9wpq e375245911897443ac31c26b30bf5d3bf6390ab4ebd5ec7e2ffd85f17be786fb network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-l9wpq e375245911897443ac31c26b30bf5d3bf6390ab4ebd5ec7e2ffd85f17be786fb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:2f:00:33:6a:15\",\"name\":\"e37524591189744\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/64fd854b-49bd-46cf-802e-794ea0fe8e8d\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:10:10.065481 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0428 11:10:10.065542 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0428 11:10:10.065587 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0428 11:10:10.088333 3665 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0428 11:10:10.109563 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0428 11:10:10.109569 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0428 11:10:10.109732 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0428 11:10:10.141103 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0428 11:10:10.141104 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0428 11:10:10.141276 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0428 11:10:11.076086 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0428 11:10:11.101493 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0428 11:10:11.129641 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0428 11:10:21.082241 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 11:10:29.514986 3665 cni.go:452] [openshift-console/console-65ff4cc5-j65g9 2f24dc1ed5bd2c789acda8fb9953b438dbe17b318edd77d0e2e5ae5a6d41ab0d network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-65ff4cc5-j65g9 2f24dc1ed5bd2c789acda8fb9953b438dbe17b318edd77d0e2e5ae5a6d41ab0d network default NAD default NAD key ] I0428 11:10:29.608860 3665 cni.go:473] [openshift-console/console-65ff4cc5-j65g9 2f24dc1ed5bd2c789acda8fb9953b438dbe17b318edd77d0e2e5ae5a6d41ab0d network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-65ff4cc5-j65g9 2f24dc1ed5bd2c789acda8fb9953b438dbe17b318edd77d0e2e5ae5a6d41ab0d network default NAD default NAD key default], result "{}", err I0428 11:10:29.654413 3665 obj_retry.go:498] Detected object openshift-console/console-65ff4cc5-j65g9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:29.654446 3665 pods.go:176] Deleting pod: openshift-console/console-65ff4cc5-j65g9 I0428 11:10:29.655775 3665 pods.go:236] Attempting to release IPs for pod: openshift-console/console-65ff4cc5-j65g9, ips: 10.129.0.20 I0428 11:10:29.655800 3665 obj_retry.go:498] Detected object openshift-console/console-65ff4cc5-j65g9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:29.959162 3665 obj_retry.go:498] Detected object openshift-image-registry/image-registry-66f8f49b55-bhjwj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:29.959740 3665 obj_retry.go:498] Detected object openshift-image-registry/image-registry-66f8f49b55-bhjwj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:30.147985 3665 obj_retry.go:492] Detected object openshift-console/console-65ff4cc5-j65g9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:30.148013 3665 obj_retry.go:492] Detected object openshift-console/console-65ff4cc5-j65g9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:30.150917 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-65ff4cc5-j65g9 I0428 11:10:30.150951 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-65ff4cc5-j65g9 I0428 11:10:30.411291 3665 obj_retry.go:492] Detected object openshift-image-registry/image-registry-66f8f49b55-bhjwj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:30.411318 3665 obj_retry.go:492] Detected object openshift-image-registry/image-registry-66f8f49b55-bhjwj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:30.418138 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-66f8f49b55-bhjwj I0428 11:10:30.418158 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-66f8f49b55-bhjwj I0428 11:10:31.208611 3665 cni.go:452] [openshift-ingress/router-default-c45c8f597-g4js8 e78ef727a09a8120533947cdbc98e98b30fa448c6b92997506cbe7b7f1e1a778 network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-c45c8f597-g4js8 e78ef727a09a8120533947cdbc98e98b30fa448c6b92997506cbe7b7f1e1a778 network default NAD default NAD key ] I0428 11:10:31.286689 3665 cni.go:473] [openshift-ingress/router-default-c45c8f597-g4js8 e78ef727a09a8120533947cdbc98e98b30fa448c6b92997506cbe7b7f1e1a778 network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-c45c8f597-g4js8 e78ef727a09a8120533947cdbc98e98b30fa448c6b92997506cbe7b7f1e1a778 network default NAD default NAD key default], result "{}", err I0428 11:10:31.337591 3665 obj_retry.go:498] Detected object openshift-ingress/router-default-c45c8f597-g4js8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:31.337625 3665 pods.go:176] Deleting pod: openshift-ingress/router-default-c45c8f597-g4js8 I0428 11:10:31.338921 3665 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-c45c8f597-g4js8, ips: 10.129.0.15 I0428 11:10:31.338949 3665 obj_retry.go:498] Detected object openshift-ingress/router-default-c45c8f597-g4js8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:32.161277 3665 obj_retry.go:492] Detected object openshift-ingress/router-default-c45c8f597-g4js8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:32.161306 3665 obj_retry.go:492] Detected object openshift-ingress/router-default-c45c8f597-g4js8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:32.164555 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-c45c8f597-g4js8 I0428 11:10:32.164583 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-c45c8f597-g4js8 I0428 11:10:54.175587 3665 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:54.179289 3665 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:54.907302 3665 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:54.907332 3665 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:54.913338 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0428 11:10:54.913365 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0428 11:10:58.563229 3665 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:58.563872 3665 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:58.935877 3665 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:58.935905 3665 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:58.943248 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0428 11:10:58.943272 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0428 11:11:00.818099 3665 base_network_controller_pods.go:487] [default/openshift-console/console-c7d9596f5-lmw6h] creating logical port openshift-console_console-c7d9596f5-lmw6h for pod on switch ip-10-0-2-140.ec2.internal I0428 11:11:00.818231 3665 kube.go:256] Updating pod openshift-console/console-c7d9596f5-lmw6h I0428 11:11:00.843382 3665 pod.go:62] [openshift-console/console-c7d9596f5-lmw6h] pod update took 25.174777ms I0428 11:11:00.843406 3665 base_network_controller_pods.go:951] [default/openshift-console/console-c7d9596f5-lmw6h] addLogicalPort annotation time took 25.203132ms I0428 11:11:00.844660 3665 pods.go:271] [openshift-console/console-c7d9596f5-lmw6h] addLogicalPort took 26.577654ms, libovsdb time 898.244µs I0428 11:11:01.180139 3665 cni.go:452] [openshift-console/console-c7d9596f5-lmw6h 7d2d036a5e4c930c3224aa5e29ba26c7ad5d4e9d86749002accf58c5ce039aba network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-c7d9596f5-lmw6h 7d2d036a5e4c930c3224aa5e29ba26c7ad5d4e9d86749002accf58c5ce039aba network default NAD default NAD key ] I0428 11:11:01.189920 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-c7d9596f5-lmw6h, hostIfaceName: 7d2d036a5e4c930, network: default, NAD default, SandboxID: "7d2d036a5e4c930c3224aa5e29ba26c7ad5d4e9d86749002accf58c5ce039aba", PCI device ID: , UID: "b9400554-fd71-4742-8b29-3174ea0133d3", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0428 11:11:01.242436 3665 cni.go:473] [openshift-console/console-c7d9596f5-lmw6h 7d2d036a5e4c930c3224aa5e29ba26c7ad5d4e9d86749002accf58c5ce039aba network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-c7d9596f5-lmw6h 7d2d036a5e4c930c3224aa5e29ba26c7ad5d4e9d86749002accf58c5ce039aba network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:5a:9a:75:ad:2c\",\"name\":\"7d2d036a5e4c930\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/40576645-7198-43a5-9249-de69e78efa67\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:11:11.331122 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 11:11:16.679794 3665 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:16.680344 3665 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:17.611391 3665 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:17.611420 3665 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:17.616194 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0428 11:11:17.616228 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0428 11:11:26.104153 3665 obj_retry.go:498] Detected object openshift-console/console-77cf9c8d8b-8nxwp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:26.104704 3665 obj_retry.go:498] Detected object openshift-console/console-77cf9c8d8b-8nxwp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:27.072067 3665 obj_retry.go:492] Detected object openshift-console/console-77cf9c8d8b-8nxwp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:27.072092 3665 obj_retry.go:492] Detected object openshift-console/console-77cf9c8d8b-8nxwp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:27.078085 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-77cf9c8d8b-8nxwp I0428 11:11:27.078115 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-77cf9c8d8b-8nxwp I0428 11:11:36.647469 3665 obj_retry.go:498] Detected object openshift-console/console-77cf9c8d8b-b9d9x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:36.648024 3665 obj_retry.go:498] Detected object openshift-console/console-77cf9c8d8b-b9d9x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:36.986484 3665 obj_retry.go:492] Detected object openshift-console/console-77cf9c8d8b-b9d9x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:36.986511 3665 obj_retry.go:492] Detected object openshift-console/console-77cf9c8d8b-b9d9x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:36.992790 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-77cf9c8d8b-b9d9x I0428 11:11:36.992814 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-77cf9c8d8b-b9d9x I0428 11:12:01.537046 3665 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:12:01.537600 3665 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:12:01.801577 3665 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:12:01.801599 3665 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:12:01.808119 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0428 11:12:01.808143 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0428 11:13:40.776986 3665 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0428 11:13:42.399791 3665 obj_retry.go:498] Detected object openshift-console/console-c7d9596f5-d2f7n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:13:42.400311 3665 obj_retry.go:498] Detected object openshift-console/console-c7d9596f5-d2f7n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:13:42.580051 3665 obj_retry.go:492] Detected object openshift-console/console-c7d9596f5-d2f7n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:13:42.580082 3665 obj_retry.go:492] Detected object openshift-console/console-c7d9596f5-d2f7n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:13:42.582676 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-c7d9596f5-d2f7n I0428 11:13:42.582704 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-c7d9596f5-d2f7n I0428 11:13:51.777668 3665 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0428 11:13:53.337810 3665 cni.go:452] [openshift-console/console-c7d9596f5-lmw6h 7d2d036a5e4c930c3224aa5e29ba26c7ad5d4e9d86749002accf58c5ce039aba network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-c7d9596f5-lmw6h 7d2d036a5e4c930c3224aa5e29ba26c7ad5d4e9d86749002accf58c5ce039aba network default NAD default NAD key ] I0428 11:13:53.408482 3665 cni.go:473] [openshift-console/console-c7d9596f5-lmw6h 7d2d036a5e4c930c3224aa5e29ba26c7ad5d4e9d86749002accf58c5ce039aba network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-c7d9596f5-lmw6h 7d2d036a5e4c930c3224aa5e29ba26c7ad5d4e9d86749002accf58c5ce039aba network default NAD default NAD key default], result "{}", err I0428 11:13:53.455278 3665 obj_retry.go:498] Detected object openshift-console/console-c7d9596f5-lmw6h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:13:53.455310 3665 pods.go:176] Deleting pod: openshift-console/console-c7d9596f5-lmw6h I0428 11:13:53.456534 3665 pods.go:236] Attempting to release IPs for pod: openshift-console/console-c7d9596f5-lmw6h, ips: 10.129.0.24 I0428 11:13:53.456565 3665 obj_retry.go:498] Detected object openshift-console/console-c7d9596f5-lmw6h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:13:53.929621 3665 obj_retry.go:492] Detected object openshift-console/console-c7d9596f5-lmw6h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:13:53.929644 3665 obj_retry.go:492] Detected object openshift-console/console-c7d9596f5-lmw6h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:13:53.936438 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-c7d9596f5-lmw6h I0428 11:13:53.936467 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-c7d9596f5-lmw6h I0428 11:14:12.689500 3665 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0428 11:14:49.689428 3665 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0428 11:14:52.814985 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0428 11:15:13.787471 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0428 11:15:21.703797 3665 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=612 I0428 11:15:42.781630 3665 namespace.go:98] [tsf] adding namespace I0428 11:15:42.782556 3665 namespace.go:102] [tsf] adding namespace took 900.427µs I0428 11:15:42.805809 3665 namespace.go:142] [tsf] updating namespace I0428 11:15:42.809917 3665 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0428 11:15:42.813895 3665 namespace.go:142] [tsf] updating namespace I0428 11:15:46.851200 3665 namespace.go:98] [cert-manager-operator] adding namespace I0428 11:15:46.851362 3665 namespace.go:98] [konflux-ui] adding namespace I0428 11:15:46.852098 3665 namespace.go:102] [cert-manager-operator] adding namespace took 876.117µs I0428 11:15:46.852127 3665 namespace.go:98] [konflux-operator] adding namespace I0428 11:15:46.852559 3665 namespace.go:102] [konflux-operator] adding namespace took 425.032µs I0428 11:15:46.853090 3665 namespace.go:102] [konflux-ui] adding namespace took 1.711689ms I0428 11:15:46.855764 3665 namespace.go:98] [rhbk-operator] adding namespace I0428 11:15:46.855894 3665 namespace.go:98] [tsf-tpa] adding namespace I0428 11:15:46.855926 3665 namespace.go:98] [tsf-keycloak] adding namespace I0428 11:15:46.856015 3665 namespace.go:98] [tsf-tas] adding namespace I0428 11:15:46.856157 3665 namespace.go:98] [rhtpa-operator] adding namespace I0428 11:15:46.856491 3665 namespace.go:102] [rhbk-operator] adding namespace took 711.004µs I0428 11:15:46.857221 3665 namespace.go:102] [tsf-tpa] adding namespace took 1.307181ms I0428 11:15:46.857772 3665 namespace.go:102] [tsf-keycloak] adding namespace took 1.831197ms I0428 11:15:46.858301 3665 namespace.go:102] [tsf-tas] adding namespace took 2.272113ms I0428 11:15:46.858890 3665 namespace.go:102] [rhtpa-operator] adding namespace took 2.71939ms I0428 11:15:46.870122 3665 namespace.go:142] [cert-manager-operator] updating namespace I0428 11:15:46.883251 3665 namespace.go:142] [cert-manager-operator] updating namespace I0428 11:15:46.894639 3665 namespace.go:142] [rhbk-operator] updating namespace I0428 11:15:46.894676 3665 namespace.go:142] [konflux-ui] updating namespace I0428 11:15:46.900757 3665 namespace.go:142] [tsf-tpa] updating namespace I0428 11:15:46.900864 3665 namespace.go:142] [konflux-operator] updating namespace I0428 11:15:46.918849 3665 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:15:46.932366 3665 namespace.go:142] [tsf-tas] updating namespace I0428 11:15:46.932405 3665 namespace.go:142] [rhtpa-operator] updating namespace I0428 11:15:46.932530 3665 namespace.go:142] [cert-manager-operator] updating namespace I0428 11:15:46.951047 3665 namespace.go:142] [konflux-ui] updating namespace I0428 11:15:46.970664 3665 namespace.go:142] [konflux-ui] updating namespace I0428 11:15:46.988789 3665 namespace.go:142] [rhbk-operator] updating namespace I0428 11:15:47.012797 3665 namespace.go:142] [rhbk-operator] updating namespace I0428 11:15:47.025046 3665 namespace.go:142] [tsf-tpa] updating namespace I0428 11:15:47.045989 3665 namespace.go:142] [tsf-tpa] updating namespace I0428 11:15:47.063565 3665 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:15:47.077664 3665 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:15:47.098754 3665 namespace.go:142] [tsf-tas] updating namespace I0428 11:15:47.114645 3665 namespace.go:142] [tsf-tas] updating namespace I0428 11:15:47.165842 3665 namespace.go:142] [rhtpa-operator] updating namespace I0428 11:15:47.172784 3665 namespace.go:142] [rhtpa-operator] updating namespace I0428 11:15:47.188161 3665 namespace.go:142] [konflux-operator] updating namespace I0428 11:15:47.197801 3665 namespace.go:142] [konflux-operator] updating namespace I0428 11:15:48.451122 3665 namespace.go:142] [tsf-tpa] updating namespace I0428 11:15:48.453750 3665 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:15:53.847473 3665 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2g9f85 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:53.848077 3665 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2g9f85 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:53.848110 3665 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2g9f85 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:15:53.848120 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 11:15:53.848132 3665 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2g9f85 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:15:53.848138 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 11:15:54.876263 3665 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2g9f85 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:54.876288 3665 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2g9f85 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:54.876309 3665 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2g9f85 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:54.876322 3665 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2g9f85 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:55.856387 3665 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51p8xgr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:55.856953 3665 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51p8xgr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:55.856983 3665 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51p8xgr of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:15:55.856991 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 11:15:55.857001 3665 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51p8xgr of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:15:55.857005 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 11:15:56.888229 3665 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51p8xgr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:56.888261 3665 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51p8xgr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:56.888273 3665 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51p8xgr of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:56.888281 3665 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51p8xgr of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:56.892040 3665 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588dkfv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:56.892541 3665 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588dkfv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:56.892568 3665 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588dkfv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:15:56.892578 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 11:15:56.892589 3665 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588dkfv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:15:56.892594 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 11:15:57.916841 3665 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588dkfv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:57.916863 3665 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588dkfv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:57.916882 3665 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588dkfv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:57.916892 3665 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588dkfv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:59.898749 3665 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dp6f7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:59.899327 3665 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dp6f7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:59.899349 3665 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dp6f7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:15:59.899358 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 11:15:59.899376 3665 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dp6f7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:15:59.899383 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 11:16:00.922463 3665 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136ptrgd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:16:00.923033 3665 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136ptrgd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:16:00.923063 3665 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136ptrgd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:16:00.923071 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 11:16:00.923081 3665 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136ptrgd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:16:00.923086 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 11:16:00.934881 3665 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dp6f7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:00.934901 3665 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dp6f7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:00.934914 3665 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dp6f7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:00.934923 3665 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dp6f7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:01.288989 3665 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-fb86cd8d9-8qxzv] creating logical port rhbk-operator_rhbk-operator-fb86cd8d9-8qxzv for pod on switch ip-10-0-2-140.ec2.internal I0428 11:16:01.289130 3665 kube.go:256] Updating pod rhbk-operator/rhbk-operator-fb86cd8d9-8qxzv I0428 11:16:01.309103 3665 pod.go:62] [rhbk-operator/rhbk-operator-fb86cd8d9-8qxzv] pod update took 19.995581ms I0428 11:16:01.309131 3665 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-fb86cd8d9-8qxzv] addLogicalPort annotation time took 20.029175ms I0428 11:16:01.310320 3665 pods.go:271] [rhbk-operator/rhbk-operator-fb86cd8d9-8qxzv] addLogicalPort took 21.348689ms, libovsdb time 879.015µs I0428 11:16:01.646484 3665 cni.go:452] [rhbk-operator/rhbk-operator-fb86cd8d9-8qxzv 797cdd788c7489b64caf15774492c6dbd43832d904431b2ba10a29f30387648c network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-8qxzv 797cdd788c7489b64caf15774492c6dbd43832d904431b2ba10a29f30387648c network default NAD default NAD key ] I0428 11:16:01.656210 3665 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-fb86cd8d9-8qxzv, hostIfaceName: 797cdd788c7489b, network: default, NAD default, SandboxID: "797cdd788c7489b64caf15774492c6dbd43832d904431b2ba10a29f30387648c", PCI device ID: , UID: "add4d6a0-e87b-4b3d-b8da-37e2a1ed8884", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0428 11:16:01.712110 3665 cni.go:473] [rhbk-operator/rhbk-operator-fb86cd8d9-8qxzv 797cdd788c7489b64caf15774492c6dbd43832d904431b2ba10a29f30387648c network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-8qxzv 797cdd788c7489b64caf15774492c6dbd43832d904431b2ba10a29f30387648c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:61:1a:c8:ea:a0\",\"name\":\"797cdd788c7489b\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dd583016-4938-4467-b118-de02d0669c1c\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:16:01.948069 3665 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136ptrgd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:01.948092 3665 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136ptrgd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:01.948104 3665 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136ptrgd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:01.948116 3665 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136ptrgd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:04.252529 3665 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-xhjb8] creating logical port tsf-tpa_rhtpa-operator-controller-manager-586cbb496b-xhjb8 for pod on switch ip-10-0-2-140.ec2.internal I0428 11:16:04.252645 3665 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-xhjb8 I0428 11:16:04.268211 3665 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-xhjb8] pod update took 15.5901ms I0428 11:16:04.268244 3665 base_network_controller_pods.go:951] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-xhjb8] addLogicalPort annotation time took 15.625301ms I0428 11:16:04.269577 3665 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-xhjb8] addLogicalPort took 17.068075ms, libovsdb time 882.066µs I0428 11:16:04.615158 3665 cni.go:452] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-xhjb8 4544f39e880fefe01cf6141312a64e71325923736280821074608121ad67fa0f network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-xhjb8 4544f39e880fefe01cf6141312a64e71325923736280821074608121ad67fa0f network default NAD default NAD key ] I0428 11:16:04.624963 3665 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-586cbb496b-xhjb8, hostIfaceName: 4544f39e880fefe, network: default, NAD default, SandboxID: "4544f39e880fefe01cf6141312a64e71325923736280821074608121ad67fa0f", PCI device ID: , UID: "3b8a2185-daa1-4c61-bf6a-13ffa4ee327b", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0428 11:16:04.676291 3665 cni.go:473] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-xhjb8 4544f39e880fefe01cf6141312a64e71325923736280821074608121ad67fa0f network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-xhjb8 4544f39e880fefe01cf6141312a64e71325923736280821074608121ad67fa0f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:6d:58:d2:84:ce\",\"name\":\"4544f39e880fefe\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7206de7f-71c4-4dba-a88b-1e4dd47def73\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:16:10.469554 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0428 11:16:10.963098 3665 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d2f898 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:16:10.963644 3665 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d2f898 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:16:10.963673 3665 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d2f898 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:16:10.963681 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 11:16:10.963695 3665 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d2f898 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:16:10.963702 3665 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 11:16:11.993497 3665 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d2f898 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:11.993521 3665 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d2f898 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:11.993538 3665 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d2f898 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:11.993548 3665 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d2f898 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:13.135643 3665 namespace.go:142] [openshift-operators] updating namespace I0428 11:16:13.162998 3665 namespace.go:142] [openshift-operators] updating namespace I0428 11:16:18.797204 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 W0428 11:16:19.893325 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0428 11:16:21.303918 3665 namespace.go:142] [openshift-operators] updating namespace W0428 11:16:21.484893 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0428 11:16:22.529909 3665 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-5864bd5898-44qmv] creating logical port openshift-operators_tekton-operator-webhook-5864bd5898-44qmv for pod on switch ip-10-0-2-140.ec2.internal I0428 11:16:22.530048 3665 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-5864bd5898-44qmv I0428 11:16:22.561019 3665 pod.go:62] [openshift-operators/tekton-operator-webhook-5864bd5898-44qmv] pod update took 31.003889ms I0428 11:16:22.561042 3665 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-5864bd5898-44qmv] addLogicalPort annotation time took 31.032512ms I0428 11:16:22.562249 3665 pods.go:271] [openshift-operators/tekton-operator-webhook-5864bd5898-44qmv] addLogicalPort took 32.363081ms, libovsdb time 831.511µs I0428 11:16:22.804467 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0428 11:16:22.911182 3665 cni.go:452] [openshift-operators/tekton-operator-webhook-5864bd5898-44qmv b11713ab55e839f66837fc481f5f0a2d4048c1250c58f1a5c2f3aec3a0a84636 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-44qmv b11713ab55e839f66837fc481f5f0a2d4048c1250c58f1a5c2f3aec3a0a84636 network default NAD default NAD key ] I0428 11:16:22.920612 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-5864bd5898-44qmv, hostIfaceName: b11713ab55e839f, network: default, NAD default, SandboxID: "b11713ab55e839f66837fc481f5f0a2d4048c1250c58f1a5c2f3aec3a0a84636", PCI device ID: , UID: "d9077ec0-2378-4724-8fd7-63dd204a6488", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0428 11:16:22.974741 3665 cni.go:473] [openshift-operators/tekton-operator-webhook-5864bd5898-44qmv b11713ab55e839f66837fc481f5f0a2d4048c1250c58f1a5c2f3aec3a0a84636 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-44qmv b11713ab55e839f66837fc481f5f0a2d4048c1250c58f1a5c2f3aec3a0a84636 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:60:94:4c:14:d3\",\"name\":\"b11713ab55e839f\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4ca59831-10d7-4c6d-8880-d7e25faed363\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:16:26.212320 3665 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-t4dgw] creating logical port openshift-operators_rhtas-operator-controller-manager-68888cb75-t4dgw for pod on switch ip-10-0-2-140.ec2.internal I0428 11:16:26.212436 3665 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-68888cb75-t4dgw I0428 11:16:26.232731 3665 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-68888cb75-t4dgw] pod update took 20.300879ms I0428 11:16:26.232758 3665 base_network_controller_pods.go:951] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-t4dgw] addLogicalPort annotation time took 20.353221ms I0428 11:16:26.233935 3665 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-68888cb75-t4dgw] addLogicalPort took 21.6403ms, libovsdb time 849.693µs I0428 11:16:26.574013 3665 cni.go:452] [openshift-operators/rhtas-operator-controller-manager-68888cb75-t4dgw f2bf5b649872b23c4f4d08afd069279e40962aa419b51f0ff15d620e39f91abc network default NAD default NAD key ] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-t4dgw f2bf5b649872b23c4f4d08afd069279e40962aa419b51f0ff15d620e39f91abc network default NAD default NAD key ] I0428 11:16:26.584068 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-68888cb75-t4dgw, hostIfaceName: f2bf5b649872b23, network: default, NAD default, SandboxID: "f2bf5b649872b23c4f4d08afd069279e40962aa419b51f0ff15d620e39f91abc", PCI device ID: , UID: "fe6ec5f7-3e90-4ef2-8db3-49b06ca963f6", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0428 11:16:26.638663 3665 cni.go:473] [openshift-operators/rhtas-operator-controller-manager-68888cb75-t4dgw f2bf5b649872b23c4f4d08afd069279e40962aa419b51f0ff15d620e39f91abc network default NAD default NAD key default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-t4dgw f2bf5b649872b23c4f4d08afd069279e40962aa419b51f0ff15d620e39f91abc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:f3:44:d6:e9:b5\",\"name\":\"f2bf5b649872b23\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0d2a0af-472a-403d-9f85-bc6fa7701460\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:16:28.865851 3665 namespace.go:98] [trusted-artifact-signer] adding namespace I0428 11:16:28.866846 3665 namespace.go:102] [trusted-artifact-signer] adding namespace took 967.097µs I0428 11:16:28.912857 3665 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 11:16:28.931408 3665 namespace.go:142] [trusted-artifact-signer] updating namespace W0428 11:16:29.554625 3665 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0428 11:16:29.978684 3665 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-7d56d9fd65-pzzmc] creating logical port trusted-artifact-signer_cli-server-7d56d9fd65-pzzmc for pod on switch ip-10-0-2-140.ec2.internal I0428 11:16:29.978793 3665 kube.go:256] Updating pod trusted-artifact-signer/cli-server-7d56d9fd65-pzzmc I0428 11:16:29.994623 3665 pod.go:62] [trusted-artifact-signer/cli-server-7d56d9fd65-pzzmc] pod update took 15.855798ms I0428 11:16:29.994651 3665 base_network_controller_pods.go:951] [default/trusted-artifact-signer/cli-server-7d56d9fd65-pzzmc] addLogicalPort annotation time took 15.889618ms I0428 11:16:29.996127 3665 pods.go:271] [trusted-artifact-signer/cli-server-7d56d9fd65-pzzmc] addLogicalPort took 17.463001ms, libovsdb time 1.058241ms I0428 11:16:30.340922 3665 cni.go:452] [trusted-artifact-signer/cli-server-7d56d9fd65-pzzmc 5fab80da161bf8b535a00e57844d077005790cfd242e00f99311b60ee239d1f5 network default NAD default NAD key ] ADD starting CNI request [trusted-artifact-signer/cli-server-7d56d9fd65-pzzmc 5fab80da161bf8b535a00e57844d077005790cfd242e00f99311b60ee239d1f5 network default NAD default NAD key ] I0428 11:16:30.351883 3665 helper_linux.go:556] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-7d56d9fd65-pzzmc, hostIfaceName: 5fab80da161bf8b, network: default, NAD default, SandboxID: "5fab80da161bf8b535a00e57844d077005790cfd242e00f99311b60ee239d1f5", PCI device ID: , UID: "65c0c752-001e-48d3-80ca-d838bfc3572c", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0428 11:16:30.408914 3665 cni.go:473] [trusted-artifact-signer/cli-server-7d56d9fd65-pzzmc 5fab80da161bf8b535a00e57844d077005790cfd242e00f99311b60ee239d1f5 network default NAD default NAD key default] ADD finished CNI request [trusted-artifact-signer/cli-server-7d56d9fd65-pzzmc 5fab80da161bf8b535a00e57844d077005790cfd242e00f99311b60ee239d1f5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:16:66:d1:73:8c\",\"name\":\"5fab80da161bf8b\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cce00859-890e-4ed8-9527-8e080b7c07a9\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:16:37.858864 3665 namespace.go:98] [openshift-pipelines] adding namespace I0428 11:16:37.859859 3665 namespace.go:102] [openshift-pipelines] adding namespace took 971.732µs I0428 11:16:37.924929 3665 namespace.go:142] [openshift-pipelines] updating namespace I0428 11:16:38.694632 3665 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=60 I0428 11:16:39.661780 3665 namespace.go:142] [cert-manager-operator] updating namespace I0428 11:16:40.099635 3665 namespace.go:142] [konflux-operator] updating namespace I0428 11:16:40.119249 3665 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-98llw] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-5b66478c58-98llw for pod on switch ip-10-0-2-140.ec2.internal I0428 11:16:40.119408 3665 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-98llw I0428 11:16:40.145817 3665 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-98llw] pod update took 26.436172ms I0428 11:16:40.145851 3665 base_network_controller_pods.go:951] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-98llw] addLogicalPort annotation time took 26.469975ms I0428 11:16:40.147082 3665 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-98llw] addLogicalPort took 27.849658ms, libovsdb time 897.677µs I0428 11:16:40.312224 3665 namespace.go:142] [konflux-ui] updating namespace I0428 11:16:40.480972 3665 cni.go:452] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-98llw e0d9172b2400b9b617beae33b8ee03eb27ba1921624550c4eee7abf55e380051 network default NAD default NAD key ] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-98llw e0d9172b2400b9b617beae33b8ee03eb27ba1921624550c4eee7abf55e380051 network default NAD default NAD key ] I0428 11:16:40.491491 3665 helper_linux.go:556] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-5b66478c58-98llw, hostIfaceName: e0d9172b2400b9b, network: default, NAD default, SandboxID: "e0d9172b2400b9b617beae33b8ee03eb27ba1921624550c4eee7abf55e380051", PCI device ID: , UID: "610342c2-4285-42a9-9d28-7444a3676e64", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0428 11:16:40.501989 3665 namespace.go:142] [open-cluster-management-2pv477t4c9srti67smm59tsf841pmq8b] updating namespace I0428 11:16:40.547747 3665 cni.go:473] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-98llw e0d9172b2400b9b617beae33b8ee03eb27ba1921624550c4eee7abf55e380051 network default NAD default NAD key default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-98llw e0d9172b2400b9b617beae33b8ee03eb27ba1921624550c4eee7abf55e380051 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:4a:2c:6e:2b:0b\",\"name\":\"e0d9172b2400b9b\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2229e62e-4a1a-4cc3-a4d2-a5b977314c8d\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:16:40.575746 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0428 11:16:40.908677 3665 namespace.go:142] [rhbk-operator] updating namespace I0428 11:16:41.105103 3665 namespace.go:142] [rhtpa-operator] updating namespace I0428 11:16:41.514815 3665 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 11:16:42.044696 3665 namespace.go:142] [tsf] updating namespace I0428 11:16:42.312107 3665 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:16:42.510753 3665 namespace.go:142] [tsf-tas] updating namespace I0428 11:16:42.745777 3665 namespace.go:98] [cert-manager] adding namespace I0428 11:16:42.746701 3665 namespace.go:102] [cert-manager] adding namespace took 903.223µs I0428 11:16:42.947201 3665 namespace.go:142] [cert-manager] updating namespace I0428 11:16:42.999886 3665 namespace.go:142] [cert-manager] updating namespace I0428 11:16:43.090646 3665 namespace.go:142] [tsf-tpa] updating namespace I0428 11:16:43.480285 3665 namespace.go:142] [cert-manager-operator] updating namespace I0428 11:16:43.505610 3665 namespace.go:142] [default] updating namespace I0428 11:16:43.525908 3665 namespace.go:142] [konflux-operator] updating namespace I0428 11:16:43.540667 3665 namespace.go:142] [konflux-ui] updating namespace I0428 11:16:43.567970 3665 namespace.go:142] [open-cluster-management-2pv477t4c9srti67smm59tsf841pmq8b] updating namespace I0428 11:16:43.586640 3665 namespace.go:142] [openshift] updating namespace I0428 11:16:43.600810 3665 namespace.go:142] [rhbk-operator] updating namespace I0428 11:16:43.627702 3665 namespace.go:142] [rhtpa-operator] updating namespace I0428 11:16:43.670744 3665 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 11:16:43.702754 3665 namespace.go:142] [tsf] updating namespace I0428 11:16:43.721647 3665 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:16:43.762599 3665 namespace.go:142] [tsf-tas] updating namespace I0428 11:16:43.834744 3665 namespace.go:142] [tsf-tpa] updating namespace I0428 11:16:44.068780 3665 namespace.go:142] [cert-manager-operator] updating namespace I0428 11:16:44.197665 3665 namespace.go:142] [default] updating namespace I0428 11:16:44.370921 3665 namespace.go:142] [konflux-operator] updating namespace I0428 11:16:44.622992 3665 namespace.go:142] [konflux-ui] updating namespace I0428 11:16:44.921835 3665 namespace.go:142] [open-cluster-management-2pv477t4c9srti67smm59tsf841pmq8b] updating namespace I0428 11:16:45.165623 3665 namespace.go:142] [openshift] updating namespace I0428 11:16:45.556931 3665 namespace.go:142] [rhbk-operator] updating namespace I0428 11:16:45.718669 3665 namespace.go:142] [rhtpa-operator] updating namespace I0428 11:16:45.837051 3665 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 11:16:45.936011 3665 namespace.go:142] [tsf] updating namespace I0428 11:16:46.069659 3665 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:16:46.185771 3665 namespace.go:142] [tsf-tas] updating namespace I0428 11:16:46.304754 3665 namespace.go:142] [tsf-tpa] updating namespace I0428 11:16:47.090766 3665 namespace.go:142] [openshift-pipelines] updating namespace I0428 11:16:47.886690 3665 namespace.go:142] [cert-manager] updating namespace W0428 11:16:48.109727 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0428 11:16:48.109749 3665 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0428 11:16:48.744594 3665 namespace.go:142] [cert-manager] updating namespace I0428 11:16:49.086011 3665 namespace.go:142] [cert-manager] updating namespace I0428 11:16:49.165837 3665 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5d99449ffd-sfwlk] creating logical port cert-manager_cert-manager-cainjector-5d99449ffd-sfwlk for pod on switch ip-10-0-2-140.ec2.internal I0428 11:16:49.165933 3665 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5d99449ffd-sfwlk I0428 11:16:49.194546 3665 pod.go:62] [cert-manager/cert-manager-cainjector-5d99449ffd-sfwlk] pod update took 28.631945ms I0428 11:16:49.194572 3665 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-cainjector-5d99449ffd-sfwlk] addLogicalPort annotation time took 28.666435ms I0428 11:16:49.195863 3665 pods.go:271] [cert-manager/cert-manager-cainjector-5d99449ffd-sfwlk] addLogicalPort took 30.04381ms, libovsdb time 887.12µs I0428 11:16:49.521434 3665 cni.go:452] [cert-manager/cert-manager-cainjector-5d99449ffd-sfwlk 0c265a0ee55d58951184fa7aedf481441becfbc76b3168352fa1165c5837cb2e network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-sfwlk 0c265a0ee55d58951184fa7aedf481441becfbc76b3168352fa1165c5837cb2e network default NAD default NAD key ] I0428 11:16:49.531153 3665 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5d99449ffd-sfwlk, hostIfaceName: 0c265a0ee55d589, network: default, NAD default, SandboxID: "0c265a0ee55d58951184fa7aedf481441becfbc76b3168352fa1165c5837cb2e", PCI device ID: , UID: "1212855a-c7b1-4336-96c5-1716c23b41ff", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0428 11:16:49.586372 3665 cni.go:473] [cert-manager/cert-manager-cainjector-5d99449ffd-sfwlk 0c265a0ee55d58951184fa7aedf481441becfbc76b3168352fa1165c5837cb2e network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-sfwlk 0c265a0ee55d58951184fa7aedf481441becfbc76b3168352fa1165c5837cb2e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:dc:b2:9d:b4:25\",\"name\":\"0c265a0ee55d589\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8fedce70-d2aa-4144-a01c-2c1b2e61d27a\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:16:54.690499 3665 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=173 I0428 11:16:56.480835 3665 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gl98c] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-85f5bb4dfb-gl98c for pod on switch ip-10-0-2-140.ec2.internal I0428 11:16:56.480936 3665 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gl98c I0428 11:16:56.498773 3665 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gl98c] pod update took 17.864934ms I0428 11:16:56.498799 3665 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gl98c] addLogicalPort annotation time took 17.895818ms I0428 11:16:56.500166 3665 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gl98c] addLogicalPort took 19.346503ms, libovsdb time 1.007741ms I0428 11:16:56.588004 3665 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-n8zx8] creating logical port openshift-pipelines_tekton-pipelines-webhook-7b6d6f4c94-n8zx8 for pod on switch ip-10-0-2-140.ec2.internal I0428 11:16:56.588132 3665 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-n8zx8 I0428 11:16:56.606796 3665 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-n8zx8] pod update took 18.693271ms I0428 11:16:56.606821 3665 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-n8zx8] addLogicalPort annotation time took 18.724248ms I0428 11:16:56.608043 3665 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-n8zx8] addLogicalPort took 20.05956ms, libovsdb time 853.534µs I0428 11:16:56.665076 3665 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-4vp9l] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-57f84cff6f-4vp9l for pod on switch ip-10-0-2-140.ec2.internal I0428 11:16:56.665218 3665 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-4vp9l I0428 11:16:56.684980 3665 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-4vp9l] pod update took 19.786682ms I0428 11:16:56.685002 3665 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-4vp9l] addLogicalPort annotation time took 19.816226ms I0428 11:16:56.686320 3665 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-4vp9l] addLogicalPort took 21.267344ms, libovsdb time 913.532µs I0428 11:16:56.842916 3665 cni.go:452] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gl98c 763c795116620208c700c1d769d05760d5621b27cca1890d3a1af6757c955aac network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gl98c 763c795116620208c700c1d769d05760d5621b27cca1890d3a1af6757c955aac network default NAD default NAD key ] I0428 11:16:56.853079 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-85f5bb4dfb-gl98c, hostIfaceName: 763c79511662020, network: default, NAD default, SandboxID: "763c795116620208c700c1d769d05760d5621b27cca1890d3a1af6757c955aac", PCI device ID: , UID: "49e80be3-516b-455e-8246-8bdf09aa757f", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0428 11:16:56.910763 3665 cni.go:473] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gl98c 763c795116620208c700c1d769d05760d5621b27cca1890d3a1af6757c955aac network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gl98c 763c795116620208c700c1d769d05760d5621b27cca1890d3a1af6757c955aac network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:04:b9:60:96:d8\",\"name\":\"763c79511662020\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f99a04d7-c32d-4243-b20f-3ac1412daffb\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:16:56.948979 3665 cni.go:452] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-n8zx8 9c0ca9bd31a0afdb175005ac9511a2c1a3837ff3c28a56af04ffd8e05ba0151e network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-n8zx8 9c0ca9bd31a0afdb175005ac9511a2c1a3837ff3c28a56af04ffd8e05ba0151e network default NAD default NAD key ] I0428 11:16:56.959194 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-7b6d6f4c94-n8zx8, hostIfaceName: 9c0ca9bd31a0afd, network: default, NAD default, SandboxID: "9c0ca9bd31a0afdb175005ac9511a2c1a3837ff3c28a56af04ffd8e05ba0151e", PCI device ID: , UID: "6accc42f-f031-4c21-aa10-d7fbb21386de", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0428 11:16:57.014768 3665 cni.go:473] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-n8zx8 9c0ca9bd31a0afdb175005ac9511a2c1a3837ff3c28a56af04ffd8e05ba0151e network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-n8zx8 9c0ca9bd31a0afdb175005ac9511a2c1a3837ff3c28a56af04ffd8e05ba0151e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:6d:f4:f7:99:d6\",\"name\":\"9c0ca9bd31a0afd\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cb849dbe-05f8-4b28-9292-769d0cdf19f4\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:16:57.034690 3665 cni.go:452] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-4vp9l 5acb9b7cadaaf115e662644dcbf4bfbec07fbbdc478cb24cd83ca081058069e0 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-4vp9l 5acb9b7cadaaf115e662644dcbf4bfbec07fbbdc478cb24cd83ca081058069e0 network default NAD default NAD key ] I0428 11:16:57.045381 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-57f84cff6f-4vp9l, hostIfaceName: 5acb9b7cadaaf11, network: default, NAD default, SandboxID: "5acb9b7cadaaf115e662644dcbf4bfbec07fbbdc478cb24cd83ca081058069e0", PCI device ID: , UID: "6c052b5a-7743-44b0-be38-2ef7aac04219", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0428 11:16:57.100387 3665 cni.go:473] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-4vp9l 5acb9b7cadaaf115e662644dcbf4bfbec07fbbdc478cb24cd83ca081058069e0 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-4vp9l 5acb9b7cadaaf115e662644dcbf4bfbec07fbbdc478cb24cd83ca081058069e0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:e9:4e:18:49:de\",\"name\":\"5acb9b7cadaaf11\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c3810a89-447e-457b-81de-5f5fcd2f5fca\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:17:00.103761 3665 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 11:17:00.103783 3665 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 11:17:00.103790 3665 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 11:17:00.128987 3665 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 11:17:00.129010 3665 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 11:17:00.129015 3665 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0428 11:17:01.543150 3665 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:17:01.543680 3665 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:17:01.565547 3665 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:17:01.565574 3665 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:17:01.571745 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0428 11:17:01.571776 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions W0428 11:17:01.757015 3665 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 11:17:01.757036 3665 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 11:17:01.757043 3665 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 11:17:01.757047 3665 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0428 11:17:09.685934 3665 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=84 W0428 11:17:11.460606 3665 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0428 11:17:11.484606 3665 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0428 11:17:12.738368 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 11:17:12.738388 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 11:17:12.738394 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 11:17:12.738398 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0428 11:17:19.684993 3665 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-zrc55] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-zrc55 for pod on switch ip-10-0-2-140.ec2.internal I0428 11:17:19.685098 3665 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-zrc55 I0428 11:17:19.698768 3665 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-zrc55] pod update took 13.701883ms I0428 11:17:19.698795 3665 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-zrc55] addLogicalPort annotation time took 13.733524ms I0428 11:17:19.700101 3665 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-zrc55] addLogicalPort took 15.126909ms, libovsdb time 890.38µs I0428 11:17:20.071323 3665 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-zrc55 f17bd0919339a4c2fdcea320ab78900dd6e162a726d0e87204d21a24f14461b0 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-zrc55 f17bd0919339a4c2fdcea320ab78900dd6e162a726d0e87204d21a24f14461b0 network default NAD default NAD key ] I0428 11:17:20.081726 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-zrc55, hostIfaceName: f17bd0919339a4c, network: default, NAD default, SandboxID: "f17bd0919339a4c2fdcea320ab78900dd6e162a726d0e87204d21a24f14461b0", PCI device ID: , UID: "54aa6679-9f5a-42dc-850d-4b160f0d904d", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0428 11:17:20.138844 3665 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-zrc55 f17bd0919339a4c2fdcea320ab78900dd6e162a726d0e87204d21a24f14461b0 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-zrc55 f17bd0919339a4c2fdcea320ab78900dd6e162a726d0e87204d21a24f14461b0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:24:72:8e:4d:be\",\"name\":\"f17bd0919339a4c\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b57a550a-1c67-4586-9dcf-eee0d05470e3\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:17:24.245452 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0428 11:17:26.782122 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0428 11:17:34.721360 3665 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:17:34.721931 3665 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:17:34.743350 3665 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:17:34.743373 3665 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:17:34.750822 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0428 11:17:34.750852 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0428 11:17:36.670909 3665 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0428 11:17:36.671673 3665 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0428 11:17:36.671696 3665 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0428 11:17:36.674046 3665 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0428 11:17:37.050007 3665 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-140.ec2.internal I0428 11:17:37.050113 3665 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0428 11:17:37.073332 3665 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 23.250204ms I0428 11:17:37.073357 3665 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 23.280292ms I0428 11:17:37.074700 3665 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 24.70758ms, libovsdb time 977.861µs I0428 11:17:37.421906 3665 cni.go:452] [tsf-keycloak/keycloak-0 b80c586229429d128d30c67f613bb7f6b155d546ce658217646e059f3916bf19 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-0 b80c586229429d128d30c67f613bb7f6b155d546ce658217646e059f3916bf19 network default NAD default NAD key ] I0428 11:17:37.431935 3665 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: b80c586229429d1, network: default, NAD default, SandboxID: "b80c586229429d128d30c67f613bb7f6b155d546ce658217646e059f3916bf19", PCI device ID: , UID: "796beca9-57a8-494c-b18e-135b39b3c0a0", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0428 11:17:37.489866 3665 cni.go:473] [tsf-keycloak/keycloak-0 b80c586229429d128d30c67f613bb7f6b155d546ce658217646e059f3916bf19 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-0 b80c586229429d128d30c67f613bb7f6b155d546ce658217646e059f3916bf19 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:c4:e2:e1:ae:bb\",\"name\":\"b80c586229429d1\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d47aee8d-2cdb-4cfa-82a2-fba291412f0a\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:17:41.495422 3665 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0428 11:17:42.932001 3665 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0428 11:17:42.932030 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0428 11:17:42.935782 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 11:17:42.935891 3665 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service I0428 11:17:52.290921 3665 namespace.go:142] [openshift-pipelines] updating namespace W0428 11:18:05.737702 3665 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 11:18:05.737749 3665 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 11:18:05.737754 3665 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 11:18:09.683936 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 11:18:09.683968 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9000: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 11:18:09.687562 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0428 11:18:09.687590 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak I0428 11:18:27.031610 3665 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:18:27.032198 3665 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:18:27.077232 3665 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:18:27.077256 3665 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:18:27.084027 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0428 11:18:27.084056 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0428 11:18:29.724455 3665 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-x6w59-7xlws] creating logical port tsf-tas_ctlog-createtree-job-x6w59-7xlws for pod on switch ip-10-0-2-140.ec2.internal I0428 11:18:29.724559 3665 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-x6w59-7xlws I0428 11:18:29.740347 3665 pod.go:62] [tsf-tas/ctlog-createtree-job-x6w59-7xlws] pod update took 15.808762ms I0428 11:18:29.740459 3665 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-createtree-job-x6w59-7xlws] addLogicalPort annotation time took 15.926649ms I0428 11:18:29.742027 3665 pods.go:271] [tsf-tas/ctlog-createtree-job-x6w59-7xlws] addLogicalPort took 17.599456ms, libovsdb time 1.091798ms I0428 11:18:29.791986 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0428 11:18:30.011401 3665 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-7666c7ddcb-j5phm] creating logical port tsf-tas_trillian-logserver-7666c7ddcb-j5phm for pod on switch ip-10-0-2-140.ec2.internal I0428 11:18:30.011501 3665 kube.go:256] Updating pod tsf-tas/trillian-logserver-7666c7ddcb-j5phm I0428 11:18:30.031259 3665 pod.go:62] [tsf-tas/trillian-logserver-7666c7ddcb-j5phm] pod update took 19.783339ms I0428 11:18:30.031293 3665 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logserver-7666c7ddcb-j5phm] addLogicalPort annotation time took 19.823308ms I0428 11:18:30.032584 3665 pods.go:271] [tsf-tas/trillian-logserver-7666c7ddcb-j5phm] addLogicalPort took 21.201364ms, libovsdb time 882.156µs I0428 11:18:30.090333 3665 cni.go:452] [tsf-tas/ctlog-createtree-job-x6w59-7xlws e560a50ec93f4ad91450aec7dadb8de7498e17c4487f9baefcdd0b73be8d60e7 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-createtree-job-x6w59-7xlws e560a50ec93f4ad91450aec7dadb8de7498e17c4487f9baefcdd0b73be8d60e7 network default NAD default NAD key ] I0428 11:18:30.100026 3665 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-x6w59-7xlws, hostIfaceName: e560a50ec93f4ad, network: default, NAD default, SandboxID: "e560a50ec93f4ad91450aec7dadb8de7498e17c4487f9baefcdd0b73be8d60e7", PCI device ID: , UID: "662f104e-b0e0-4f8e-a4c0-2a8aea468264", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0428 11:18:30.158332 3665 cni.go:473] [tsf-tas/ctlog-createtree-job-x6w59-7xlws e560a50ec93f4ad91450aec7dadb8de7498e17c4487f9baefcdd0b73be8d60e7 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-x6w59-7xlws e560a50ec93f4ad91450aec7dadb8de7498e17c4487f9baefcdd0b73be8d60e7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:74:52:65:b4:94\",\"name\":\"e560a50ec93f4ad\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/90ec2598-4d9f-4b7c-bac4-3c444ec309d8\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:18:30.275840 3665 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-42jbw] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-42jbw for pod on switch ip-10-0-2-140.ec2.internal I0428 11:18:30.275979 3665 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-42jbw I0428 11:18:30.294791 3665 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-42jbw] pod update took 18.833452ms I0428 11:18:30.294815 3665 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-42jbw] addLogicalPort annotation time took 18.864131ms I0428 11:18:30.296442 3665 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-42jbw] addLogicalPort took 20.626104ms, libovsdb time 1.079076ms I0428 11:18:30.638611 3665 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-42jbw ef0dc57b6618a5a71aacd0e89ca68b73e1c1be808026e906b8157ba054827b7f network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-42jbw ef0dc57b6618a5a71aacd0e89ca68b73e1c1be808026e906b8157ba054827b7f network default NAD default NAD key ] I0428 11:18:30.648683 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-42jbw, hostIfaceName: ef0dc57b6618a5a, network: default, NAD default, SandboxID: "ef0dc57b6618a5a71aacd0e89ca68b73e1c1be808026e906b8157ba054827b7f", PCI device ID: , UID: "06630351-bcd4-4ef3-9ba9-f5eedb3a9a37", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0428 11:18:30.704880 3665 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-42jbw ef0dc57b6618a5a71aacd0e89ca68b73e1c1be808026e906b8157ba054827b7f network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-42jbw ef0dc57b6618a5a71aacd0e89ca68b73e1c1be808026e906b8157ba054827b7f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:98:16:79:6d:b9\",\"name\":\"ef0dc57b6618a5a\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7b35c57a-7523-43dc-ad9d-df3fbf44bb6e\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:18:30.814184 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=11 I0428 11:18:30.977509 3665 cni.go:452] [tsf-tas/trillian-logserver-7666c7ddcb-j5phm a0319fe420dbe394658d2e271691dc9751c806e82a1be6e2513d47d1adfb6c0b network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logserver-7666c7ddcb-j5phm a0319fe420dbe394658d2e271691dc9751c806e82a1be6e2513d47d1adfb6c0b network default NAD default NAD key ] I0428 11:18:30.988477 3665 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-7666c7ddcb-j5phm, hostIfaceName: a0319fe420dbe39, network: default, NAD default, SandboxID: "a0319fe420dbe394658d2e271691dc9751c806e82a1be6e2513d47d1adfb6c0b", PCI device ID: , UID: "19b06c92-0ab7-486f-b407-801d6b82b62a", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0428 11:18:31.050601 3665 cni.go:473] [tsf-tas/trillian-logserver-7666c7ddcb-j5phm a0319fe420dbe394658d2e271691dc9751c806e82a1be6e2513d47d1adfb6c0b network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logserver-7666c7ddcb-j5phm a0319fe420dbe394658d2e271691dc9751c806e82a1be6e2513d47d1adfb6c0b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:9e:2b:66:79:03\",\"name\":\"a0319fe420dbe39\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/22e8a85c-6cbe-4371-ad3b-14310affbd3f\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:18:33.209016 3665 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0428 11:18:34.924972 3665 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0428 11:18:34.924996 3665 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0428 11:18:37.949455 3665 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0428 11:18:39.649949 3665 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0428 11:18:39.649978 3665 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0428 11:18:40.252251 3665 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0428 11:18:41.667649 3665 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0428 11:18:42.108411 3665 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-sgpl9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:18:42.109038 3665 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-sgpl9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:18:42.809280 3665 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-hlf7g-lklvg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:18:42.809889 3665 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-hlf7g-lklvg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:18:42.996514 3665 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-sgpl9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:18:42.996546 3665 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-sgpl9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:18:43.709283 3665 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-hlf7g-lklvg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:18:43.709303 3665 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-hlf7g-lklvg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 11:18:44.219690 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0428 11:19:08.352622 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0428 11:19:19.324849 3665 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:19.325403 3665 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:19.343305 3665 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:19.343326 3665 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:19.350638 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0428 11:19:19.350664 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0428 11:19:19.459888 3665 cni.go:452] [tsf-tas/ctlog-createtree-job-x6w59-7xlws e560a50ec93f4ad91450aec7dadb8de7498e17c4487f9baefcdd0b73be8d60e7 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/ctlog-createtree-job-x6w59-7xlws e560a50ec93f4ad91450aec7dadb8de7498e17c4487f9baefcdd0b73be8d60e7 network default NAD default NAD key ] I0428 11:19:19.534043 3665 cni.go:473] [tsf-tas/ctlog-createtree-job-x6w59-7xlws e560a50ec93f4ad91450aec7dadb8de7498e17c4487f9baefcdd0b73be8d60e7 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-x6w59-7xlws e560a50ec93f4ad91450aec7dadb8de7498e17c4487f9baefcdd0b73be8d60e7 network default NAD default NAD key default], result "{}", err I0428 11:19:19.582990 3665 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-x6w59-7xlws of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:19.583027 3665 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-x6w59-7xlws I0428 11:19:19.584444 3665 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-x6w59-7xlws, ips: 10.129.0.37 I0428 11:19:19.584476 3665 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-x6w59-7xlws of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:20.473655 3665 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-x6w59-7xlws of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:20.473682 3665 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-x6w59-7xlws of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 11:19:27.215535 3665 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0428 11:19:28.369743 3665 obj_retry.go:498] Detected object tsf/patch-tekton-config-v46r2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:28.370301 3665 obj_retry.go:498] Detected object tsf/patch-tekton-config-v46r2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:29.065339 3665 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-4tjjw-7kcbn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:29.065930 3665 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-4tjjw-7kcbn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:29.239234 3665 obj_retry.go:492] Detected object tsf/patch-tekton-config-v46r2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:29.239257 3665 obj_retry.go:492] Detected object tsf/patch-tekton-config-v46r2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:29.952957 3665 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-4tjjw-7kcbn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:29.952992 3665 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-4tjjw-7kcbn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:33.886345 3665 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-865d7c784d-2t5gl] creating logical port tsf-tas_rekor-server-865d7c784d-2t5gl for pod on switch ip-10-0-2-140.ec2.internal I0428 11:19:33.886436 3665 kube.go:256] Updating pod tsf-tas/rekor-server-865d7c784d-2t5gl I0428 11:19:33.902824 3665 pod.go:62] [tsf-tas/rekor-server-865d7c784d-2t5gl] pod update took 16.412724ms I0428 11:19:33.902852 3665 base_network_controller_pods.go:951] [default/tsf-tas/rekor-server-865d7c784d-2t5gl] addLogicalPort annotation time took 16.445237ms I0428 11:19:33.904121 3665 pods.go:271] [tsf-tas/rekor-server-865d7c784d-2t5gl] addLogicalPort took 17.791486ms, libovsdb time 871.638µs W0428 11:19:34.976128 3665 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0428 11:19:34.994331 3665 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0428 11:19:35.398418 3665 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-7s4vj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:35.398961 3665 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-7s4vj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:36.277876 3665 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-7s4vj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:36.277902 3665 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-7s4vj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:36.312297 3665 obj_retry.go:492] Detected object tsf/patch-tekton-config-v46r2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:36.312319 3665 obj_retry.go:492] Detected object tsf/patch-tekton-config-v46r2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:36.315509 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-v46r2 I0428 11:19:36.315537 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-v46r2 I0428 11:19:40.441112 3665 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:40.441647 3665 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:40.461958 3665 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:40.461981 3665 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:40.468121 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0428 11:19:40.468146 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0428 11:19:44.448049 3665 cni.go:452] [tsf-tas/rekor-server-865d7c784d-2t5gl 9c52a89ea943f14c762b8ce9525aa171597f45cfe64a6ea019f8ef9e3a2974c3 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-server-865d7c784d-2t5gl 9c52a89ea943f14c762b8ce9525aa171597f45cfe64a6ea019f8ef9e3a2974c3 network default NAD default NAD key ] I0428 11:19:44.460686 3665 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-865d7c784d-2t5gl, hostIfaceName: 9c52a89ea943f14, network: default, NAD default, SandboxID: "9c52a89ea943f14c762b8ce9525aa171597f45cfe64a6ea019f8ef9e3a2974c3", PCI device ID: , UID: "75290392-3914-4830-8e2d-71367f3261db", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0428 11:19:44.521310 3665 cni.go:473] [tsf-tas/rekor-server-865d7c784d-2t5gl 9c52a89ea943f14c762b8ce9525aa171597f45cfe64a6ea019f8ef9e3a2974c3 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-server-865d7c784d-2t5gl 9c52a89ea943f14c762b8ce9525aa171597f45cfe64a6ea019f8ef9e3a2974c3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:01:33:d3:33:3b\",\"name\":\"9c52a89ea943f14\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ea38af4a-c241-4546-902a-00bc5f57069e\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:19:46.465787 3665 obj_retry.go:498] Detected object tsf-tpa/create-db-wgdmf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:46.466434 3665 obj_retry.go:498] Detected object tsf-tpa/create-db-wgdmf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0428 11:19:46.601780 3665 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0428 11:19:47.361301 3665 obj_retry.go:492] Detected object tsf-tpa/create-db-wgdmf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:47.361327 3665 obj_retry.go:492] Detected object tsf-tpa/create-db-wgdmf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:47.466184 3665 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-2xgzg] creating logical port tsf-tpa_migrate-db-2xgzg for pod on switch ip-10-0-2-140.ec2.internal I0428 11:19:47.466313 3665 kube.go:256] Updating pod tsf-tpa/migrate-db-2xgzg I0428 11:19:47.479876 3665 pod.go:62] [tsf-tpa/migrate-db-2xgzg] pod update took 13.608576ms I0428 11:19:47.479900 3665 base_network_controller_pods.go:951] [default/tsf-tpa/migrate-db-2xgzg] addLogicalPort annotation time took 13.6394ms I0428 11:19:47.481187 3665 pods.go:271] [tsf-tpa/migrate-db-2xgzg] addLogicalPort took 15.024965ms, libovsdb time 894.716µs I0428 11:19:47.824962 3665 cni.go:452] [tsf-tpa/migrate-db-2xgzg cb7509e58ae789e5892109789f103d2b79101f0e39640751b49bcfc0879a1079 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/migrate-db-2xgzg cb7509e58ae789e5892109789f103d2b79101f0e39640751b49bcfc0879a1079 network default NAD default NAD key ] I0428 11:19:47.834475 3665 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-2xgzg, hostIfaceName: cb7509e58ae789e, network: default, NAD default, SandboxID: "cb7509e58ae789e5892109789f103d2b79101f0e39640751b49bcfc0879a1079", PCI device ID: , UID: "158b587c-983b-4f0a-a166-09740e06c95c", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0428 11:19:47.890494 3665 cni.go:473] [tsf-tpa/migrate-db-2xgzg cb7509e58ae789e5892109789f103d2b79101f0e39640751b49bcfc0879a1079 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/migrate-db-2xgzg cb7509e58ae789e5892109789f103d2b79101f0e39640751b49bcfc0879a1079 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:8c:2b:9d:65:d3\",\"name\":\"cb7509e58ae789e\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b824c6c7-c1cd-47cd-b5ae-abfc0defe32e\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:19:52.629374 3665 cni.go:452] [tsf-tpa/migrate-db-2xgzg cb7509e58ae789e5892109789f103d2b79101f0e39640751b49bcfc0879a1079 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/migrate-db-2xgzg cb7509e58ae789e5892109789f103d2b79101f0e39640751b49bcfc0879a1079 network default NAD default NAD key ] I0428 11:19:52.719950 3665 cni.go:473] [tsf-tpa/migrate-db-2xgzg cb7509e58ae789e5892109789f103d2b79101f0e39640751b49bcfc0879a1079 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/migrate-db-2xgzg cb7509e58ae789e5892109789f103d2b79101f0e39640751b49bcfc0879a1079 network default NAD default NAD key default], result "{}", err I0428 11:19:52.770603 3665 obj_retry.go:498] Detected object tsf-tpa/migrate-db-2xgzg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:52.770648 3665 pods.go:176] Deleting pod: tsf-tpa/migrate-db-2xgzg I0428 11:19:52.772005 3665 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-2xgzg, ips: 10.129.0.41 I0428 11:19:52.772040 3665 obj_retry.go:498] Detected object tsf-tpa/migrate-db-2xgzg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:53.648450 3665 obj_retry.go:492] Detected object tsf-tpa/migrate-db-2xgzg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:53.648480 3665 obj_retry.go:492] Detected object tsf-tpa/migrate-db-2xgzg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:53.829234 3665 base_network_controller_pods.go:487] [default/tsf-tpa/create-importers-bklhb] creating logical port tsf-tpa_create-importers-bklhb for pod on switch ip-10-0-2-140.ec2.internal I0428 11:19:53.829359 3665 kube.go:256] Updating pod tsf-tpa/create-importers-bklhb I0428 11:19:53.842256 3665 pod.go:62] [tsf-tpa/create-importers-bklhb] pod update took 12.915814ms I0428 11:19:53.842286 3665 base_network_controller_pods.go:951] [default/tsf-tpa/create-importers-bklhb] addLogicalPort annotation time took 12.950529ms I0428 11:19:53.843553 3665 pods.go:271] [tsf-tpa/create-importers-bklhb] addLogicalPort took 14.33806ms, libovsdb time 866.771µs I0428 11:19:54.191048 3665 cni.go:452] [tsf-tpa/create-importers-bklhb 3340ac22e22f6a32ea23795a87ca0df7a3f8ae958ebcf499aedaf056ef77d600 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-importers-bklhb 3340ac22e22f6a32ea23795a87ca0df7a3f8ae958ebcf499aedaf056ef77d600 network default NAD default NAD key ] I0428 11:19:54.200974 3665 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-importers-bklhb, hostIfaceName: 3340ac22e22f6a3, network: default, NAD default, SandboxID: "3340ac22e22f6a32ea23795a87ca0df7a3f8ae958ebcf499aedaf056ef77d600", PCI device ID: , UID: "19b4a856-4590-4379-983a-5d7cc734c646", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0428 11:19:54.255862 3665 cni.go:473] [tsf-tpa/create-importers-bklhb 3340ac22e22f6a32ea23795a87ca0df7a3f8ae958ebcf499aedaf056ef77d600 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-importers-bklhb 3340ac22e22f6a32ea23795a87ca0df7a3f8ae958ebcf499aedaf056ef77d600 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:44:a1:36:94:5d\",\"name\":\"3340ac22e22f6a3\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb0db192-614f-4c23-8a49-8042e58c5cba\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:19:55.648459 3665 cni.go:452] [tsf-tpa/create-importers-bklhb 3340ac22e22f6a32ea23795a87ca0df7a3f8ae958ebcf499aedaf056ef77d600 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-importers-bklhb 3340ac22e22f6a32ea23795a87ca0df7a3f8ae958ebcf499aedaf056ef77d600 network default NAD default NAD key ] I0428 11:19:55.726994 3665 cni.go:473] [tsf-tpa/create-importers-bklhb 3340ac22e22f6a32ea23795a87ca0df7a3f8ae958ebcf499aedaf056ef77d600 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-importers-bklhb 3340ac22e22f6a32ea23795a87ca0df7a3f8ae958ebcf499aedaf056ef77d600 network default NAD default NAD key default], result "{}", err I0428 11:19:55.777770 3665 obj_retry.go:498] Detected object tsf-tpa/create-importers-bklhb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:55.777808 3665 pods.go:176] Deleting pod: tsf-tpa/create-importers-bklhb I0428 11:19:55.779258 3665 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-importers-bklhb, ips: 10.129.0.42 I0428 11:19:55.779289 3665 obj_retry.go:498] Detected object tsf-tpa/create-importers-bklhb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:19:56.802415 3665 obj_retry.go:492] Detected object tsf-tpa/create-importers-bklhb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:19:56.802442 3665 obj_retry.go:492] Detected object tsf-tpa/create-importers-bklhb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 11:19:57.599667 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0428 11:20:00.188188 3665 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-576k8] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29622920-576k8 for pod on switch ip-10-0-2-140.ec2.internal I0428 11:20:00.188289 3665 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-576k8 I0428 11:20:00.205764 3665 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-576k8] pod update took 17.496133ms I0428 11:20:00.205790 3665 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-576k8] addLogicalPort annotation time took 17.527469ms I0428 11:20:00.207098 3665 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-576k8] addLogicalPort took 18.932418ms, libovsdb time 881.918µs I0428 11:20:00.548696 3665 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-576k8 f32134a0acfbf23a6e6d1246c30807c092907ab3690f77c1b5c3fb3dd97bea01 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-576k8 f32134a0acfbf23a6e6d1246c30807c092907ab3690f77c1b5c3fb3dd97bea01 network default NAD default NAD key ] I0428 11:20:00.558622 3665 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29622920-576k8, hostIfaceName: f32134a0acfbf23, network: default, NAD default, SandboxID: "f32134a0acfbf23a6e6d1246c30807c092907ab3690f77c1b5c3fb3dd97bea01", PCI device ID: , UID: "760966de-ef59-4623-ad9d-7b9b877f5103", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0428 11:20:00.614629 3665 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-576k8 f32134a0acfbf23a6e6d1246c30807c092907ab3690f77c1b5c3fb3dd97bea01 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-576k8 f32134a0acfbf23a6e6d1246c30807c092907ab3690f77c1b5c3fb3dd97bea01 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:04:9c:e9:a0:15\",\"name\":\"f32134a0acfbf23\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bde7bfd8-85c9-4554-bf70-6a5dc0495ee3\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:20:01.112593 3665 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622910-bnwtf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:20:01.113177 3665 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622910-bnwtf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:20:01.121964 3665 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622910-bnwtf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:20:01.121987 3665 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622910-bnwtf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:20:01.128257 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29622910-bnwtf I0428 11:20:01.128284 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29622910-bnwtf I0428 11:20:02.765336 3665 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-qfkzs-4jcdg] creating logical port tsf-tas_tuf-repository-init-qfkzs-4jcdg for pod on switch ip-10-0-2-140.ec2.internal I0428 11:20:02.765433 3665 kube.go:256] Updating pod tsf-tas/tuf-repository-init-qfkzs-4jcdg I0428 11:20:02.778681 3665 pod.go:62] [tsf-tas/tuf-repository-init-qfkzs-4jcdg] pod update took 13.267246ms I0428 11:20:02.778728 3665 base_network_controller_pods.go:951] [default/tsf-tas/tuf-repository-init-qfkzs-4jcdg] addLogicalPort annotation time took 13.304837ms I0428 11:20:02.780135 3665 pods.go:271] [tsf-tas/tuf-repository-init-qfkzs-4jcdg] addLogicalPort took 14.814847ms, libovsdb time 852.654µs W0428 11:20:05.453353 3665 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0428 11:20:07.027532 3665 cni.go:452] [tsf-tas/tuf-repository-init-qfkzs-4jcdg 45ac529933641df48f2e5f19f1e57755e14c6ce5908a3afa586177feb88a210f network default NAD default NAD key ] ADD starting CNI request [tsf-tas/tuf-repository-init-qfkzs-4jcdg 45ac529933641df48f2e5f19f1e57755e14c6ce5908a3afa586177feb88a210f network default NAD default NAD key ] I0428 11:20:07.037467 3665 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-qfkzs-4jcdg, hostIfaceName: 45ac529933641df, network: default, NAD default, SandboxID: "45ac529933641df48f2e5f19f1e57755e14c6ce5908a3afa586177feb88a210f", PCI device ID: , UID: "5aa42e6d-02a1-42f0-a486-27101caed581", MAC: 0a:58:0a:81:00:2c, IPs: [10.129.0.44/23] I0428 11:20:07.095625 3665 cni.go:473] [tsf-tas/tuf-repository-init-qfkzs-4jcdg 45ac529933641df48f2e5f19f1e57755e14c6ce5908a3afa586177feb88a210f network default NAD default NAD key default] ADD finished CNI request [tsf-tas/tuf-repository-init-qfkzs-4jcdg 45ac529933641df48f2e5f19f1e57755e14c6ce5908a3afa586177feb88a210f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:b2:84:f1:1a:8e\",\"name\":\"45ac529933641df\"},{\"mac\":\"0a:58:0a:81:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1bc811df-aa1c-4f8b-8a2c-60744a63eed0\"}],\"ips\":[{\"address\":\"10.129.0.44/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:20:08.608730 3665 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:20:08.609311 3665 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:20:08.629922 3665 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:20:08.629950 3665 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:20:08.636343 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0428 11:20:08.636372 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0428 11:20:08.693630 3665 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=67 I0428 11:20:09.404283 3665 namespace.go:98] [integration-service] adding namespace I0428 11:20:09.404442 3665 namespace.go:98] [image-controller] adding namespace I0428 11:20:09.405292 3665 namespace.go:102] [integration-service] adding namespace took 986.099µs I0428 11:20:09.406021 3665 namespace.go:102] [image-controller] adding namespace took 1.559769ms I0428 11:20:09.416814 3665 namespace.go:98] [default-tenant] adding namespace I0428 11:20:09.416978 3665 namespace.go:98] [build-service] adding namespace I0428 11:20:09.417613 3665 namespace.go:102] [default-tenant] adding namespace took 779.57µs I0428 11:20:09.418237 3665 namespace.go:102] [build-service] adding namespace took 1.241764ms I0428 11:20:09.469693 3665 namespace.go:142] [integration-service] updating namespace I0428 11:20:09.520035 3665 namespace.go:142] [integration-service] updating namespace I0428 11:20:09.572245 3665 namespace.go:142] [image-controller] updating namespace I0428 11:20:09.620930 3665 namespace.go:142] [image-controller] updating namespace I0428 11:20:09.652628 3665 namespace.go:142] [default-tenant] updating namespace I0428 11:20:09.687976 3665 namespace.go:142] [default-tenant] updating namespace I0428 11:20:09.707876 3665 namespace.go:142] [build-service] updating namespace I0428 11:20:09.746911 3665 namespace.go:142] [build-service] updating namespace I0428 11:20:09.763869 3665 namespace.go:142] [integration-service] updating namespace I0428 11:20:09.783832 3665 namespace.go:142] [build-service] updating namespace I0428 11:20:09.822687 3665 namespace.go:98] [release-service] adding namespace I0428 11:20:09.823571 3665 namespace.go:102] [release-service] adding namespace took 862.101µs I0428 11:20:09.869091 3665 namespace.go:142] [konflux-ui] updating namespace I0428 11:20:09.887787 3665 namespace.go:142] [release-service] updating namespace I0428 11:20:09.936437 3665 namespace.go:142] [release-service] updating namespace I0428 11:20:10.067367 3665 namespace.go:98] [konflux-info] adding namespace I0428 11:20:10.068192 3665 namespace.go:102] [konflux-info] adding namespace took 808.625µs I0428 11:20:10.170693 3665 namespace.go:98] [namespace-lister] adding namespace I0428 11:20:10.171642 3665 namespace.go:102] [namespace-lister] adding namespace took 906.675µs I0428 11:20:10.249911 3665 namespace.go:142] [konflux-info] updating namespace I0428 11:20:10.315277 3665 namespace.go:98] [enterprise-contract-service] adding namespace I0428 11:20:10.316290 3665 namespace.go:102] [enterprise-contract-service] adding namespace took 989.252µs I0428 11:20:10.502883 3665 namespace.go:142] [image-controller] updating namespace I0428 11:20:10.512851 3665 namespace.go:142] [konflux-info] updating namespace I0428 11:20:10.811911 3665 namespace.go:142] [namespace-lister] updating namespace I0428 11:20:11.185916 3665 namespace.go:142] [namespace-lister] updating namespace I0428 11:20:11.456933 3665 namespace.go:142] [enterprise-contract-service] updating namespace I0428 11:20:11.524806 3665 namespace.go:142] [default-tenant] updating namespace I0428 11:20:11.725767 3665 namespace.go:142] [enterprise-contract-service] updating namespace I0428 11:20:11.984855 3665 namespace.go:98] [segment-bridge] adding namespace I0428 11:20:11.985703 3665 namespace.go:102] [segment-bridge] adding namespace took 822.915µs I0428 11:20:12.056776 3665 namespace.go:98] [konflux-cli] adding namespace I0428 11:20:12.057697 3665 namespace.go:102] [konflux-cli] adding namespace took 896.187µs I0428 11:20:12.231864 3665 namespace.go:142] [segment-bridge] updating namespace I0428 11:20:12.381651 3665 namespace.go:142] [segment-bridge] updating namespace I0428 11:20:12.436651 3665 namespace.go:142] [konflux-cli] updating namespace I0428 11:20:12.529916 3665 namespace.go:142] [konflux-cli] updating namespace I0428 11:20:12.742566 3665 cni.go:452] [tsf-tas/tuf-repository-init-qfkzs-4jcdg 45ac529933641df48f2e5f19f1e57755e14c6ce5908a3afa586177feb88a210f network default NAD default NAD key ] DEL starting CNI request [tsf-tas/tuf-repository-init-qfkzs-4jcdg 45ac529933641df48f2e5f19f1e57755e14c6ce5908a3afa586177feb88a210f network default NAD default NAD key ] I0428 11:20:12.818984 3665 cni.go:473] [tsf-tas/tuf-repository-init-qfkzs-4jcdg 45ac529933641df48f2e5f19f1e57755e14c6ce5908a3afa586177feb88a210f network default NAD default NAD key default] DEL finished CNI request [tsf-tas/tuf-repository-init-qfkzs-4jcdg 45ac529933641df48f2e5f19f1e57755e14c6ce5908a3afa586177feb88a210f network default NAD default NAD key default], result "{}", err I0428 11:20:12.987987 3665 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-qfkzs-4jcdg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:20:12.988020 3665 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-qfkzs-4jcdg I0428 11:20:12.989436 3665 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-qfkzs-4jcdg, ips: 10.129.0.44 I0428 11:20:12.989473 3665 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-qfkzs-4jcdg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:20:13.083817 3665 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0428 11:20:13.084482 3665 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0428 11:20:13.084502 3665 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0428 11:20:13.087494 3665 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0428 11:20:13.134826 3665 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0428 11:20:13.134854 3665 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0428 11:20:13.135685 3665 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0428 11:20:13.338622 3665 base_network_controller_pods.go:487] [default/konflux-ui/dex-f5b7985bb-bhs4c] creating logical port konflux-ui_dex-f5b7985bb-bhs4c for pod on switch ip-10-0-2-140.ec2.internal I0428 11:20:13.338746 3665 kube.go:256] Updating pod konflux-ui/dex-f5b7985bb-bhs4c I0428 11:20:13.361275 3665 base_network_controller_pods.go:487] [default/konflux-ui/proxy-579597865b-cntsh] creating logical port konflux-ui_proxy-579597865b-cntsh for pod on switch ip-10-0-2-140.ec2.internal I0428 11:20:13.361384 3665 kube.go:256] Updating pod konflux-ui/proxy-579597865b-cntsh I0428 11:20:13.363667 3665 pod.go:62] [konflux-ui/dex-f5b7985bb-bhs4c] pod update took 24.969822ms I0428 11:20:13.363690 3665 base_network_controller_pods.go:951] [default/konflux-ui/dex-f5b7985bb-bhs4c] addLogicalPort annotation time took 25.000335ms I0428 11:20:13.365021 3665 pods.go:271] [konflux-ui/dex-f5b7985bb-bhs4c] addLogicalPort took 26.422627ms, libovsdb time 929.114µs I0428 11:20:13.386440 3665 pod.go:62] [konflux-ui/proxy-579597865b-cntsh] pod update took 25.093752ms I0428 11:20:13.386463 3665 base_network_controller_pods.go:951] [default/konflux-ui/proxy-579597865b-cntsh] addLogicalPort annotation time took 25.122239ms I0428 11:20:13.387701 3665 pods.go:271] [konflux-ui/proxy-579597865b-cntsh] addLogicalPort took 26.441466ms, libovsdb time 808.454µs I0428 11:20:13.540069 3665 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-6f859968fc-lfb8f] creating logical port image-controller_image-controller-controller-manager-6f859968fc-lfb8f for pod on switch ip-10-0-2-140.ec2.internal I0428 11:20:13.540167 3665 kube.go:256] Updating pod image-controller/image-controller-controller-manager-6f859968fc-lfb8f I0428 11:20:13.575519 3665 pod.go:62] [image-controller/image-controller-controller-manager-6f859968fc-lfb8f] pod update took 35.374925ms I0428 11:20:13.575551 3665 base_network_controller_pods.go:951] [default/image-controller/image-controller-controller-manager-6f859968fc-lfb8f] addLogicalPort annotation time took 35.413279ms I0428 11:20:13.576892 3665 pods.go:271] [image-controller/image-controller-controller-manager-6f859968fc-lfb8f] addLogicalPort took 36.83805ms, libovsdb time 856.661µs I0428 11:20:13.782104 3665 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-7b666bf6d8-rcm45] creating logical port integration-service_integration-service-controller-manager-7b666bf6d8-rcm45 for pod on switch ip-10-0-2-140.ec2.internal I0428 11:20:13.782231 3665 kube.go:256] Updating pod integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 I0428 11:20:13.810205 3665 pod.go:62] [integration-service/integration-service-controller-manager-7b666bf6d8-rcm45] pod update took 27.997629ms I0428 11:20:13.810304 3665 base_network_controller_pods.go:951] [default/integration-service/integration-service-controller-manager-7b666bf6d8-rcm45] addLogicalPort annotation time took 28.103051ms I0428 11:20:13.811595 3665 pods.go:271] [integration-service/integration-service-controller-manager-7b666bf6d8-rcm45] addLogicalPort took 29.508423ms, libovsdb time 842.698µs I0428 11:20:13.944755 3665 namespace.go:142] [build-service] updating namespace I0428 11:20:14.046664 3665 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7cbf488fcf-5qcvc] creating logical port build-service_build-service-controller-manager-7cbf488fcf-5qcvc for pod on switch ip-10-0-2-140.ec2.internal I0428 11:20:14.046832 3665 kube.go:256] Updating pod build-service/build-service-controller-manager-7cbf488fcf-5qcvc I0428 11:20:14.103621 3665 pod.go:62] [build-service/build-service-controller-manager-7cbf488fcf-5qcvc] pod update took 56.809035ms I0428 11:20:14.103654 3665 base_network_controller_pods.go:951] [default/build-service/build-service-controller-manager-7cbf488fcf-5qcvc] addLogicalPort annotation time took 56.848908ms I0428 11:20:14.103851 3665 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-qfkzs-4jcdg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:20:14.103876 3665 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-qfkzs-4jcdg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:20:14.105177 3665 pods.go:271] [build-service/build-service-controller-manager-7cbf488fcf-5qcvc] addLogicalPort took 58.534729ms, libovsdb time 955.704µs I0428 11:20:14.208638 3665 cni.go:452] [image-controller/image-controller-controller-manager-6f859968fc-lfb8f bfee378a95201d9240e3c498e89bfca412239273be03c8ab7edca9c21bdf5a42 network default NAD default NAD key ] ADD starting CNI request [image-controller/image-controller-controller-manager-6f859968fc-lfb8f bfee378a95201d9240e3c498e89bfca412239273be03c8ab7edca9c21bdf5a42 network default NAD default NAD key ] I0428 11:20:14.218646 3665 helper_linux.go:556] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-6f859968fc-lfb8f, hostIfaceName: bfee378a95201d9, network: default, NAD default, SandboxID: "bfee378a95201d9240e3c498e89bfca412239273be03c8ab7edca9c21bdf5a42", PCI device ID: , UID: "9d2b5b74-eb82-4046-b46a-58a4d44c1334", MAC: 0a:58:0a:81:00:2f, IPs: [10.129.0.47/23] I0428 11:20:14.277905 3665 cni.go:473] [image-controller/image-controller-controller-manager-6f859968fc-lfb8f bfee378a95201d9240e3c498e89bfca412239273be03c8ab7edca9c21bdf5a42 network default NAD default NAD key default] ADD finished CNI request [image-controller/image-controller-controller-manager-6f859968fc-lfb8f bfee378a95201d9240e3c498e89bfca412239273be03c8ab7edca9c21bdf5a42 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:97:3c:a7:b9:96\",\"name\":\"bfee378a95201d9\"},{\"mac\":\"0a:58:0a:81:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8cc768df-723a-4d0a-b93b-1e5ddac6c202\"}],\"ips\":[{\"address\":\"10.129.0.47/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:20:14.434801 3665 namespace.go:142] [default-tenant] updating namespace I0428 11:20:14.579749 3665 namespace.go:142] [enterprise-contract-service] updating namespace I0428 11:20:14.711558 3665 cni.go:452] [build-service/build-service-controller-manager-7cbf488fcf-5qcvc 0b885fa2a465cee8e2ee5697e49033c53c5060f1fdd27069da9253fbc296e082 network default NAD default NAD key ] ADD starting CNI request [build-service/build-service-controller-manager-7cbf488fcf-5qcvc 0b885fa2a465cee8e2ee5697e49033c53c5060f1fdd27069da9253fbc296e082 network default NAD default NAD key ] I0428 11:20:14.722475 3665 helper_linux.go:556] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7cbf488fcf-5qcvc, hostIfaceName: 0b885fa2a465cee, network: default, NAD default, SandboxID: "0b885fa2a465cee8e2ee5697e49033c53c5060f1fdd27069da9253fbc296e082", PCI device ID: , UID: "86ad330a-d30e-4d2b-8c4b-20d0c3a189c6", MAC: 0a:58:0a:81:00:31, IPs: [10.129.0.49/23] I0428 11:20:14.766321 3665 namespace.go:142] [image-controller] updating namespace I0428 11:20:14.787796 3665 cni.go:473] [build-service/build-service-controller-manager-7cbf488fcf-5qcvc 0b885fa2a465cee8e2ee5697e49033c53c5060f1fdd27069da9253fbc296e082 network default NAD default NAD key default] ADD finished CNI request [build-service/build-service-controller-manager-7cbf488fcf-5qcvc 0b885fa2a465cee8e2ee5697e49033c53c5060f1fdd27069da9253fbc296e082 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:ff:bc:b8:55:85\",\"name\":\"0b885fa2a465cee\"},{\"mac\":\"0a:58:0a:81:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/de850b6f-d31a-4bf5-b795-1cdaf691c57d\"}],\"ips\":[{\"address\":\"10.129.0.49/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:20:14.977834 3665 namespace.go:142] [integration-service] updating namespace I0428 11:20:15.051817 3665 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-6bfdb6bcb6-pldrd] creating logical port release-service_release-service-controller-manager-6bfdb6bcb6-pldrd for pod on switch ip-10-0-2-140.ec2.internal I0428 11:20:15.051984 3665 kube.go:256] Updating pod release-service/release-service-controller-manager-6bfdb6bcb6-pldrd I0428 11:20:15.105841 3665 pod.go:62] [release-service/release-service-controller-manager-6bfdb6bcb6-pldrd] pod update took 53.897729ms I0428 11:20:15.105877 3665 base_network_controller_pods.go:951] [default/release-service/release-service-controller-manager-6bfdb6bcb6-pldrd] addLogicalPort annotation time took 53.930696ms I0428 11:20:15.107419 3665 pods.go:271] [release-service/release-service-controller-manager-6bfdb6bcb6-pldrd] addLogicalPort took 55.677672ms, libovsdb time 1.076542ms I0428 11:20:15.233590 3665 namespace.go:142] [konflux-cli] updating namespace I0428 11:20:15.262116 3665 base_network_controller_pods.go:487] [default/tsf-tas/tuf-97fc8ff76-5vd7v] creating logical port tsf-tas_tuf-97fc8ff76-5vd7v for pod on switch ip-10-0-2-140.ec2.internal I0428 11:20:15.262216 3665 kube.go:256] Updating pod tsf-tas/tuf-97fc8ff76-5vd7v I0428 11:20:15.296399 3665 pod.go:62] [tsf-tas/tuf-97fc8ff76-5vd7v] pod update took 34.204935ms I0428 11:20:15.296430 3665 base_network_controller_pods.go:951] [default/tsf-tas/tuf-97fc8ff76-5vd7v] addLogicalPort annotation time took 34.243145ms I0428 11:20:15.297961 3665 pods.go:271] [tsf-tas/tuf-97fc8ff76-5vd7v] addLogicalPort took 35.870997ms, libovsdb time 1.018342ms I0428 11:20:15.399990 3665 namespace.go:142] [konflux-info] updating namespace I0428 11:20:15.602857 3665 namespace.go:142] [namespace-lister] updating namespace I0428 11:20:15.796154 3665 namespace.go:142] [release-service] updating namespace I0428 11:20:16.001289 3665 namespace.go:142] [segment-bridge] updating namespace I0428 11:20:16.122183 3665 namespace.go:142] [build-service] updating namespace I0428 11:20:16.138208 3665 namespace.go:142] [default-tenant] updating namespace I0428 11:20:16.156088 3665 namespace.go:142] [enterprise-contract-service] updating namespace I0428 11:20:16.172415 3665 namespace.go:142] [image-controller] updating namespace I0428 11:20:16.190335 3665 namespace.go:142] [integration-service] updating namespace I0428 11:20:16.210646 3665 namespace.go:142] [konflux-cli] updating namespace I0428 11:20:16.224543 3665 namespace.go:142] [konflux-info] updating namespace I0428 11:20:16.243704 3665 namespace.go:142] [namespace-lister] updating namespace I0428 11:20:16.252877 3665 namespace.go:142] [release-service] updating namespace I0428 11:20:16.270227 3665 namespace.go:142] [segment-bridge] updating namespace I0428 11:20:16.347802 3665 namespace.go:142] [build-service] updating namespace I0428 11:20:16.422571 3665 namespace.go:142] [default-tenant] updating namespace I0428 11:20:16.475339 3665 namespace.go:142] [enterprise-contract-service] updating namespace I0428 11:20:16.564332 3665 namespace.go:142] [image-controller] updating namespace I0428 11:20:16.661243 3665 namespace.go:142] [integration-service] updating namespace I0428 11:20:16.763437 3665 namespace.go:142] [konflux-cli] updating namespace I0428 11:20:16.866129 3665 namespace.go:142] [konflux-info] updating namespace I0428 11:20:16.962208 3665 namespace.go:142] [namespace-lister] updating namespace I0428 11:20:17.176920 3665 namespace.go:142] [release-service] updating namespace I0428 11:20:17.373706 3665 namespace.go:142] [segment-bridge] updating namespace W0428 11:20:17.776959 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0428 11:20:17.806534 3665 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0428 11:20:18.343925 3665 cni.go:452] [integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 24e7bf4538c479ba5108f0ee0aa04a56e15c2a310ccabacfa84f9e9257fda651 network default NAD default NAD key ] ADD starting CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 24e7bf4538c479ba5108f0ee0aa04a56e15c2a310ccabacfa84f9e9257fda651 network default NAD default NAD key ] I0428 11:20:18.354553 3665 helper_linux.go:556] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-7b666bf6d8-rcm45, hostIfaceName: 24e7bf4538c479b, network: default, NAD default, SandboxID: "24e7bf4538c479ba5108f0ee0aa04a56e15c2a310ccabacfa84f9e9257fda651", PCI device ID: , UID: "db6d2f8e-0b47-49ff-97e2-f0b9d0deace8", MAC: 0a:58:0a:81:00:30, IPs: [10.129.0.48/23] I0428 11:20:18.413971 3665 cni.go:473] [integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 24e7bf4538c479ba5108f0ee0aa04a56e15c2a310ccabacfa84f9e9257fda651 network default NAD default NAD key default] ADD finished CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 24e7bf4538c479ba5108f0ee0aa04a56e15c2a310ccabacfa84f9e9257fda651 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:94:43:6b:3b:5a\",\"name\":\"24e7bf4538c479b\"},{\"mac\":\"0a:58:0a:81:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e14cae31-8e77-4164-a08c-4b70f05222bb\"}],\"ips\":[{\"address\":\"10.129.0.48/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:20:20.240337 3665 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0428 11:20:20.795840 3665 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0428 11:20:20.798948 3665 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service I0428 11:20:21.502020 3665 cni.go:452] [konflux-ui/dex-f5b7985bb-bhs4c 470ccb41df2ce332735a713e5af1065c0b0c6329fd0b163f62fc7c6d82e5410e network default NAD default NAD key ] ADD starting CNI request [konflux-ui/dex-f5b7985bb-bhs4c 470ccb41df2ce332735a713e5af1065c0b0c6329fd0b163f62fc7c6d82e5410e network default NAD default NAD key ] I0428 11:20:21.512464 3665 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: dex-f5b7985bb-bhs4c, hostIfaceName: 470ccb41df2ce33, network: default, NAD default, SandboxID: "470ccb41df2ce332735a713e5af1065c0b0c6329fd0b163f62fc7c6d82e5410e", PCI device ID: , UID: "70091976-d639-4d86-a964-d0d21d402b3d", MAC: 0a:58:0a:81:00:2d, IPs: [10.129.0.45/23] I0428 11:20:21.570255 3665 cni.go:473] [konflux-ui/dex-f5b7985bb-bhs4c 470ccb41df2ce332735a713e5af1065c0b0c6329fd0b163f62fc7c6d82e5410e network default NAD default NAD key default] ADD finished CNI request [konflux-ui/dex-f5b7985bb-bhs4c 470ccb41df2ce332735a713e5af1065c0b0c6329fd0b163f62fc7c6d82e5410e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:23:b9:79:ca:1d\",\"name\":\"470ccb41df2ce33\"},{\"mac\":\"0a:58:0a:81:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e2567ae9-28d6-4964-9082-98598f9a1910\"}],\"ips\":[{\"address\":\"10.129.0.45/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 11:20:22.129026 3665 cni.go:452] [konflux-ui/proxy-579597865b-cntsh 54c914e1158340d09db427aee10accb0ddcbed8091f0673520cfc13064f8acb8 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-579597865b-cntsh 54c914e1158340d09db427aee10accb0ddcbed8091f0673520cfc13064f8acb8 network default NAD default NAD key ] I0428 11:20:22.140242 3665 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-579597865b-cntsh, hostIfaceName: 54c914e1158340d, network: default, NAD default, SandboxID: "54c914e1158340d09db427aee10accb0ddcbed8091f0673520cfc13064f8acb8", PCI device ID: , UID: "86fe40ce-7f46-4cb7-8f79-a16ceec8161d", MAC: 0a:58:0a:81:00:2e, IPs: [10.129.0.46/23] I0428 11:20:22.206439 3665 cni.go:473] [konflux-ui/proxy-579597865b-cntsh 54c914e1158340d09db427aee10accb0ddcbed8091f0673520cfc13064f8acb8 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-579597865b-cntsh 54c914e1158340d09db427aee10accb0ddcbed8091f0673520cfc13064f8acb8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:02:17:19:3e:ac\",\"name\":\"54c914e1158340d\"},{\"mac\":\"0a:58:0a:81:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/98bea695-df90-433a-9610-6988f9558dc5\"}],\"ips\":[{\"address\":\"10.129.0.46/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:20:22.259385 3665 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0428 11:20:22.259960 3665 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service I0428 11:20:23.226093 3665 cni.go:452] [release-service/release-service-controller-manager-6bfdb6bcb6-pldrd 04589ddaef42f90ec53cf4bb50473bb7163910f1946622fa00671fc5d8256f3d network default NAD default NAD key ] ADD starting CNI request [release-service/release-service-controller-manager-6bfdb6bcb6-pldrd 04589ddaef42f90ec53cf4bb50473bb7163910f1946622fa00671fc5d8256f3d network default NAD default NAD key ] I0428 11:20:23.238674 3665 helper_linux.go:556] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-6bfdb6bcb6-pldrd, hostIfaceName: 04589ddaef42f90, network: default, NAD default, SandboxID: "04589ddaef42f90ec53cf4bb50473bb7163910f1946622fa00671fc5d8256f3d", PCI device ID: , UID: "0d575e39-8a8e-40c5-9f6d-a1b93be311e1", MAC: 0a:58:0a:81:00:32, IPs: [10.129.0.50/23] I0428 11:20:23.312415 3665 cni.go:473] [release-service/release-service-controller-manager-6bfdb6bcb6-pldrd 04589ddaef42f90ec53cf4bb50473bb7163910f1946622fa00671fc5d8256f3d network default NAD default NAD key default] ADD finished CNI request [release-service/release-service-controller-manager-6bfdb6bcb6-pldrd 04589ddaef42f90ec53cf4bb50473bb7163910f1946622fa00671fc5d8256f3d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:bb:d0:48:79:d7\",\"name\":\"04589ddaef42f90\"},{\"mac\":\"0a:58:0a:81:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3b16132e-3054-4931-a1c4-c3c269d984b0\"}],\"ips\":[{\"address\":\"10.129.0.50/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:20:23.811883 3665 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0428 11:20:27.509084 3665 cni.go:452] [tsf-tas/tuf-97fc8ff76-5vd7v 409f0ffc382b15c2b2cdc20c5a1ab813fb5dcfa9bd0be3959f48f56e80a5c1e9 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/tuf-97fc8ff76-5vd7v 409f0ffc382b15c2b2cdc20c5a1ab813fb5dcfa9bd0be3959f48f56e80a5c1e9 network default NAD default NAD key ] I0428 11:20:27.518862 3665 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: tuf-97fc8ff76-5vd7v, hostIfaceName: 409f0ffc382b15c, network: default, NAD default, SandboxID: "409f0ffc382b15c2b2cdc20c5a1ab813fb5dcfa9bd0be3959f48f56e80a5c1e9", PCI device ID: , UID: "1e4e44c7-ef7c-453b-b95c-87e4d6e8ae49", MAC: 0a:58:0a:81:00:33, IPs: [10.129.0.51/23] I0428 11:20:27.581905 3665 cni.go:473] [tsf-tas/tuf-97fc8ff76-5vd7v 409f0ffc382b15c2b2cdc20c5a1ab813fb5dcfa9bd0be3959f48f56e80a5c1e9 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/tuf-97fc8ff76-5vd7v 409f0ffc382b15c2b2cdc20c5a1ab813fb5dcfa9bd0be3959f48f56e80a5c1e9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:b2:7b:38:0d:de\",\"name\":\"409f0ffc382b15c\"},{\"mac\":\"0a:58:0a:81:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3a6b77cd-e4be-45c0-9e40-e2bc942a7854\"}],\"ips\":[{\"address\":\"10.129.0.51/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 11:20:27.848216 3665 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0428 11:20:27.848263 3665 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0428 11:20:27.874301 3665 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0428 11:20:27.874436 3665 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0428 11:20:28.303475 3665 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0428 11:20:28.303496 3665 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0428 11:20:28.783946 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0428 11:20:28.809449 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0428 11:20:28.818244 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 W0428 11:20:28.881551 3665 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0428 11:20:31.799927 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0428 11:20:31.802907 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service I0428 11:20:33.982136 3665 cni.go:452] [integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 24e7bf4538c479ba5108f0ee0aa04a56e15c2a310ccabacfa84f9e9257fda651 network default NAD default NAD key ] DEL starting CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 24e7bf4538c479ba5108f0ee0aa04a56e15c2a310ccabacfa84f9e9257fda651 network default NAD default NAD key ] I0428 11:20:34.079897 3665 cni.go:473] [integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 24e7bf4538c479ba5108f0ee0aa04a56e15c2a310ccabacfa84f9e9257fda651 network default NAD default NAD key default] DEL finished CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 24e7bf4538c479ba5108f0ee0aa04a56e15c2a310ccabacfa84f9e9257fda651 network default NAD default NAD key default], result "{}", err I0428 11:20:34.717457 3665 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:20:34.717519 3665 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 I0428 11:20:34.718964 3665 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-7b666bf6d8-rcm45, ips: 10.129.0.48 I0428 11:20:34.718994 3665 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:20:34.887110 3665 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:20:34.887135 3665 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:20:34.889570 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 I0428 11:20:34.889604 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-rcm45 I0428 11:20:36.693329 3665 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 W0428 11:20:38.860353 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service W0428 11:20:38.860592 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0428 11:20:39.870015 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex W0428 11:20:39.902363 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf W0428 11:21:06.368020 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0428 11:21:06.368046 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0428 11:21:06.368062 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0428 11:21:06.368069 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0428 11:21:07.936086 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0428 11:21:07.936107 3665 util.go:959] Failed to build node endpoints for node ip-10-0-2-140.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0428 11:21:32.781483 3665 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0428 11:21:36.301288 3665 cni.go:452] [konflux-ui/proxy-579597865b-cntsh 54c914e1158340d09db427aee10accb0ddcbed8091f0673520cfc13064f8acb8 network default NAD default NAD key ] DEL starting CNI request [konflux-ui/proxy-579597865b-cntsh 54c914e1158340d09db427aee10accb0ddcbed8091f0673520cfc13064f8acb8 network default NAD default NAD key ] I0428 11:21:36.394922 3665 cni.go:473] [konflux-ui/proxy-579597865b-cntsh 54c914e1158340d09db427aee10accb0ddcbed8091f0673520cfc13064f8acb8 network default NAD default NAD key default] DEL finished CNI request [konflux-ui/proxy-579597865b-cntsh 54c914e1158340d09db427aee10accb0ddcbed8091f0673520cfc13064f8acb8 network default NAD default NAD key default], result "{}", err I0428 11:21:36.470062 3665 obj_retry.go:498] Detected object konflux-ui/proxy-579597865b-cntsh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:21:36.470105 3665 pods.go:176] Deleting pod: konflux-ui/proxy-579597865b-cntsh I0428 11:21:36.471545 3665 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-579597865b-cntsh, ips: 10.129.0.46 I0428 11:21:36.471576 3665 obj_retry.go:498] Detected object konflux-ui/proxy-579597865b-cntsh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:21:37.247001 3665 obj_retry.go:492] Detected object konflux-ui/proxy-579597865b-cntsh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:21:37.247032 3665 obj_retry.go:492] Detected object konflux-ui/proxy-579597865b-cntsh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:21:37.253137 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-579597865b-cntsh I0428 11:21:37.253162 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-579597865b-cntsh I0428 11:21:38.780822 3665 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0428 11:21:41.870411 3665 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:21:41.871041 3665 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:21:41.905677 3665 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:21:41.905728 3665 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:21:41.911582 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0428 11:21:41.911612 3665 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0428 11:22:02.935929 3665 namespace.go:98] [default-managed-tenant-tag81] adding namespace I0428 11:22:02.936937 3665 namespace.go:102] [default-managed-tenant-tag81] adding namespace took 984.269µs I0428 11:22:02.974852 3665 namespace.go:142] [default-managed-tenant-tag81] updating namespace I0428 11:22:03.010070 3665 namespace.go:142] [default-managed-tenant-tag81] updating namespace I0428 11:22:03.995233 3665 namespace.go:142] [default-managed-tenant-tag81] updating namespace I0428 11:22:04.126277 3665 namespace.go:142] [default-managed-tenant-tag81] updating namespace I0428 11:22:04.223036 3665 namespace.go:142] [default-managed-tenant-tag81] updating namespace I0428 11:22:57.699511 3665 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=123 I0428 11:23:08.785672 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0428 11:23:49.813656 3665 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0428 11:23:51.796510 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0428 11:23:59.790841 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0428 11:24:34.707853 3665 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=640 I0428 11:24:35.809249 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0428 11:24:44.802582 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0428 11:24:46.694896 3665 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=134 I0428 11:25:02.819155 3665 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7