+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-253.ec2.internal ++ [[ -n ip-10-0-0-253.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-253.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0423 19:49:09.385907636 - disable conntrack on geneve port' I0423 19:49:09.385907636 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0423 19:49:09.403234586 - starting ovnkube-node + echo 'I0423 19:49:09.403234586 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-253.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-253.ec2.internal --init-node ip-10-0-0-253.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-253.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0423 19:49:09.443373 3773 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0423 19:49:09.443428 3773 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-548518d40a.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0423 19:49:09.445240 3773 kube.go:431] Waiting for certificate I0423 19:49:09.445272 3773 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0423 19:49:09.445306 3773 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0423 19:49:09.446035 3773 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0423 19:49:09.484801 3773 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 19:49:09.484819 3773 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 19:49:09.490780 3773 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 19:49:09.490869 3773 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-6fnkn" I0423 19:49:09.504976 3773 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-6fnkn" I0423 19:49:09.505057 3773 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 19:49:10.445723 3773 kube.go:438] Certificate found I0423 19:49:10.446340 3773 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0423 19:49:10.447709 3773 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0423 19:49:10.507662 3773 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-24 19:44:09 +0000 UTC" deadline="2026-04-24 14:53:46.215561537 +0000 UTC" I0423 19:49:10.507693 3773 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h4m35.70787951s" I0423 19:49:10.518610 3773 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0423 19:49:10.533668 3773 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0423 19:49:10.533739 3773 factory.go:561] Starting watch factory I0423 19:49:10.533817 3773 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0423 19:49:10.533842 3773 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.533852 3773 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.533868 3773 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.533874 3773 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.533844 3773 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.533888 3773 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.533900 3773 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.533871 3773 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.533940 3773 server.go:101] MetricServer registers OVS metrics I0423 19:49:10.534104 3773 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.534126 3773 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.534671 3773 server.go:105] MetricServer registers OVN DB metrics I0423 19:49:10.535260 3773 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.535280 3773 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.538173 3773 controller_manager.go:370] Starting the ovnkube controller I0423 19:49:10.538183 3773 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-253.ec2.internal I0423 19:49:10.538223 3773 controller_manager.go:395] NBDB zone sync took: 33.404µs I0423 19:49:10.538231 3773 factory.go:561] Starting watch factory I0423 19:49:10.539155 3773 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0423 19:49:10.539191 3773 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0423 19:49:10.552329 3773 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0423 19:49:10.552359 3773 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0423 19:49:10.554230 3773 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.554269 3773 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.554700 3773 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.554839 3773 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.558796 3773 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0423 19:49:10.558873 3773 server.go:109] MetricServer registers OVN Controller metrics I0423 19:49:10.560391 3773 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.563643 3773 server.go:113] MetricServer registers OVN Northd metrics I0423 19:49:10.563810 3773 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:49:10.568461 3773 metrics.go:519] OVN Metrics Server starts to run ... I0423 19:49:10.639400 3773 factory.go:1977] *v1.Pod informer cache synced successfully I0423 19:49:10.639420 3773 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0423 19:49:10.639424 3773 factory.go:1977] *v1.Namespace informer cache synced successfully I0423 19:49:10.639427 3773 factory.go:1977] *v1.Node informer cache synced successfully I0423 19:49:10.639430 3773 factory.go:1977] *v1.Service informer cache synced successfully I0423 19:49:10.639433 3773 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0423 19:49:10.639510 3773 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:49:10.639523 3773 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:49:10.639521 3773 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:49:10.639541 3773 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:49:10.643549 3773 factory.go:1977] *v1.Namespace informer cache synced successfully I0423 19:49:10.643562 3773 factory.go:1977] *v1.Node informer cache synced successfully I0423 19:49:10.643565 3773 factory.go:1977] *v1.Service informer cache synced successfully I0423 19:49:10.643569 3773 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0423 19:49:10.643572 3773 factory.go:1977] *v1.Pod informer cache synced successfully I0423 19:49:10.643575 3773 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0423 19:49:10.663772 3773 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:49:10.682055 3773 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:49:10.745429 3773 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0423 19:49:10.745446 3773 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0423 19:49:10.745567 3773 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:10.745581 3773 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:10.749819 3773 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0423 19:49:10.749833 3773 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0423 19:49:10.762846 3773 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:10.851485 3773 factory.go:1977] *v1.EgressIP informer cache synced successfully I0423 19:49:10.851598 3773 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:10.851616 3773 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:10.855674 3773 factory.go:1977] *v1.EgressIP informer cache synced successfully I0423 19:49:10.870409 3773 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:10.957111 3773 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0423 19:49:10.957250 3773 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:10.957264 3773 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:10.961325 3773 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0423 19:49:10.976117 3773 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.062933 3773 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0423 19:49:11.063031 3773 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.063044 3773 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.067121 3773 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0423 19:49:11.081028 3773 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.168673 3773 factory.go:1977] *v1.EgressService informer cache synced successfully I0423 19:49:11.168793 3773 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.168807 3773 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.172862 3773 factory.go:1977] *v1.EgressService informer cache synced successfully I0423 19:49:11.191521 3773 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.274300 3773 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0423 19:49:11.274494 3773 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 19:49:11.274508 3773 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 19:49:11.278639 3773 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0423 19:49:11.293851 3773 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 19:49:11.379931 3773 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0423 19:49:11.380006 3773 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.380018 3773 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.380082 3773 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.380100 3773 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.384616 3773 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0423 19:49:11.435736 3773 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.508968 3773 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:49:11.586046 3773 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0423 19:49:11.586063 3773 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0423 19:49:11.586071 3773 factory.go:680] Watch Factory start up complete, took: 1.052334594s I0423 19:49:11.586169 3773 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0423 19:49:11.591653 3773 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0423 19:49:11.591670 3773 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0423 19:49:11.591683 3773 factory.go:680] Watch Factory start up complete, took: 1.053445884s I0423 19:49:11.591693 3773 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-0-253.ec2.internal" zone I0423 19:49:11.619736 3773 default_node_network_controller.go:710] Initializing the default node network controller I0423 19:49:11.672497 3773 udn_isolation.go:101] Starting UDN host isolation manager I0423 19:49:11.679087 3773 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0423 19:49:11.728321 3773 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0423 19:49:11.728410 3773 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0423 19:49:11.728431 3773 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0423 19:49:11.759561 3773 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0423 19:49:11.759635 3773 default_node_network_controller.go:809] Node ip-10-0-0-253.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0423 19:49:11.765246 3773 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.253"] k8s.ovn.org/zone-name:ip-10-0-0-253.ec2.internal] on node ip-10-0-0-253.ec2.internal I0423 19:49:11.784895 3773 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0423 19:49:11.791417 3773 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0423 19:49:11.791699 3773 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0423 19:49:11.792004 3773 gateway_init.go:263] Preparing Gateway I0423 19:49:11.792019 3773 gateway_shared_intf.go:1687] Creating new gateway I0423 19:49:11.841773 3773 controller_manager.go:429] Waiting for node in zone sync took: 250.065147ms I0423 19:49:11.848601 3773 controller_manager.go:611] Switching to transit router for layer2 networks I0423 19:49:11.848621 3773 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-253.ec2.internal I0423 19:49:11.863866 3773 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0423 19:49:11.869923 3773 controller_manager.go:290] SCTP support detected in OVN I0423 19:49:11.870564 3773 services_controller.go:69] Creating services controller for network=default I0423 19:49:11.870693 3773 udn_enabled_svc.go:80] Waiting for service informer to sync I0423 19:49:11.870783 3773 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0423 19:49:11.870816 3773 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0423 19:49:11.870830 3773 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0423 19:49:11.870838 3773 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0423 19:49:11.871005 3773 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0423 19:49:11.871046 3773 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0423 19:49:11.871151 3773 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 111.642µs I0423 19:49:11.871198 3773 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 127.933µs I0423 19:49:11.871208 3773 udn_enabled_svc.go:95] Performing full resync I0423 19:49:11.871211 3773 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 163.012µs I0423 19:49:11.871222 3773 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0423 19:49:11.871231 3773 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0423 19:49:11.871243 3773 default_network_controller.go:325] Starting the default network controller I0423 19:49:11.871313 3773 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 15.068µs I0423 19:49:11.871520 3773 udn_enabled_svc.go:99] Waiting for handler to sync I0423 19:49:11.871531 3773 udn_enabled_svc.go:104] Starting worker I0423 19:49:11.877018 3773 iptables.go:108] Creating table: filter chain: FORWARD I0423 19:49:11.881023 3773 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0423 19:49:11.881086 3773 repair.go:33] Syncing exgw routes took 51.004µs I0423 19:49:11.881109 3773 default_network_controller.go:414] Starting all the Watchers... I0423 19:49:11.881280 3773 namespace.go:98] [open-cluster-management-2ps2c5s4oqe20tlffn1rjju2utl40p4b] adding namespace I0423 19:49:11.881293 3773 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0423 19:49:11.881299 3773 namespace.go:98] [openshift-route-controller-manager] adding namespace I0423 19:49:11.881302 3773 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0423 19:49:11.881300 3773 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0423 19:49:11.881316 3773 namespace.go:98] [openshift-backplane-cse] adding namespace I0423 19:49:11.881323 3773 namespace.go:98] [kube-system] adding namespace I0423 19:49:11.881328 3773 namespace.go:98] [openshift-dns-operator] adding namespace I0423 19:49:11.881336 3773 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0423 19:49:11.881341 3773 namespace.go:98] [openshift-backplane-csm] adding namespace I0423 19:49:11.881345 3773 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0423 19:49:11.881348 3773 namespace.go:98] [openshift-service-ca-operator] adding namespace I0423 19:49:11.881354 3773 namespace.go:98] [kube-node-lease] adding namespace I0423 19:49:11.881358 3773 namespace.go:98] [openshift-config-operator] adding namespace I0423 19:49:11.881375 3773 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0423 19:49:11.881852 3773 namespace.go:102] [open-cluster-management-2ps2c5s4oqe20tlffn1rjju2utl40p4b] adding namespace took 559.508µs I0423 19:49:11.881868 3773 namespace.go:98] [openshift-dns] adding namespace I0423 19:49:11.882469 3773 namespace.go:102] [openshift-dns] adding namespace took 594.019µs I0423 19:49:11.882486 3773 namespace.go:98] [openshift-image-registry] adding namespace I0423 19:49:11.883106 3773 namespace.go:102] [openshift-image-registry] adding namespace took 613.779µs I0423 19:49:11.883120 3773 namespace.go:98] [openshift-backplane-tam] adding namespace I0423 19:49:11.883549 3773 iptables.go:108] Creating table: filter chain: OUTPUT I0423 19:49:11.883696 3773 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 2.39252ms I0423 19:49:11.883712 3773 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0423 19:49:11.884152 3773 namespace.go:102] [openshift-route-controller-manager] adding namespace took 2.84296ms I0423 19:49:11.884161 3773 namespace.go:98] [openshift-backplane-mobb] adding namespace I0423 19:49:11.884682 3773 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 3.368407ms I0423 19:49:11.884699 3773 namespace.go:98] [openshift-backplane-cee] adding namespace I0423 19:49:11.885348 3773 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 4.031118ms I0423 19:49:11.885363 3773 namespace.go:98] [default] adding namespace I0423 19:49:11.885860 3773 namespace.go:102] [openshift-backplane-cse] adding namespace took 4.531425ms I0423 19:49:11.885882 3773 namespace.go:98] [openshift-operators] adding namespace I0423 19:49:11.886362 3773 namespace.go:102] [kube-system] adding namespace took 5.029088ms I0423 19:49:11.886380 3773 namespace.go:98] [openshift-cluster-version] adding namespace I0423 19:49:11.886898 3773 namespace.go:102] [openshift-dns-operator] adding namespace took 5.562478ms I0423 19:49:11.886914 3773 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0423 19:49:11.887476 3773 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.134059ms I0423 19:49:11.887501 3773 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0423 19:49:11.887973 3773 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.622891ms I0423 19:49:11.887992 3773 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0423 19:49:11.888487 3773 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.137777ms I0423 19:49:11.888500 3773 namespace.go:98] [openshift-authentication-operator] adding namespace I0423 19:49:11.888917 3773 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.565117ms I0423 19:49:11.888928 3773 namespace.go:98] [openshift-multus] adding namespace I0423 19:49:11.889573 3773 namespace.go:102] [kube-node-lease] adding namespace took 8.215333ms I0423 19:49:11.889582 3773 namespace.go:98] [openshift-backplane] adding namespace I0423 19:49:11.890027 3773 namespace.go:102] [openshift-config-operator] adding namespace took 8.665398ms I0423 19:49:11.890035 3773 namespace.go:98] [openshift-network-operator] adding namespace I0423 19:49:11.890546 3773 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 9.159382ms I0423 19:49:11.890560 3773 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0423 19:49:11.891019 3773 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.894173ms I0423 19:49:11.891029 3773 namespace.go:98] [openshift-kube-apiserver] adding namespace I0423 19:49:11.891491 3773 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 7.774448ms I0423 19:49:11.891504 3773 namespace.go:98] [openshift-must-gather-operator] adding namespace I0423 19:49:11.891896 3773 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.729717ms I0423 19:49:11.891909 3773 namespace.go:98] [openshift-apiserver] adding namespace I0423 19:49:11.892277 3773 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.572866ms I0423 19:49:11.892287 3773 namespace.go:98] [openshift-machine-api] adding namespace I0423 19:49:11.892574 3773 gateway_shared_intf.go:1816] Gateway Creation Complete I0423 19:49:11.892587 3773 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-253.ec2.internal","mac-address":"12:0b:ce:de:90:65","ip-addresses":["10.0.0.253/24"],"ip-address":"10.0.0.253/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:97eff3ac-215d-45a8-acf3-183427241612 k8s.ovn.org/node-encap-ips:["10.0.0.253"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.253/24"} k8s.ovn.org/zone-name:ip-10-0-0-253.ec2.internal] on node ip-10-0-0-253.ec2.internal I0423 19:49:11.892718 3773 namespace.go:102] [default] adding namespace took 7.350211ms I0423 19:49:11.892731 3773 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0423 19:49:11.893256 3773 namespace.go:102] [openshift-operators] adding namespace took 7.36577ms I0423 19:49:11.893267 3773 namespace.go:98] [openshift-console-operator] adding namespace I0423 19:49:11.893790 3773 namespace.go:102] [openshift-cluster-version] adding namespace took 7.406916ms I0423 19:49:11.893797 3773 namespace.go:98] [openshift-insights] adding namespace I0423 19:49:11.894270 3773 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 7.352286ms I0423 19:49:11.894283 3773 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0423 19:49:11.894682 3773 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.175387ms I0423 19:49:11.894709 3773 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0423 19:49:11.895126 3773 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.126842ms I0423 19:49:11.895170 3773 namespace.go:98] [openshift-ovn-kubernetes] adding namespace W0423 19:49:11.895534 3773 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-tn7rh: pod openshift-multus/network-metrics-daemon-tn7rh: no pod IPs found W0423 19:49:11.895546 3773 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-v22pt: pod openshift-multus/network-metrics-daemon-v22pt: no pod IPs found I0423 19:49:11.895582 3773 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.077788ms I0423 19:49:11.895592 3773 namespace.go:98] [openshift-kube-scheduler] adding namespace I0423 19:49:11.895994 3773 namespace.go:102] [openshift-multus] adding namespace took 7.060688ms I0423 19:49:11.896005 3773 namespace.go:98] [openshift-authentication] adding namespace I0423 19:49:11.896452 3773 namespace.go:102] [openshift-backplane] adding namespace took 6.864083ms I0423 19:49:11.896464 3773 namespace.go:98] [openshift-infra] adding namespace I0423 19:49:11.897187 3773 namespace.go:102] [openshift-network-operator] adding namespace took 7.146262ms I0423 19:49:11.897203 3773 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0423 19:49:11.897690 3773 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 7.125531ms I0423 19:49:11.897703 3773 namespace.go:98] [openshift-package-operator] adding namespace I0423 19:49:11.898097 3773 namespace.go:102] [openshift-kube-apiserver] adding namespace took 7.063377ms I0423 19:49:11.898469 3773 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.957328ms I0423 19:49:11.898486 3773 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0423 19:49:11.898914 3773 namespace.go:102] [openshift-apiserver] adding namespace took 6.999284ms I0423 19:49:11.898928 3773 namespace.go:98] [openshift-logging] adding namespace I0423 19:49:11.899324 3773 namespace.go:102] [openshift-machine-api] adding namespace took 7.031279ms I0423 19:49:11.899336 3773 namespace.go:98] [openshift-network-node-identity] adding namespace I0423 19:49:11.899696 3773 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.960551ms I0423 19:49:11.899707 3773 namespace.go:98] [openshift] adding namespace I0423 19:49:11.900088 3773 namespace.go:102] [openshift-console-operator] adding namespace took 6.81559ms I0423 19:49:11.900100 3773 namespace.go:98] [openshift-apiserver-operator] adding namespace I0423 19:49:11.900463 3773 namespace.go:102] [openshift-insights] adding namespace took 6.66163ms I0423 19:49:11.900475 3773 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0423 19:49:11.900843 3773 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.555873ms I0423 19:49:11.900855 3773 namespace.go:98] [openshift-config] adding namespace I0423 19:49:11.901233 3773 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.51668ms I0423 19:49:11.901244 3773 namespace.go:98] [openshift-operators-redhat] adding namespace I0423 19:49:11.901597 3773 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.421491ms I0423 19:49:11.901609 3773 namespace.go:98] [openshift-network-diagnostics] adding namespace I0423 19:49:11.901952 3773 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.354522ms I0423 19:49:11.901963 3773 namespace.go:98] [openshift-machine-config-operator] adding namespace I0423 19:49:11.902419 3773 namespace.go:102] [openshift-authentication] adding namespace took 6.409058ms I0423 19:49:11.902430 3773 namespace.go:98] [openshift-network-console] adding namespace I0423 19:49:11.902780 3773 namespace.go:102] [openshift-infra] adding namespace took 6.311362ms I0423 19:49:11.902791 3773 namespace.go:98] [openshift-console] adding namespace I0423 19:49:11.903155 3773 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 5.946683ms I0423 19:49:11.903166 3773 namespace.go:98] [openshift-console-user-settings] adding namespace I0423 19:49:11.903533 3773 namespace.go:102] [openshift-package-operator] adding namespace took 5.825636ms I0423 19:49:11.903544 3773 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0423 19:49:11.903912 3773 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 5.421227ms I0423 19:49:11.903924 3773 namespace.go:98] [openshift-customer-monitoring] adding namespace I0423 19:49:11.904302 3773 namespace.go:102] [openshift-logging] adding namespace took 5.368897ms I0423 19:49:11.904315 3773 namespace.go:98] [openshift-backplane-srep] adding namespace I0423 19:49:11.904649 3773 namespace.go:102] [openshift-network-node-identity] adding namespace took 5.308417ms I0423 19:49:11.904659 3773 namespace.go:98] [openshift-monitoring] adding namespace I0423 19:49:11.905023 3773 namespace.go:102] [openshift] adding namespace took 5.311403ms I0423 19:49:11.905036 3773 namespace.go:98] [openshift-ingress-operator] adding namespace I0423 19:49:11.905408 3773 namespace.go:102] [openshift-apiserver-operator] adding namespace took 5.303164ms I0423 19:49:11.905420 3773 namespace.go:98] [openshift-node] adding namespace I0423 19:49:11.905773 3773 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 5.294047ms I0423 19:49:11.905784 3773 namespace.go:98] [openshift-etcd] adding namespace I0423 19:49:11.906164 3773 namespace.go:102] [openshift-config] adding namespace took 5.303706ms I0423 19:49:11.906179 3773 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace W0423 19:49:11.906762 3773 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-v6tps: pod openshift-network-diagnostics/network-check-target-v6tps: no pod IPs found W0423 19:49:11.906775 3773 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-f4w6t: pod openshift-network-diagnostics/network-check-target-f4w6t: no pod IPs found I0423 19:49:11.906802 3773 namespace.go:102] [openshift-operators-redhat] adding namespace took 5.552471ms I0423 19:49:11.906815 3773 namespace.go:98] [openshift-marketplace] adding namespace I0423 19:49:11.907461 3773 namespace.go:102] [openshift-network-diagnostics] adding namespace took 5.846727ms I0423 19:49:11.907484 3773 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0423 19:49:11.907856 3773 namespace.go:102] [openshift-machine-config-operator] adding namespace took 5.889332ms I0423 19:49:11.907866 3773 namespace.go:98] [openshift-ingress-canary] adding namespace I0423 19:49:11.908367 3773 namespace.go:102] [openshift-network-console] adding namespace took 5.929477ms I0423 19:49:11.908383 3773 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0423 19:49:11.908812 3773 namespace.go:102] [openshift-console] adding namespace took 6.015209ms I0423 19:49:11.908823 3773 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0423 19:49:11.909336 3773 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.165907ms I0423 19:49:11.909346 3773 namespace.go:98] [openshift-ingress] adding namespace I0423 19:49:11.909729 3773 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.178265ms I0423 19:49:11.909746 3773 namespace.go:98] [kube-public] adding namespace I0423 19:49:11.910124 3773 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.193009ms I0423 19:49:11.910319 3773 default_node_network_controller.go:519] Node connection status = connected I0423 19:49:11.910676 3773 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.356043ms I0423 19:49:11.910689 3773 namespace.go:98] [openshift-host-network] adding namespace I0423 19:49:11.911103 3773 namespace.go:102] [openshift-monitoring] adding namespace took 6.438786ms I0423 19:49:11.911117 3773 namespace.go:98] [openshift-config-managed] adding namespace I0423 19:49:11.911533 3773 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.491907ms I0423 19:49:11.911920 3773 namespace.go:102] [openshift-node] adding namespace took 6.495017ms I0423 19:49:11.912327 3773 namespace.go:102] [openshift-etcd] adding namespace took 6.538475ms I0423 19:49:11.912714 3773 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.530537ms I0423 19:49:11.913062 3773 namespace.go:102] [openshift-marketplace] adding namespace took 6.242115ms I0423 19:49:11.913075 3773 namespace.go:98] [openshift-controller-manager] adding namespace I0423 19:49:11.913429 3773 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 5.94098ms I0423 19:49:11.913799 3773 namespace.go:102] [openshift-ingress-canary] adding namespace took 5.928199ms I0423 19:49:11.914156 3773 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 5.768018ms I0423 19:49:11.914172 3773 namespace.go:98] [dedicated-admin] adding namespace I0423 19:49:11.914603 3773 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 5.774743ms I0423 19:49:11.914964 3773 namespace.go:102] [openshift-ingress] adding namespace took 5.613027ms I0423 19:49:11.915308 3773 namespace.go:102] [kube-public] adding namespace took 5.557131ms I0423 19:49:11.915746 3773 namespace.go:102] [openshift-host-network] adding namespace took 5.051665ms I0423 19:49:11.916151 3773 namespace.go:102] [openshift-config-managed] adding namespace took 5.025868ms I0423 19:49:11.916964 3773 namespace.go:102] [openshift-controller-manager] adding namespace took 3.883064ms I0423 19:49:11.917380 3773 namespace.go:102] [dedicated-admin] adding namespace took 3.202892ms W0423 19:49:11.917466 3773 master.go:471] Unable to parse remote node ip-10-0-1-57.ec2.internal chassis-id annotation. Chassis may be removed during sync I0423 19:49:11.917574 3773 master.go:548] Adding or Updating local node "ip-10-0-0-253.ec2.internal" for network "default" I0423 19:49:11.917621 3773 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-57.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-57.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-57.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-57.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-57.ec2.internal W0423 19:49:11.924276 3773 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-f4w6t: pod openshift-network-diagnostics/network-check-target-f4w6t: no pod IPs found W0423 19:49:11.924300 3773 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-tn7rh: pod openshift-multus/network-metrics-daemon-tn7rh: no pod IPs found I0423 19:49:11.925206 3773 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-253.ec2.internal for the network default I0423 19:49:11.925525 3773 zone_ic_handler.go:195] Time taken to create transit switch: 308.085µs I0423 19:49:11.926240 3773 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0423 19:49:11.926261 3773 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0423 19:49:11.926274 3773 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0423 19:49:11.926297 3773 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-253.ec2.internal in network "default" I0423 19:49:11.926341 3773 node_tracker.go:169] Node ip-10-0-0-253.ec2.internal switch + router changed, syncing services in network "default" I0423 19:49:11.926356 3773 services_controller.go:574] Full service sync requested for network=default I0423 19:49:11.926363 3773 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-57.ec2.internal in network "default" I0423 19:49:11.926371 3773 node_tracker.go:235] Node ip-10-0-1-57.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-57.ec2.internal" I0423 19:49:11.926377 3773 node_tracker.go:169] Node ip-10-0-1-57.ec2.internal switch + router changed, syncing services in network "default" I0423 19:49:11.926381 3773 services_controller.go:574] Full service sync requested for network=default I0423 19:49:12.026879 3773 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0423 19:49:12.026897 3773 services_controller.go:208] Setting up event handlers for services for network=default I0423 19:49:12.027098 3773 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0423 19:49:12.027160 3773 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0423 19:49:12.027173 3773 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0423 19:49:12.127987 3773 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0423 19:49:12.128005 3773 repair.go:60] Starting repairing loop for services I0423 19:49:12.128085 3773 repair.go:131] Deleted 0 stale service LBs I0423 19:49:12.128102 3773 repair.go:137] Deleted 0 stale Chassis Template Vars I0423 19:49:12.128120 3773 repair.go:62] Finished repairing loop for services: 116.45µs I0423 19:49:12.128160 3773 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0423 19:49:12.128169 3773 services_controller.go:254] Starting workers for network=default I0423 19:49:12.128490 3773 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-f4w6t] creating logical port openshift-network-diagnostics_network-check-target-f4w6t for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:12.128502 3773 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-v6tps, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-v6tps: suppressed error logged: pod openshift-network-diagnostics/network-check-target-v6tps: no pod IPs found I0423 19:49:12.128541 3773 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-v22pt, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-v22pt: suppressed error logged: pod openshift-multus/network-metrics-daemon-v22pt: no pod IPs found I0423 19:49:12.128562 3773 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-tn7rh] creating logical port openshift-multus_network-metrics-daemon-tn7rh for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:12.128654 3773 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-tn7rh I0423 19:49:12.128653 3773 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-f4w6t I0423 19:49:12.146797 3773 pod.go:62] [openshift-multus/network-metrics-daemon-tn7rh] pod update took 18.171075ms I0423 19:49:12.146817 3773 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-tn7rh] addLogicalPort annotation time took 18.192736ms I0423 19:49:12.147659 3773 pods.go:271] [openshift-multus/network-metrics-daemon-tn7rh] addLogicalPort took 19.107974ms, libovsdb time 659.614µs I0423 19:49:12.148025 3773 pod.go:62] [openshift-network-diagnostics/network-check-target-f4w6t] pod update took 19.456999ms I0423 19:49:12.148046 3773 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-f4w6t] addLogicalPort annotation time took 19.478539ms I0423 19:49:12.148949 3773 pods.go:271] [openshift-network-diagnostics/network-check-target-f4w6t] addLogicalPort took 20.470434ms, libovsdb time 631.204µs I0423 19:49:12.149418 3773 admin_network_policy_controller.go:227] Starting controller default-network-controller I0423 19:49:12.149436 3773 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0423 19:49:12.149446 3773 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0423 19:49:12.149451 3773 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0423 19:49:12.149626 3773 repair.go:29] Repairing admin network policies took 170.106µs I0423 19:49:12.149720 3773 repair.go:92] Repairing baseline admin network policies took 86.531µs I0423 19:49:12.150261 3773 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0423 19:49:12.150277 3773 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0423 19:49:12.151669 3773 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0423 19:49:12.151690 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0423 19:49:12.151700 3773 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0423 19:49:12.153176 3773 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0423 19:49:12.153199 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0423 19:49:12.153209 3773 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0423 19:49:12.154891 3773 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0423 19:49:12.154913 3773 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0423 19:49:12.154922 3773 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0423 19:49:12.156498 3773 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0423 19:49:12.156520 3773 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0423 19:49:12.156530 3773 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0423 19:49:12.158429 3773 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0423 19:49:12.158448 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0423 19:49:12.158456 3773 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0423 19:49:12.159083 3773 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0423 19:49:12.159106 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0423 19:49:12.159115 3773 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0423 19:49:12.161077 3773 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0423 19:49:12.161096 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0423 19:49:12.161103 3773 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0423 19:49:12.161634 3773 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0423 19:49:12.161652 3773 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0423 19:49:12.161662 3773 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0423 19:49:12.162190 3773 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0423 19:49:12.162205 3773 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0423 19:49:12.162211 3773 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0423 19:49:12.162496 3773 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0423 19:49:12.162516 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0423 19:49:12.162525 3773 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0423 19:49:12.162980 3773 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0423 19:49:12.162995 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0423 19:49:12.163360 3773 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0423 19:49:12.163372 3773 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0423 19:49:12.164152 3773 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0423 19:49:12.164178 3773 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0423 19:49:12.164186 3773 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0423 19:49:12.165296 3773 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0423 19:49:12.165313 3773 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0423 19:49:12.165323 3773 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0423 19:49:12.166365 3773 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0423 19:49:12.166381 3773 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0423 19:49:12.166390 3773 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0423 19:49:12.168122 3773 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0423 19:49:12.168155 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0423 19:49:12.168163 3773 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0423 19:49:12.168660 3773 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0423 19:49:12.168679 3773 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0423 19:49:12.168689 3773 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0423 19:49:12.169882 3773 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0423 19:49:12.169900 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0423 19:49:12.169908 3773 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0423 19:49:12.170592 3773 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0423 19:49:12.170610 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0423 19:49:12.170618 3773 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0423 19:49:12.171152 3773 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0423 19:49:12.171170 3773 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0423 19:49:12.171189 3773 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0423 19:49:12.172059 3773 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0423 19:49:12.172079 3773 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0423 19:49:12.172088 3773 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0423 19:49:12.173061 3773 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0423 19:49:12.173079 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0423 19:49:12.173085 3773 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0423 19:49:12.173984 3773 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0423 19:49:12.174002 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0423 19:49:12.174023 3773 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0423 19:49:12.174678 3773 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0423 19:49:12.174696 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0423 19:49:12.174704 3773 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0423 19:49:12.175225 3773 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0423 19:49:12.175243 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0423 19:49:12.175249 3773 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0423 19:49:12.175520 3773 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0423 19:49:12.175536 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0423 19:49:12.175542 3773 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0423 19:49:12.175782 3773 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0423 19:49:12.175795 3773 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0423 19:49:12.175800 3773 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0423 19:49:12.177130 3773 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0423 19:49:12.177165 3773 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0423 19:49:12.177175 3773 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0423 19:49:12.178161 3773 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0423 19:49:12.178178 3773 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0423 19:49:12.178190 3773 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0423 19:49:12.179440 3773 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0423 19:49:12.183772 3773 egressfirewall.go:374] Starting EgressFirewall controller I0423 19:49:12.183798 3773 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0423 19:49:12.183798 3773 controller.go:133] Adding controller egress-firewall-controller event handlers I0423 19:49:12.183820 3773 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0423 19:49:12.183820 3773 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0423 19:49:12.183833 3773 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0423 19:49:12.183836 3773 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0423 19:49:12.184080 3773 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0423 19:49:12.184093 3773 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0423 19:49:12.184099 3773 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0423 19:49:12.184106 3773 egressqos.go:193] Setting up event handlers for EgressQoS I0423 19:49:12.184114 3773 egressfirewall.go:723] Syncing node "ip-10-0-0-253.ec2.internal" for egress firewall I0423 19:49:12.184123 3773 egressfirewall.go:723] Syncing node "ip-10-0-1-57.ec2.internal" for egress firewall I0423 19:49:12.184239 3773 egressqos.go:245] Starting EgressQoS Controller I0423 19:49:12.184249 3773 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0423 19:49:12.184255 3773 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0423 19:49:12.184260 3773 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0423 19:49:12.184264 3773 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0423 19:49:12.184270 3773 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0423 19:49:12.184274 3773 shared_informer.go:356] "Caches are synced" controller="egressqos" I0423 19:49:12.184278 3773 egressqos.go:259] Repairing EgressQoSes I0423 19:49:12.184282 3773 egressqos.go:399] Starting repairing loop for egressqos I0423 19:49:12.184354 3773 egressqos.go:401] Finished repairing loop for egressqos: 72.233µs I0423 19:49:12.184369 3773 egressservice_zone.go:132] Setting up event handlers for Egress Services I0423 19:49:12.184399 3773 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-253.ec2.internal I0423 19:49:12.184445 3773 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-253.ec2.internal : 47.851µs I0423 19:49:12.184536 3773 egressservice_zone.go:210] Starting Egress Services Controller I0423 19:49:12.184551 3773 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0423 19:49:12.184558 3773 shared_informer.go:356] "Caches are synced" controller="egressservices" I0423 19:49:12.184564 3773 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0423 19:49:12.184569 3773 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0423 19:49:12.184574 3773 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0423 19:49:12.184579 3773 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0423 19:49:12.184584 3773 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0423 19:49:12.184589 3773 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0423 19:49:12.184592 3773 egressservice_zone.go:228] Repairing Egress Services I0423 19:49:12.184863 3773 master_controller.go:88] Starting Admin Policy Based Route Controller I0423 19:49:12.184873 3773 external_controller.go:278] Starting Admin Policy Based Route Controller I0423 19:49:12.184884 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-253.ec2.internal I0423 19:49:12.184896 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-253.ec2.internal: 12.779µs I0423 19:49:12.184902 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-57.ec2.internal I0423 19:49:12.184905 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-57.ec2.internal: 3.826µs I0423 19:49:12.184978 3773 default_network_controller.go:585] Completing all the Watchers took 303.860418ms I0423 19:49:12.184990 3773 default_network_controller.go:589] Starting unidling controllers I0423 19:49:12.185000 3773 unidle.go:46] Registering OVN SB ControllerEvent handler I0423 19:49:12.185012 3773 unidle.go:63] Populating Initial ContollerEvent events I0423 19:49:12.185023 3773 unidle.go:79] Setting up event handlers for services I0423 19:49:12.185069 3773 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0423 19:49:12.411407 3773 default_node_network_controller.go:519] Node connection status = connected I0423 19:49:12.911309 3773 default_node_network_controller.go:519] Node connection status = connected I0423 19:49:13.110599 3773 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0423 19:49:13.235289 3773 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-42.ec2.internal in network "default" I0423 19:49:13.235314 3773 node_tracker.go:220] Node ip-10-0-2-42.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0423 19:49:13.235332 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-42.ec2.internal I0423 19:49:13.235350 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-42.ec2.internal: 24.229µs I0423 19:49:13.235338 3773 egressfirewall.go:723] Syncing node "ip-10-0-2-42.ec2.internal" for egress firewall E0423 19:49:13.235372 3773 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-2-42.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0423 19:49:13.235394 3773 obj_retry.go:578] Failed to create *v1.Node ip-10-0-2-42.ec2.internal, error: [adding or updating remote node chassis ip-10-0-2-42.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-42.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-42.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-42.ec2.internal, error parsing annotation for node ip-10-0-2-42.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0423 19:49:13.247153 3773 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-2-42.ec2.internal annotation: unexpected end of JSON input I0423 19:49:13.247250 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0423 19:49:13.247283 3773 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-2-42.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0423 19:49:13.264809 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-42.ec2.internal I0423 19:49:13.264827 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-42.ec2.internal: 22.943µs I0423 19:49:13.264813 3773 egressfirewall.go:723] Syncing node "ip-10-0-2-42.ec2.internal" for egress firewall E0423 19:49:13.264860 3773 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-2-42.ec2.internal annotation: unexpected end of JSON input I0423 19:49:13.264906 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0423 19:49:13.264935 3773 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-2-42.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0423 19:49:13.282377 3773 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-2-42.ec2.internal annotation: unexpected end of JSON input E0423 19:49:13.282414 3773 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-2-42.ec2.internal annotation: unexpected end of JSON input I0423 19:49:13.282436 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0423 19:49:13.282484 3773 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-2-42.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-2-42.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-2-42.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-2-42.ec2.internal" I0423 19:49:13.294190 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0423 19:49:13.411510 3773 default_node_network_controller.go:519] Node connection status = connected I0423 19:49:13.429092 3773 bridgeconfig.go:537] Gateway is ready I0423 19:49:13.429112 3773 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0423 19:49:13.440544 3773 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.253. Updating OVN encap IP. I0423 19:49:13.445919 3773 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.253 - it is already configured I0423 19:49:13.445931 3773 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.253/24:{} 172.20.0.1/32:{}]. Updating annotations. I0423 19:49:13.446170 3773 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.253/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-253.ec2.internal","mac-address":"12:0b:ce:de:90:65","ip-addresses":["10.0.0.253/24"],"ip-address":"10.0.0.253/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:97eff3ac-215d-45a8-acf3-183427241612 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.253/24"}] on node ip-10-0-0-253.ec2.internal I0423 19:49:13.465530 3773 egressfirewall.go:723] Syncing node "ip-10-0-0-253.ec2.internal" for egress firewall I0423 19:49:13.465577 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-253.ec2.internal I0423 19:49:13.465588 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-253.ec2.internal: 13.997µs I0423 19:49:13.465592 3773 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-253.ec2.internal in network "default" I0423 19:49:13.465626 3773 master.go:548] Adding or Updating local node "ip-10-0-0-253.ec2.internal" for network "default" I0423 19:49:13.465674 3773 node_tracker.go:169] Node ip-10-0-0-253.ec2.internal switch + router changed, syncing services in network "default" I0423 19:49:13.465693 3773 services_controller.go:574] Full service sync requested for network=default I0423 19:49:13.466796 3773 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.0.253 I0423 19:49:13.466895 3773 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-253.ec2.internal I0423 19:49:13.470512 3773 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-253.ec2.internal. Updating no re-route policies I0423 19:49:13.470580 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-9dnwc I0423 19:49:13.470584 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-f4w6t I0423 19:49:13.470596 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-9dnwc I0423 19:49:13.470602 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-f4w6t I0423 19:49:13.470599 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-d7q5x I0423 19:49:13.470615 3773 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-9dnwc after 0 failed attempt(s) I0423 19:49:13.470619 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-d7q5x I0423 19:49:13.470619 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-twb5p I0423 19:49:13.470628 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-6j7ns I0423 19:49:13.470633 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-6j7ns I0423 19:49:13.470640 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-twb5p I0423 19:49:13.470645 3773 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-f4w6t] creating logical port openshift-network-diagnostics_network-check-target-f4w6t for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:13.470644 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-253.ec2.internal I0423 19:49:13.470661 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-253.ec2.internal I0423 19:49:13.470661 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-j29sk I0423 19:49:13.470667 3773 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-253.ec2.internal I0423 19:49:13.470671 3773 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-twb5p after 0 failed attempt(s) I0423 19:49:13.470672 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-tn7rh I0423 19:49:13.470674 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-ttg8h I0423 19:49:13.470683 3773 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-253.ec2.internal after 0 failed attempt(s) I0423 19:49:13.470685 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-n7b97 I0423 19:49:13.470676 3773 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-253.ec2.internal I0423 19:49:13.470698 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-tn7rh I0423 19:49:13.470702 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-v22pt I0423 19:49:13.470709 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-n7b97 I0423 19:49:13.470716 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-v22pt I0423 19:49:13.470737 3773 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-n7b97 after 0 failed attempt(s) I0423 19:49:13.470647 3773 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-6j7ns after 0 failed attempt(s) I0423 19:49:13.470755 3773 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-v22pt, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-v22pt: suppressed error logged: pod openshift-multus/network-metrics-daemon-v22pt: no pod IPs found I0423 19:49:13.470690 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-ttg8h I0423 19:49:13.470785 3773 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-253.ec2.internal after 0 failed attempt(s) I0423 19:49:13.470650 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-99rvg I0423 19:49:13.470805 3773 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-ttg8h after 0 failed attempt(s) I0423 19:49:13.470810 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-99rvg I0423 19:49:13.470674 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-j29sk I0423 19:49:13.470815 3773 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-tn7rh] creating logical port openshift-multus_network-metrics-daemon-tn7rh for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:13.470831 3773 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-99rvg after 0 failed attempt(s) I0423 19:49:13.470671 3773 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-42lfq I0423 19:49:13.470851 3773 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-42lfq I0423 19:49:13.470870 3773 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-42lfq after 0 failed attempt(s) I0423 19:49:13.470639 3773 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-d7q5x after 0 failed attempt(s) I0423 19:49:13.470920 3773 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-j29sk after 0 failed attempt(s) I0423 19:49:13.471430 3773 pods.go:271] [openshift-network-diagnostics/network-check-target-f4w6t] addLogicalPort took 796.985µs, libovsdb time 237.89µs I0423 19:49:13.471447 3773 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-f4w6t after 0 failed attempt(s) I0423 19:49:13.471598 3773 pods.go:271] [openshift-multus/network-metrics-daemon-tn7rh] addLogicalPort took 796.946µs, libovsdb time 294.58µs I0423 19:49:13.471610 3773 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-tn7rh after 0 failed attempt(s) I0423 19:49:13.472120 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-42.ec2.internal I0423 19:49:13.472153 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-42.ec2.internal: 36.283µs I0423 19:49:13.472172 3773 egressfirewall.go:723] Syncing node "ip-10-0-2-42.ec2.internal" for egress firewall I0423 19:49:13.472194 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:49:13.486733 3773 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0423 19:49:13.487459 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:49:13.500082 3773 iptables.go:108] Creating table: mangle chain: OUTPUT I0423 19:49:13.503299 3773 iptables.go:108] Creating table: nat chain: OUTPUT I0423 19:49:13.526677 3773 iptables.go:108] Creating table: nat chain: PREROUTING I0423 19:49:13.574102 3773 iptables.go:108] Creating table: filter chain: FORWARD I0423 19:49:13.591953 3773 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0423 19:49:13.592113 3773 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0423 19:49:13.592184 3773 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0423 19:49:13.592307 3773 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0423 19:49:13.592371 3773 gateway_init.go:325] Gateway and management port readiness took 1.685878982s I0423 19:49:13.592405 3773 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0423 19:49:13.592445 3773 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0423 19:49:13.592460 3773 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0423 19:49:13.592474 3773 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0423 19:49:13.592537 3773 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0423 19:49:13.592640 3773 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 72.796µs I0423 19:49:13.592889 3773 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0423 19:49:13.592920 3773 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 19.838µs I0423 19:49:13.592932 3773 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 41.802µs I0423 19:49:13.592940 3773 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0423 19:49:13.592953 3773 nad_controller.go:379] [node-nad-controller NAD controller]: started I0423 19:49:13.592961 3773 default_node_network_controller.go:899] Starting the default node network controller I0423 19:49:13.592967 3773 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0423 19:49:13.593043 3773 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.639µs I0423 19:49:13.593202 3773 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0423 19:49:13.593222 3773 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0423 19:49:13.593262 3773 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a95040 0xc0002dda40 0xc0002ddab0 0xc0002ddb20 0xc0002ddb90 0xc0002ddc70 0xc0002ddce0 0xc000aa53b0 0xc0002ddd50 0xc0002dddc0 0xc0001363c0 0xc0065d33b0 0xc0002dde30 map[0x2ff9080:0xc002d05200 0x2ff95c0:0xc002d05380 0x2ffa2e0:0xc002d05140 0x2ffa580:0xc002d051a0 0x2ffad60:0xc002d05260 0x2ffb000:0xc002d052c0 0x2ffb2a0:0xc002d05320 0x2ffba80:0xc002d053e0 0x2ffbd20:0xc002d05440 0x3026980:0xc001606c60 0x3026d20:0xc001606e40 0x3027800:0xc000af9500 0x3029160:0xc001c42000 0x30298a0:0xc001606ea0 0x302a720:0xc001606b40] 0xc000ae21c0 0} I0423 19:49:13.593322 3773 gateway.go:151] Starting gateway service sync I0423 19:49:13.593786 3773 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0423 19:49:13.595981 3773 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0423 19:49:13.605465 3773 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0423 19:49:13.606636 3773 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0423 19:49:13.607742 3773 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0423 19:49:13.641913 3773 gateway.go:168] Gateway service sync done. Time taken: 48.578802ms I0423 19:49:13.642642 3773 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30794 I0423 19:49:13.642795 3773 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a95040 0xc0002dda40 0xc0002ddab0 0xc0002ddb20 0xc0002ddb90 0xc0002ddc70 0xc0002ddce0 0xc000aa53b0 0xc0002ddd50 0xc0002dddc0 0xc0001363c0 0xc0065d33b0 0xc0002dde30 map[0x2ff9080:0xc002d05200 0x2ff95c0:0xc002d05380 0x2ffa2e0:0xc002d05140 0x2ffa580:0xc002d051a0 0x2ffad60:0xc002d05260 0x2ffb000:0xc002d052c0 0x2ffb2a0:0xc002d05320 0x2ffba80:0xc002d053e0 0x2ffbd20:0xc002d05440 0x3026980:0xc001606c60 0x3026d20:0xc001606e40 0x3027800:0xc000af9500 0x3029160:0xc001c42000 0x30298a0:0xc001606ea0 0x302a720:0xc001606b40] 0xc000ae21c0 0} I0423 19:49:13.643319 3773 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a95040 0xc0002dda40 0xc0002ddab0 0xc0002ddb20 0xc0002ddb90 0xc0002ddc70 0xc0002ddce0 0xc000aa53b0 0xc0002ddd50 0xc0002dddc0 0xc0001363c0 0xc0065d33b0 0xc0002dde30 map[0x2ff9080:0xc002d05200 0x2ff95c0:0xc002d05380 0x2ffa2e0:0xc002d05140 0x2ffa580:0xc002d051a0 0x2ffad60:0xc002d05260 0x2ffb000:0xc002d052c0 0x2ffb2a0:0xc002d05320 0x2ffba80:0xc002d053e0 0x2ffbd20:0xc002d05440 0x3026980:0xc001606c60 0x3026d20:0xc001606e40 0x3027800:0xc000af9500 0x3029160:0xc001c42000 0x30298a0:0xc001606ea0 0x302a720:0xc001606b40] 0xc000ae21c0 0} I0423 19:49:13.643364 3773 gateway.go:532] Reconciling gateway with updates I0423 19:49:13.649108 3773 gateway.go:339] Spawning Conntrack Rule Check Thread I0423 19:49:13.649153 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 19:49:13.649172 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 19:49:13.649185 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 19:49:13.649196 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 19:49:13.649193 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 19:49:13.649210 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 19:49:13.649220 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 19:49:13.649233 3773 default_node_network_controller.go:944] Gateway and management port readiness took 56.002437ms I0423 19:49:13.649240 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 19:49:13.649256 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 19:49:13.649266 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0423 19:49:13.649273 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 19:49:13.649282 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0423 19:49:13.649290 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 19:49:13.649299 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 19:49:13.649299 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0423 19:49:13.649307 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 19:49:13.649300 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0423 19:49:13.649314 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0423 19:49:13.649274 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0423 19:49:13.649210 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 19:49:13.649311 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 19:49:13.649351 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 19:49:13.649351 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0423 19:49:13.649363 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 19:49:13.649361 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 19:49:13.649368 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 19:49:13.649372 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 19:49:13.649375 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0423 19:49:13.649381 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 19:49:13.649378 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0423 19:49:13.649387 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 19:49:13.649391 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0423 19:49:13.649391 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 19:49:13.649398 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 19:49:13.649401 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 19:49:13.649406 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 19:49:13.649404 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0423 19:49:13.649411 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 19:49:13.649415 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0423 19:49:13.649273 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 19:49:13.649436 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0423 19:49:13.649366 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 19:49:13.649434 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0423 19:49:13.649459 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0423 19:49:13.649176 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0423 19:49:13.649196 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0423 19:49:13.649265 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 19:49:13.649472 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0423 19:49:13.649474 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0423 19:49:13.649361 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 19:49:13.649491 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 19:49:13.649457 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 19:49:13.649509 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0423 19:49:13.649375 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 19:49:13.649525 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 19:49:13.649287 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0423 19:49:13.649625 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0423 19:49:13.649658 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:49:13.649680 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:49:13.649473 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0423 19:49:13.649721 3773 port_claim.go:71] Svc openshift-ingress/router-default: port 30713 is already open, no action needed I0423 19:49:13.649730 3773 port_claim.go:71] Svc openshift-ingress/router-default: port 31615 is already open, no action needed I0423 19:49:13.649392 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0423 19:49:13.649767 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0423 19:49:13.649493 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 19:49:13.649374 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0423 19:49:13.649891 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0423 19:49:13.649355 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0423 19:49:13.649156 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:49:13.649910 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:49:13.649286 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0423 19:49:13.649290 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0423 19:49:13.649959 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0423 19:49:13.649300 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0423 19:49:13.649297 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 19:49:13.650024 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 19:49:13.649311 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0423 19:49:13.650077 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0423 19:49:13.649321 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 19:49:13.650113 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0423 19:49:13.649336 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0423 19:49:13.650127 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0423 19:49:13.649332 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 19:49:13.650165 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 19:49:13.649337 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 19:49:13.650229 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 19:49:13.649338 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 19:49:13.650252 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 19:49:13.649341 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 19:49:13.650274 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 19:49:13.650282 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0423 19:49:13.649357 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 19:49:13.650295 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 19:49:13.649328 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 19:49:13.650318 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 19:49:13.649361 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 19:49:13.650344 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 19:49:13.649375 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 19:49:13.650360 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 19:49:13.650372 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0423 19:49:13.649387 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 19:49:13.650391 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 19:49:13.649395 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0423 19:49:13.649410 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 19:49:13.649418 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0423 19:49:13.650584 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0423 19:49:13.649341 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 19:49:13.650599 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 19:49:13.650924 3773 node_ip_handler_linux.go:160] Node IP manager is running W0423 19:49:13.688613 3773 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0423 19:49:13.688792 3773 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0423 19:49:13.688947 3773 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0423 19:49:13.699213 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:49:13.700381 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0423 19:49:13.700405 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0423 19:49:13.700413 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0423 19:49:13.700420 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0423 19:49:13.700425 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0423 19:49:13.700431 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0423 19:49:13.700436 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0423 19:49:13.741884 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0423 19:49:13.741901 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0423 19:49:13.741926 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0423 19:49:13.789249 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0423 19:49:13.789280 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0423 19:49:13.789288 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0423 19:49:13.789294 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0423 19:49:13.789313 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0423 19:49:13.789348 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0423 19:49:13.789355 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0423 19:49:13.789361 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0423 19:49:13.789367 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0423 19:49:13.789373 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0423 19:49:13.789381 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0423 19:49:13.789387 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0423 19:49:13.789393 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0423 19:49:13.789398 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0423 19:49:13.789404 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0423 19:49:13.789410 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0423 19:49:15.309406 3773 default_node_network_controller.go:1380] Starting node controller node sync W0423 19:49:15.309492 3773 egressip_healthcheck.go:76] Health checking using insecure connection I0423 19:49:15.309543 3773 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0423 19:49:15.319978 3773 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 10.555042ms I0423 19:49:15.320124 3773 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0423 19:49:15.320385 3773 egressservice_node.go:93] Setting up event handlers for Egress Services I0423 19:49:15.320583 3773 egressservice_node.go:183] Starting Egress Services Controller I0423 19:49:15.320603 3773 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0423 19:49:15.320616 3773 shared_informer.go:356] "Caches are synced" controller="egressservices" I0423 19:49:15.320625 3773 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0423 19:49:15.320632 3773 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0423 19:49:15.320641 3773 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0423 19:49:15.320648 3773 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0423 19:49:15.320654 3773 egressservice_node.go:197] Repairing Egress Services I0423 19:49:15.330668 3773 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0423 19:49:15.330680 3773 external_controller.go:278] Starting Admin Policy Based Route Controller I0423 19:49:15.330978 3773 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0423 19:49:15.330993 3773 link_network_manager.go:119] Link manager is running I0423 19:49:15.331005 3773 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0423 19:49:15.331161 3773 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0423 19:49:15.331628 3773 vrf_manager.go:128] VRF manager is running I0423 19:49:15.331934 3773 node_controller_manager.go:437] Removing flows to drop GARP I0423 19:49:15.331946 3773 gateway.go:532] Reconciling gateway with updates I0423 19:49:15.353269 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 19:49:15.353291 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:49:15.353309 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0423 19:49:15.353306 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0423 19:49:15.353315 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0423 19:49:15.353320 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0423 19:49:15.353326 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0423 19:49:15.353328 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0423 19:49:15.353337 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 19:49:15.353342 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 19:49:15.353346 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 19:49:15.353297 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 19:49:15.353358 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0423 19:49:15.353355 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0423 19:49:15.353360 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 19:49:15.353325 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0423 19:49:15.353374 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 19:49:15.353377 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0423 19:49:15.353377 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0423 19:49:15.353383 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 19:49:15.353383 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 19:49:15.353383 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 19:49:15.353381 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 19:49:15.353394 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 19:49:15.353405 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 19:49:15.353408 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 19:49:15.353274 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 19:49:15.353428 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 19:49:15.353428 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 19:49:15.353440 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0423 19:49:15.353445 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 19:49:15.353452 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 19:49:15.353456 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 19:49:15.353460 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 19:49:15.353460 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 19:49:15.353467 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0423 19:49:15.353471 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 19:49:15.353474 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 19:49:15.353475 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 19:49:15.353476 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 19:49:15.353484 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 19:49:15.353489 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 19:49:15.353490 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 19:49:15.353494 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 19:49:15.353496 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 19:49:15.353496 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0423 19:49:15.353504 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0423 19:49:15.353507 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 19:49:15.353507 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 19:49:15.353515 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 19:49:15.353518 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0423 19:49:15.353520 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 19:49:15.353519 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 19:49:15.353313 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:49:15.353528 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0423 19:49:15.353532 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0423 19:49:15.353536 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 19:49:15.353537 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0423 19:49:15.353546 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 19:49:15.353552 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0423 19:49:15.353461 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0423 19:49:15.353559 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0423 19:49:15.353568 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0423 19:49:15.353569 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 19:49:15.353568 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 19:49:15.353574 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 19:49:15.353584 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 19:49:15.353587 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 19:49:15.353589 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0423 19:49:15.353591 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0423 19:49:15.353391 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0423 19:49:15.353594 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0423 19:49:15.353542 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0423 19:49:15.353605 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0423 19:49:15.353628 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 19:49:15.353496 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 19:49:15.353568 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0423 19:49:15.353377 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0423 19:49:15.353514 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0423 19:49:15.353346 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 19:49:15.353754 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 19:49:15.353469 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0423 19:49:15.353439 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0423 19:49:15.353453 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0423 19:49:15.353444 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0423 19:49:15.353430 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0423 19:49:15.353359 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:49:15.353817 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:49:15.353293 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 19:49:15.353858 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 19:49:15.353455 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0423 19:49:15.353465 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 19:49:15.353469 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 19:49:15.353957 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0423 19:49:15.353470 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0423 19:49:15.353472 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0423 19:49:15.353973 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0423 19:49:15.353987 3773 port_claim.go:71] Svc openshift-ingress/router-default: port 30713 is already open, no action needed I0423 19:49:15.353996 3773 port_claim.go:71] Svc openshift-ingress/router-default: port 31615 is already open, no action needed I0423 19:49:15.353519 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 19:49:15.354044 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 19:49:15.353524 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 19:49:15.353553 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 19:49:15.354119 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 19:49:15.353555 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0423 19:49:15.354172 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0423 19:49:15.354180 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0423 19:49:15.353573 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0423 19:49:15.353574 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0423 19:49:15.353674 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 19:49:15.354196 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 19:49:15.353690 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0423 19:49:15.354233 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0423 19:49:15.353692 3773 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 19:49:15.354271 3773 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 19:49:15.395422 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0423 19:49:15.395448 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0423 19:49:15.395456 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0423 19:49:15.395463 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0423 19:49:15.438174 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0423 19:49:15.438211 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0423 19:49:15.438244 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0423 19:49:15.438254 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0423 19:49:15.438263 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0423 19:49:15.438271 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0423 19:49:15.438279 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0423 19:49:15.438286 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0423 19:49:15.478205 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0423 19:49:15.478229 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0423 19:49:15.478239 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0423 19:49:15.478248 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0423 19:49:15.478258 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0423 19:49:15.478269 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0423 19:49:15.478281 3773 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0423 19:49:17.377297 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-253.ec2.internal I0423 19:49:17.377324 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-253.ec2.internal: 34.188µs I0423 19:49:17.402779 3773 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-85684f4cf6-78p8f] creating logical port openshift-image-registry_image-registry-85684f4cf6-78p8f for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.403402 3773 kube.go:256] Updating pod openshift-image-registry/image-registry-85684f4cf6-78p8f I0423 19:49:17.422860 3773 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-w46q9] creating logical port openshift-dns_dns-default-w46q9 for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.422978 3773 kube.go:256] Updating pod openshift-dns/dns-default-w46q9 I0423 19:49:17.426090 3773 pod.go:62] [openshift-image-registry/image-registry-85684f4cf6-78p8f] pod update took 23.219294ms I0423 19:49:17.426112 3773 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-85684f4cf6-78p8f] addLogicalPort annotation time took 23.243298ms I0423 19:49:17.427390 3773 pods.go:271] [openshift-image-registry/image-registry-85684f4cf6-78p8f] addLogicalPort took 24.628183ms, libovsdb time 906.251µs I0423 19:49:17.439503 3773 pod.go:62] [openshift-dns/dns-default-w46q9] pod update took 16.546395ms I0423 19:49:17.439520 3773 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-w46q9] addLogicalPort annotation time took 16.56612ms I0423 19:49:17.440434 3773 pods.go:271] [openshift-dns/dns-default-w46q9] addLogicalPort took 17.586957ms, libovsdb time 706.699µs I0423 19:49:17.445409 3773 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9pwfg] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-6944588dcc-9pwfg for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.445475 3773 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9pwfg I0423 19:49:17.448754 3773 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-cqpmg] creating logical port openshift-service-ca-operator_service-ca-operator-bc9564db4-cqpmg for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.448843 3773 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-bc9564db4-cqpmg I0423 19:49:17.448922 3773 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-2v5xt] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-2v5xt for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.449023 3773 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-2v5xt I0423 19:49:17.453768 3773 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-gxvz9] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-gxvz9 for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.453851 3773 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-gxvz9 I0423 19:49:17.456550 3773 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-744b9dcb7-v6d4v] creating logical port openshift-ingress_router-default-744b9dcb7-v6d4v for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.456618 3773 kube.go:256] Updating pod openshift-ingress/router-default-744b9dcb7-v6d4v I0423 19:49:17.468977 3773 pod.go:62] [openshift-service-ca-operator/service-ca-operator-bc9564db4-cqpmg] pod update took 20.146507ms I0423 19:49:17.468993 3773 base_network_controller_pods.go:951] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-cqpmg] addLogicalPort annotation time took 20.165734ms I0423 19:49:17.469119 3773 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9pwfg] pod update took 23.658843ms I0423 19:49:17.469149 3773 base_network_controller_pods.go:951] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9pwfg] addLogicalPort annotation time took 23.676042ms I0423 19:49:17.470513 3773 pods.go:271] [openshift-service-ca-operator/service-ca-operator-bc9564db4-cqpmg] addLogicalPort took 21.773345ms, libovsdb time 1.279552ms I0423 19:49:17.470559 3773 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-m58sc] creating logical port openshift-console-operator_console-operator-66dbbb6468-m58sc for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.470602 3773 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-m58sc I0423 19:49:17.470789 3773 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9pwfg] addLogicalPort took 25.387277ms, libovsdb time 1.372093ms I0423 19:49:17.471802 3773 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-xlngr] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-xlngr for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.471852 3773 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-xlngr I0423 19:49:17.473367 3773 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-2v5xt] pod update took 24.383146ms I0423 19:49:17.473392 3773 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-2v5xt] addLogicalPort annotation time took 24.407369ms I0423 19:49:17.474551 3773 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-2v5xt] addLogicalPort took 25.648182ms, libovsdb time 897.125µs I0423 19:49:17.474605 3773 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-kpvwv] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-kpvwv for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.474664 3773 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-kpvwv I0423 19:49:17.484234 3773 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-gxvz9] pod update took 30.397664ms I0423 19:49:17.484251 3773 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-gxvz9] addLogicalPort annotation time took 30.415726ms I0423 19:49:17.484542 3773 pod.go:62] [openshift-ingress/router-default-744b9dcb7-v6d4v] pod update took 27.942678ms I0423 19:49:17.484562 3773 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-744b9dcb7-v6d4v] addLogicalPort annotation time took 27.963065ms I0423 19:49:17.484586 3773 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-84d49796ff-mkxxq] creating logical port openshift-ingress_router-default-84d49796ff-mkxxq for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.484678 3773 kube.go:256] Updating pod openshift-ingress/router-default-84d49796ff-mkxxq I0423 19:49:17.487213 3773 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-gxvz9] addLogicalPort took 33.45269ms, libovsdb time 2.655992ms I0423 19:49:17.489882 3773 pods.go:271] [openshift-ingress/router-default-744b9dcb7-v6d4v] addLogicalPort took 33.338491ms, libovsdb time 3.917801ms I0423 19:49:17.492833 3773 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-mbh6w] creating logical port openshift-insights_insights-operator-6dbb575d6c-mbh6w for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.493002 3773 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-mbh6w I0423 19:49:17.493428 3773 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-m58sc] pod update took 22.832237ms I0423 19:49:17.493489 3773 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-m58sc] addLogicalPort annotation time took 22.897212ms I0423 19:49:17.495512 3773 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-m58sc] addLogicalPort took 24.956112ms, libovsdb time 1.479329ms I0423 19:49:17.496163 3773 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-4mgld] creating logical port openshift-network-diagnostics_network-check-source-84fd6d9d54-4mgld for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.496353 3773 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-84fd6d9d54-4mgld I0423 19:49:17.496398 3773 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-9ts9c] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-9ts9c for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.496572 3773 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-9ts9c I0423 19:49:17.496721 3773 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-kpvwv] pod update took 22.074692ms I0423 19:49:17.496778 3773 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-kpvwv] addLogicalPort annotation time took 22.130387ms I0423 19:49:17.501463 3773 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-kpvwv] addLogicalPort took 26.863731ms, libovsdb time 4.17866ms I0423 19:49:17.505596 3773 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-xlngr] pod update took 33.759077ms I0423 19:49:17.505612 3773 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-xlngr] addLogicalPort annotation time took 33.776461ms I0423 19:49:17.506563 3773 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-xlngr] addLogicalPort took 34.765868ms, libovsdb time 706.468µs I0423 19:49:17.513902 3773 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-9ts9c] pod update took 17.378233ms I0423 19:49:17.513919 3773 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-9ts9c] addLogicalPort annotation time took 17.395811ms I0423 19:49:17.514925 3773 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-9ts9c] addLogicalPort took 18.53903ms, libovsdb time 742.029µs I0423 19:49:17.518278 3773 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-mbh6w] pod update took 25.324154ms I0423 19:49:17.518299 3773 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-mbh6w] addLogicalPort annotation time took 25.347015ms I0423 19:49:17.518508 3773 pod.go:62] [openshift-network-diagnostics/network-check-source-84fd6d9d54-4mgld] pod update took 22.196215ms I0423 19:49:17.518532 3773 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-4mgld] addLogicalPort annotation time took 22.219611ms I0423 19:49:17.518538 3773 pod.go:62] [openshift-ingress/router-default-84d49796ff-mkxxq] pod update took 33.890068ms I0423 19:49:17.518552 3773 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-84d49796ff-mkxxq] addLogicalPort annotation time took 33.903677ms I0423 19:49:17.519424 3773 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-mbh6w] addLogicalPort took 26.605161ms, libovsdb time 759.439µs I0423 19:49:17.519463 3773 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-rmsgp] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67f4d568c7-rmsgp for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.519513 3773 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-rmsgp I0423 19:49:17.519705 3773 pods.go:271] [openshift-ingress/router-default-84d49796ff-mkxxq] addLogicalPort took 35.129863ms, libovsdb time 828.375µs I0423 19:49:17.519757 3773 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-pwg7s] creating logical port openshift-ingress-canary_ingress-canary-pwg7s for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:17.519826 3773 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-pwg7s I0423 19:49:17.520051 3773 pods.go:271] [openshift-network-diagnostics/network-check-source-84fd6d9d54-4mgld] addLogicalPort took 23.920336ms, libovsdb time 1.127667ms I0423 19:49:17.538281 3773 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-rmsgp] pod update took 18.783391ms I0423 19:49:17.538301 3773 base_network_controller_pods.go:951] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-rmsgp] addLogicalPort annotation time took 18.804052ms I0423 19:49:17.538383 3773 pod.go:62] [openshift-ingress-canary/ingress-canary-pwg7s] pod update took 18.582241ms I0423 19:49:17.538404 3773 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-pwg7s] addLogicalPort annotation time took 18.60283ms I0423 19:49:17.539357 3773 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-rmsgp] addLogicalPort took 19.899682ms, libovsdb time 745.489µs I0423 19:49:17.539630 3773 pods.go:271] [openshift-ingress-canary/ingress-canary-pwg7s] addLogicalPort took 19.883369ms, libovsdb time 948.66µs I0423 19:49:17.612149 3773 default_network_controller.go:1025] Node "ip-10-0-1-57.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:49:17.819889 3773 cni.go:452] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9pwfg d15971cda09f76925d203ae86eb9d28c34f5f25808a8aa06b0223ff3b3432b34 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9pwfg d15971cda09f76925d203ae86eb9d28c34f5f25808a8aa06b0223ff3b3432b34 network default NAD default NAD key ] I0423 19:49:17.838456 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-6944588dcc-9pwfg, hostIfaceName: d15971cda09f769, network: default, NAD default, SandboxID: "d15971cda09f76925d203ae86eb9d28c34f5f25808a8aa06b0223ff3b3432b34", PCI device ID: , UID: "5857f3f0-ee49-4e68-8aea-1cf429025b0a", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0423 19:49:17.860590 3773 cni.go:452] [openshift-service-ca-operator/service-ca-operator-bc9564db4-cqpmg 1876c3d01a8b198bdaf7164ab65f707a424770dcd3d52e30c5a4c3884152587d network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-cqpmg 1876c3d01a8b198bdaf7164ab65f707a424770dcd3d52e30c5a4c3884152587d network default NAD default NAD key ] I0423 19:49:17.861999 3773 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-2v5xt e743d32f1bb25eff8244000d742b870093810ac64dc59cd37ba3ee7e49020fca network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-2v5xt e743d32f1bb25eff8244000d742b870093810ac64dc59cd37ba3ee7e49020fca network default NAD default NAD key ] I0423 19:49:17.875196 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-bc9564db4-cqpmg, hostIfaceName: 1876c3d01a8b198, network: default, NAD default, SandboxID: "1876c3d01a8b198bdaf7164ab65f707a424770dcd3d52e30c5a4c3884152587d", PCI device ID: , UID: "d4c52ce1-78ff-4c35-94fa-888eed9fc21f", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0423 19:49:17.876367 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-2v5xt, hostIfaceName: e743d32f1bb25ef, network: default, NAD default, SandboxID: "e743d32f1bb25eff8244000d742b870093810ac64dc59cd37ba3ee7e49020fca", PCI device ID: , UID: "3b28d3e2-1898-4a3d-b261-1b9ce34f9ae6", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0423 19:49:17.900292 3773 cni.go:473] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9pwfg d15971cda09f76925d203ae86eb9d28c34f5f25808a8aa06b0223ff3b3432b34 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9pwfg d15971cda09f76925d203ae86eb9d28c34f5f25808a8aa06b0223ff3b3432b34 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:ae:73:ce:1e:4f\",\"name\":\"d15971cda09f769\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/90f4684e-b677-4dc5-bfed-b948bbcd7069\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:17.905048 3773 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-m58sc cd464ec8be4d69918f48d64aeeaa26c0d577c352dcff91f5cfe6d79e5dcd1b6e network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-m58sc cd464ec8be4d69918f48d64aeeaa26c0d577c352dcff91f5cfe6d79e5dcd1b6e network default NAD default NAD key ] I0423 19:49:17.916888 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-m58sc, hostIfaceName: cd464ec8be4d699, network: default, NAD default, SandboxID: "cd464ec8be4d69918f48d64aeeaa26c0d577c352dcff91f5cfe6d79e5dcd1b6e", PCI device ID: , UID: "29ad7ab2-0540-4659-846c-07d40d98c601", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0423 19:49:17.923635 3773 cni.go:473] [openshift-service-ca-operator/service-ca-operator-bc9564db4-cqpmg 1876c3d01a8b198bdaf7164ab65f707a424770dcd3d52e30c5a4c3884152587d network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-cqpmg 1876c3d01a8b198bdaf7164ab65f707a424770dcd3d52e30c5a4c3884152587d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:59:52:f1:2a:64\",\"name\":\"1876c3d01a8b198\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ebe669e9-e859-43cd-b201-23de7bf9992e\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:17.928434 3773 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-2v5xt e743d32f1bb25eff8244000d742b870093810ac64dc59cd37ba3ee7e49020fca network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-2v5xt e743d32f1bb25eff8244000d742b870093810ac64dc59cd37ba3ee7e49020fca network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:9b:c4:0d:a1:f4\",\"name\":\"e743d32f1bb25ef\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f5e4ff04-6122-46d8-a2eb-16f5ea3ffd62\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:17.961978 3773 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-m58sc cd464ec8be4d69918f48d64aeeaa26c0d577c352dcff91f5cfe6d79e5dcd1b6e network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-m58sc cd464ec8be4d69918f48d64aeeaa26c0d577c352dcff91f5cfe6d79e5dcd1b6e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:cc:7b:6d:33:5a\",\"name\":\"cd464ec8be4d699\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e78d0341-a115-4d17-a51a-1c01414e6b2d\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:17.994546 3773 cni.go:452] [openshift-ingress-canary/ingress-canary-pwg7s c3de0b2312a1807ed9132322baed1ded971d9089abfb37d2057a3ae07fc22c37 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-pwg7s c3de0b2312a1807ed9132322baed1ded971d9089abfb37d2057a3ae07fc22c37 network default NAD default NAD key ] I0423 19:49:18.006419 3773 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-mbh6w ca017c260ad0ed55c2f8f64dd046eaed560e9848f6aaf0fde570374d91f0f833 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-mbh6w ca017c260ad0ed55c2f8f64dd046eaed560e9848f6aaf0fde570374d91f0f833 network default NAD default NAD key ] I0423 19:49:18.006914 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-pwg7s, hostIfaceName: c3de0b2312a1807, network: default, NAD default, SandboxID: "c3de0b2312a1807ed9132322baed1ded971d9089abfb37d2057a3ae07fc22c37", PCI device ID: , UID: "e6fe56b3-fe9e-44bc-a175-b05a4c9fdffe", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0423 19:49:18.018069 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-mbh6w, hostIfaceName: ca017c260ad0ed5, network: default, NAD default, SandboxID: "ca017c260ad0ed55c2f8f64dd046eaed560e9848f6aaf0fde570374d91f0f833", PCI device ID: , UID: "63a72c0a-c4ba-46dd-85aa-a2cd06964436", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0423 19:49:18.037501 3773 cni.go:452] [openshift-network-diagnostics/network-check-source-84fd6d9d54-4mgld 05c7313b6f472135bcdbbc4bb9e72340c3d21461903834512e0f881c3c7180f3 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-4mgld 05c7313b6f472135bcdbbc4bb9e72340c3d21461903834512e0f881c3c7180f3 network default NAD default NAD key ] I0423 19:49:18.048240 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-84fd6d9d54-4mgld, hostIfaceName: 05c7313b6f47213, network: default, NAD default, SandboxID: "05c7313b6f472135bcdbbc4bb9e72340c3d21461903834512e0f881c3c7180f3", PCI device ID: , UID: "f3c4c52e-8ef4-442a-9ba7-250695dcbee9", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0423 19:49:18.054302 3773 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-9ts9c a4c5722ca742aae522126afd6226effb09c1ebbfd4a02cadce5cc305271b1b39 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-9ts9c a4c5722ca742aae522126afd6226effb09c1ebbfd4a02cadce5cc305271b1b39 network default NAD default NAD key ] I0423 19:49:18.060090 3773 cni.go:473] [openshift-ingress-canary/ingress-canary-pwg7s c3de0b2312a1807ed9132322baed1ded971d9089abfb37d2057a3ae07fc22c37 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-pwg7s c3de0b2312a1807ed9132322baed1ded971d9089abfb37d2057a3ae07fc22c37 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:2c:28:9d:28:ed\",\"name\":\"c3de0b2312a1807\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c245d1e9-7b57-48a0-a933-409ababed752\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:18.065989 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-9ts9c, hostIfaceName: a4c5722ca742aae, network: default, NAD default, SandboxID: "a4c5722ca742aae522126afd6226effb09c1ebbfd4a02cadce5cc305271b1b39", PCI device ID: , UID: "3339fc51-9f1c-4362-873e-155c91a45b7a", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0423 19:49:18.071550 3773 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-mbh6w ca017c260ad0ed55c2f8f64dd046eaed560e9848f6aaf0fde570374d91f0f833 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-mbh6w ca017c260ad0ed55c2f8f64dd046eaed560e9848f6aaf0fde570374d91f0f833 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:d3:4d:cb:a2:86\",\"name\":\"ca017c260ad0ed5\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/18c41d59-462b-4420-a465-99af3f441d20\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:18.098403 3773 cni.go:473] [openshift-network-diagnostics/network-check-source-84fd6d9d54-4mgld 05c7313b6f472135bcdbbc4bb9e72340c3d21461903834512e0f881c3c7180f3 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-4mgld 05c7313b6f472135bcdbbc4bb9e72340c3d21461903834512e0f881c3c7180f3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:d3:da:9c:d3:45\",\"name\":\"05c7313b6f47213\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9ad4aa83-6dbe-487f-974c-54c71ba194ef\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:18.114900 3773 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-9ts9c a4c5722ca742aae522126afd6226effb09c1ebbfd4a02cadce5cc305271b1b39 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-9ts9c a4c5722ca742aae522126afd6226effb09c1ebbfd4a02cadce5cc305271b1b39 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:d4:ea:78:01:00\",\"name\":\"a4c5722ca742aae\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d6006efd-afeb-4340-8cef-2edb9923deb6\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:23.295667 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:49:24.725692 3773 cni.go:452] [openshift-network-diagnostics/network-check-target-f4w6t f841000e6cb9519de77460ad8fdc74ef9eefb3fa006c36ac5565a34b752e8524 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-f4w6t f841000e6cb9519de77460ad8fdc74ef9eefb3fa006c36ac5565a34b752e8524 network default NAD default NAD key ] I0423 19:49:24.735672 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-f4w6t, hostIfaceName: f841000e6cb9519, network: default, NAD default, SandboxID: "f841000e6cb9519de77460ad8fdc74ef9eefb3fa006c36ac5565a34b752e8524", PCI device ID: , UID: "67c06ba7-760d-4ac6-bdc3-e4a4bcc0bfde", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0423 19:49:24.786246 3773 cni.go:473] [openshift-network-diagnostics/network-check-target-f4w6t f841000e6cb9519de77460ad8fdc74ef9eefb3fa006c36ac5565a34b752e8524 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-f4w6t f841000e6cb9519de77460ad8fdc74ef9eefb3fa006c36ac5565a34b752e8524 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:16:55:ca:a9:7f\",\"name\":\"f841000e6cb9519\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2cec6ea2-d269-4ca0-8548-b139196b926d\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 19:49:27.236707 3773 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0423 19:49:27.264938 3773 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:49:27.388661 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0423 19:49:27.660384 3773 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0423 19:49:27.661357 3773 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 950.975µs I0423 19:49:27.688918 3773 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0423 19:49:27.836078 3773 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-bns45] creating logical port openshift-kube-storage-version-migrator_migrator-846d9b8bdc-bns45 for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:27.836223 3773 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-846d9b8bdc-bns45 I0423 19:49:27.864910 3773 pod.go:62] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-bns45] pod update took 28.711255ms I0423 19:49:27.864932 3773 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-bns45] addLogicalPort annotation time took 28.739085ms I0423 19:49:27.866092 3773 pods.go:271] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-bns45] addLogicalPort took 30.033792ms, libovsdb time 830.751µs I0423 19:49:28.026095 3773 namespace.go:98] [openshift-service-ca] adding namespace I0423 19:49:28.026921 3773 namespace.go:102] [openshift-service-ca] adding namespace took 804.559µs I0423 19:49:28.055103 3773 namespace.go:142] [openshift-service-ca] updating namespace I0423 19:49:28.188369 3773 cni.go:452] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-bns45 1599bff07d270e1a889ac01a4a1c77e039ada4e92079429f7666646f0101cbb3 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-bns45 1599bff07d270e1a889ac01a4a1c77e039ada4e92079429f7666646f0101cbb3 network default NAD default NAD key ] I0423 19:49:28.199709 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-846d9b8bdc-bns45, hostIfaceName: 1599bff07d270e1, network: default, NAD default, SandboxID: "1599bff07d270e1a889ac01a4a1c77e039ada4e92079429f7666646f0101cbb3", PCI device ID: , UID: "a3615306-a47d-4165-bff7-16f96373868a", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0423 19:49:28.248706 3773 cni.go:473] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-bns45 1599bff07d270e1a889ac01a4a1c77e039ada4e92079429f7666646f0101cbb3 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-bns45 1599bff07d270e1a889ac01a4a1c77e039ada4e92079429f7666646f0101cbb3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:4a:be:81:80:66\",\"name\":\"1599bff07d270e1\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5d38d3ab-d3d4-4b7f-8a27-4610d9749156\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:28.326637 3773 egressfirewall.go:723] Syncing node "ip-10-0-1-57.ec2.internal" for egress firewall I0423 19:49:28.326737 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-57.ec2.internal I0423 19:49:28.326756 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-57.ec2.internal: 25.115µs I0423 19:49:28.326787 3773 default_network_controller.go:1025] Node "ip-10-0-1-57.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:49:29.677436 3773 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-8gk5l] creating logical port openshift-service-ca_service-ca-d9cb48497-8gk5l for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:29.677542 3773 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-8gk5l I0423 19:49:29.690568 3773 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-8gk5l] pod update took 13.052668ms I0423 19:49:29.690602 3773 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-8gk5l] addLogicalPort annotation time took 13.092726ms I0423 19:49:29.692036 3773 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-8gk5l] addLogicalPort took 14.621673ms, libovsdb time 1.020835ms I0423 19:49:30.056820 3773 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-8gk5l d07889d5492bf3ea69f5fed7442d8169a63632bea1a4768f83810572c9915c91 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-8gk5l d07889d5492bf3ea69f5fed7442d8169a63632bea1a4768f83810572c9915c91 network default NAD default NAD key ] I0423 19:49:30.066953 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-8gk5l, hostIfaceName: d07889d5492bf3e, network: default, NAD default, SandboxID: "d07889d5492bf3ea69f5fed7442d8169a63632bea1a4768f83810572c9915c91", PCI device ID: , UID: "b077f48d-e5e8-4e9d-a498-14b505a33d78", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0423 19:49:30.112968 3773 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-8gk5l d07889d5492bf3ea69f5fed7442d8169a63632bea1a4768f83810572c9915c91 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-8gk5l d07889d5492bf3ea69f5fed7442d8169a63632bea1a4768f83810572c9915c91 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:82:3d:70:05:2d\",\"name\":\"d07889d5492bf3e\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8f4cb0b5-55cb-4fe1-99c0-171bf84d414d\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:31.745385 3773 default_network_controller.go:1025] Node "ip-10-0-1-57.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:49:31.773600 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:49:32.165615 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-42.ec2.internal I0423 19:49:32.165683 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-42.ec2.internal: 72.046µs I0423 19:49:32.165626 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:49:32.165631 3773 egressfirewall.go:723] Syncing node "ip-10-0-2-42.ec2.internal" for egress firewall I0423 19:49:33.637310 3773 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-mwd5v] creating logical port openshift-insights_insights-runtime-extractor-mwd5v for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:33.637435 3773 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-mwd5v I0423 19:49:33.638681 3773 cni.go:452] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-rmsgp 486fb0e9f475809d5c633d86dd1df007ddd9b8518484e5df04d0bb1eee3ef1ca network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-rmsgp 486fb0e9f475809d5c633d86dd1df007ddd9b8518484e5df04d0bb1eee3ef1ca network default NAD default NAD key ] I0423 19:49:33.647655 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67f4d568c7-rmsgp, hostIfaceName: 486fb0e9f475809, network: default, NAD default, SandboxID: "486fb0e9f475809d5c633d86dd1df007ddd9b8518484e5df04d0bb1eee3ef1ca", PCI device ID: , UID: "f102f23f-b122-4dd4-be17-1b3e210183c4", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0423 19:49:33.651019 3773 pod.go:62] [openshift-insights/insights-runtime-extractor-mwd5v] pod update took 13.611356ms I0423 19:49:33.651039 3773 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-mwd5v] addLogicalPort annotation time took 13.637381ms I0423 19:49:33.652312 3773 pods.go:271] [openshift-insights/insights-runtime-extractor-mwd5v] addLogicalPort took 15.018138ms, libovsdb time 886.528µs I0423 19:49:33.664546 3773 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-57.ec2.internal in network "default" I0423 19:49:33.664589 3773 node_tracker.go:235] Node ip-10-0-1-57.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-57.ec2.internal" I0423 19:49:33.664606 3773 node_tracker.go:169] Node ip-10-0-1-57.ec2.internal switch + router changed, syncing services in network "default" I0423 19:49:33.664620 3773 services_controller.go:574] Full service sync requested for network=default I0423 19:49:33.664628 3773 default_network_controller.go:1025] Node "ip-10-0-1-57.ec2.internal" in remote zone "ip-10-0-1-57.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0423 19:49:33.665097 3773 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:49:33.665180 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0423 19:49:33.665196 3773 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:49:33.695044 3773 cni.go:473] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-rmsgp 486fb0e9f475809d5c633d86dd1df007ddd9b8518484e5df04d0bb1eee3ef1ca network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-rmsgp 486fb0e9f475809d5c633d86dd1df007ddd9b8518484e5df04d0bb1eee3ef1ca network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:17:54:7f:ae:7e\",\"name\":\"486fb0e9f475809\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/758dd1cc-1c14-44a3-b308-94a583c0cb1d\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:33.818388 3773 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-57.ec2.internal in network "default" I0423 19:49:33.818505 3773 node_tracker.go:169] Node ip-10-0-1-57.ec2.internal switch + router changed, syncing services in network "default" I0423 19:49:33.818524 3773 services_controller.go:574] Full service sync requested for network=default I0423 19:49:33.818410 3773 default_network_controller.go:1025] Node "ip-10-0-1-57.ec2.internal" in remote zone "ip-10-0-1-57.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0423 19:49:33.818690 3773 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:49:33.818804 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0423 19:49:33.818827 3773 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:49:33.819878 3773 zone_ic_handler.go:195] Time taken to create transit switch: 423.257µs I0423 19:49:33.819904 3773 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-57.ec2.internal for the network default I0423 19:49:33.821254 3773 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-57.ec2.internal" on network "default" took: 1.818927ms W0423 19:49:34.674326 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0423 19:49:36.373739 3773 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-57.ec2.internal in network "default" I0423 19:49:36.373759 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-57.ec2.internal I0423 19:49:36.373776 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-57.ec2.internal: 23.059µs I0423 19:49:36.373782 3773 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-57.ec2.internal", IP: 10.0.1.57 to PMTUD blocking rules I0423 19:49:36.373796 3773 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-57.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0423 19:49:36.373816 3773 node_tracker.go:169] Node ip-10-0-1-57.ec2.internal switch + router changed, syncing services in network "default" I0423 19:49:36.373823 3773 egressfirewall.go:723] Syncing node "ip-10-0-1-57.ec2.internal" for egress firewall I0423 19:49:36.373832 3773 services_controller.go:574] Full service sync requested for network=default W0423 19:49:36.374192 3773 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:49:36.374269 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0423 19:49:36.374300 3773 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:49:36.379805 3773 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-57.ec2.internal. Updating no re-route policies I0423 19:49:36.388725 3773 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-57.ec2.internal", IP: 10.0.1.57 to PMTUD blocking rules I0423 19:49:36.388744 3773 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-57.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0423 19:49:41.926616 3773 obj_retry.go:334] Retry object setup: *v1.Node ip-10-0-2-42.ec2.internal I0423 19:49:41.926641 3773 obj_retry.go:398] Adding new object: *v1.Node ip-10-0-2-42.ec2.internal I0423 19:49:41.926726 3773 obj_retry.go:409] Retry add failed for *v1.Node ip-10-0-2-42.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-2-42.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-42.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-42.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-42.ec2.internal I0423 19:49:43.433986 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:49:43.472028 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-mk98q I0423 19:49:43.472055 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-mk98q I0423 19:49:43.472028 3773 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-sg6sb I0423 19:49:43.472096 3773 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-sg6sb I0423 19:49:43.472149 3773 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-sg6sb, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-sg6sb: suppressed error logged: pod openshift-multus/network-metrics-daemon-sg6sb: no pod IPs found I0423 19:49:43.472156 3773 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-mk98q, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-mk98q: suppressed error logged: pod openshift-network-diagnostics/network-check-target-mk98q: no pod IPs found I0423 19:49:49.597739 3773 cni.go:452] [openshift-image-registry/image-registry-85684f4cf6-78p8f 5dc77e238cb4d79880e16fad3d7dfbfc3e22d0e6e7cb2c318b923b2873e5e1a0 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-85684f4cf6-78p8f 5dc77e238cb4d79880e16fad3d7dfbfc3e22d0e6e7cb2c318b923b2873e5e1a0 network default NAD default NAD key ] I0423 19:49:49.607230 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-85684f4cf6-78p8f, hostIfaceName: 5dc77e238cb4d79, network: default, NAD default, SandboxID: "5dc77e238cb4d79880e16fad3d7dfbfc3e22d0e6e7cb2c318b923b2873e5e1a0", PCI device ID: , UID: "8d5a1036-d90f-415e-8ae3-996ff0de8656", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0423 19:49:49.632455 3773 cni.go:452] [openshift-dns/dns-default-w46q9 da9780b74f81771e05f0ae4804a4ede6d85c1bdb2e4f6d277ba0fca46c101f49 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-w46q9 da9780b74f81771e05f0ae4804a4ede6d85c1bdb2e4f6d277ba0fca46c101f49 network default NAD default NAD key ] I0423 19:49:49.641321 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-w46q9, hostIfaceName: da9780b74f81771, network: default, NAD default, SandboxID: "da9780b74f81771e05f0ae4804a4ede6d85c1bdb2e4f6d277ba0fca46c101f49", PCI device ID: , UID: "2dfbc411-4ed0-488b-94a8-35c70fbc0872", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0423 19:49:49.659301 3773 cni.go:473] [openshift-image-registry/image-registry-85684f4cf6-78p8f 5dc77e238cb4d79880e16fad3d7dfbfc3e22d0e6e7cb2c318b923b2873e5e1a0 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-85684f4cf6-78p8f 5dc77e238cb4d79880e16fad3d7dfbfc3e22d0e6e7cb2c318b923b2873e5e1a0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:e0:02:84:f6:3a\",\"name\":\"5dc77e238cb4d79\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/58a8c7cb-36b1-4229-b32e-829a82ede51d\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:49.669295 3773 cni.go:452] [openshift-insights/insights-runtime-extractor-mwd5v 6a204f8e692d40776038ec427f7b52a5a6f0fc3af8847d484868bed1c1198a92 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-mwd5v 6a204f8e692d40776038ec427f7b52a5a6f0fc3af8847d484868bed1c1198a92 network default NAD default NAD key ] I0423 19:49:49.687072 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-mwd5v, hostIfaceName: 6a204f8e692d407, network: default, NAD default, SandboxID: "6a204f8e692d40776038ec427f7b52a5a6f0fc3af8847d484868bed1c1198a92", PCI device ID: , UID: "0326d5df-fa04-4cc8-9d03-6dfefd7557b7", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0423 19:49:49.690909 3773 cni.go:473] [openshift-dns/dns-default-w46q9 da9780b74f81771e05f0ae4804a4ede6d85c1bdb2e4f6d277ba0fca46c101f49 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-w46q9 da9780b74f81771e05f0ae4804a4ede6d85c1bdb2e4f6d277ba0fca46c101f49 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:33:bf:4e:b5:54\",\"name\":\"da9780b74f81771\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/07b095c0-5228-49b3-bb56-5f3e929db13b\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:49.717340 3773 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-gxvz9 78900b05feae13a946a429305a62d87365f440e9e102b0aa589b1a167c53b12b network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-gxvz9 78900b05feae13a946a429305a62d87365f440e9e102b0aa589b1a167c53b12b network default NAD default NAD key ] I0423 19:49:49.736156 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-gxvz9, hostIfaceName: 78900b05feae13a, network: default, NAD default, SandboxID: "78900b05feae13a946a429305a62d87365f440e9e102b0aa589b1a167c53b12b", PCI device ID: , UID: "8955d6c4-f5cb-4199-bccc-1602a1de53b6", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0423 19:49:49.740422 3773 cni.go:473] [openshift-insights/insights-runtime-extractor-mwd5v 6a204f8e692d40776038ec427f7b52a5a6f0fc3af8847d484868bed1c1198a92 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-mwd5v 6a204f8e692d40776038ec427f7b52a5a6f0fc3af8847d484868bed1c1198a92 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:f4:cf:11:42:3e\",\"name\":\"6a204f8e692d407\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/68b36430-b480-4b34-8a5c-56aab3622e05\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:49.745832 3773 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-kpvwv 370b9b2b1fb792a93877a45b914b4096f98cbe05c228b9bfe61455da0f07a670 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-kpvwv 370b9b2b1fb792a93877a45b914b4096f98cbe05c228b9bfe61455da0f07a670 network default NAD default NAD key ] I0423 19:49:49.755758 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-kpvwv, hostIfaceName: 370b9b2b1fb792a, network: default, NAD default, SandboxID: "370b9b2b1fb792a93877a45b914b4096f98cbe05c228b9bfe61455da0f07a670", PCI device ID: , UID: "24cf4adc-7052-4945-8ff9-8696b71c351b", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0423 19:49:49.782726 3773 cni.go:452] [openshift-ingress/router-default-744b9dcb7-v6d4v a9f4c9313a7dc8edde0d545dc63613dc7eb76ea149eca8513e27ce286d678f6b network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-744b9dcb7-v6d4v a9f4c9313a7dc8edde0d545dc63613dc7eb76ea149eca8513e27ce286d678f6b network default NAD default NAD key ] I0423 19:49:49.792599 3773 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-xlngr 136a28db81b6ba16ffc7faf5180dd123b7c79f21f749f7087f1f0a8813d609f5 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-xlngr 136a28db81b6ba16ffc7faf5180dd123b7c79f21f749f7087f1f0a8813d609f5 network default NAD default NAD key ] I0423 19:49:49.792597 3773 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-gxvz9 78900b05feae13a946a429305a62d87365f440e9e102b0aa589b1a167c53b12b network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-gxvz9 78900b05feae13a946a429305a62d87365f440e9e102b0aa589b1a167c53b12b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:fa:ec:3d:0f:f1\",\"name\":\"78900b05feae13a\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b1b05d32-b485-4008-afee-87554c89a68c\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:49.793984 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-744b9dcb7-v6d4v, hostIfaceName: a9f4c9313a7dc8e, network: default, NAD default, SandboxID: "a9f4c9313a7dc8edde0d545dc63613dc7eb76ea149eca8513e27ce286d678f6b", PCI device ID: , UID: "a98d4d44-368f-4984-8d3a-c57efd53dbee", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0423 19:49:49.813226 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-xlngr, hostIfaceName: 136a28db81b6ba1, network: default, NAD default, SandboxID: "136a28db81b6ba16ffc7faf5180dd123b7c79f21f749f7087f1f0a8813d609f5", PCI device ID: , UID: "5192521b-ec0a-4135-816d-0f3b6255a5d7", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0423 19:49:49.814974 3773 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-kpvwv 370b9b2b1fb792a93877a45b914b4096f98cbe05c228b9bfe61455da0f07a670 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-kpvwv 370b9b2b1fb792a93877a45b914b4096f98cbe05c228b9bfe61455da0f07a670 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:cf:44:6d:99:83\",\"name\":\"370b9b2b1fb792a\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7c0b6141-3d62-486d-9d8a-e3b33f0f149c\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:49.817719 3773 cni.go:452] [openshift-ingress/router-default-84d49796ff-mkxxq c1075e841d85fc9f6733156eeb0c89360cbc9292048c68c423c13955d1a26f54 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-84d49796ff-mkxxq c1075e841d85fc9f6733156eeb0c89360cbc9292048c68c423c13955d1a26f54 network default NAD default NAD key ] I0423 19:49:49.831850 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-84d49796ff-mkxxq, hostIfaceName: c1075e841d85fc9, network: default, NAD default, SandboxID: "c1075e841d85fc9f6733156eeb0c89360cbc9292048c68c423c13955d1a26f54", PCI device ID: , UID: "939109c5-ed16-4b9c-a651-8c12da4ee220", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0423 19:49:49.850109 3773 cni.go:473] [openshift-ingress/router-default-744b9dcb7-v6d4v a9f4c9313a7dc8edde0d545dc63613dc7eb76ea149eca8513e27ce286d678f6b network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-744b9dcb7-v6d4v a9f4c9313a7dc8edde0d545dc63613dc7eb76ea149eca8513e27ce286d678f6b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:8e:d3:19:91:2a\",\"name\":\"a9f4c9313a7dc8e\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b77c511f-63d0-4407-964c-c09c4a9270e4\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:49.872117 3773 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-xlngr 136a28db81b6ba16ffc7faf5180dd123b7c79f21f749f7087f1f0a8813d609f5 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-xlngr 136a28db81b6ba16ffc7faf5180dd123b7c79f21f749f7087f1f0a8813d609f5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:1a:bf:a1:1b:7b\",\"name\":\"136a28db81b6ba1\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a2863170-0b3f-4c85-b1b7-715319accaaa\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:49.893338 3773 cni.go:473] [openshift-ingress/router-default-84d49796ff-mkxxq c1075e841d85fc9f6733156eeb0c89360cbc9292048c68c423c13955d1a26f54 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-84d49796ff-mkxxq c1075e841d85fc9f6733156eeb0c89360cbc9292048c68c423c13955d1a26f54 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:04:2b:8b:e0:f7\",\"name\":\"c1075e841d85fc9\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/acfd3358-baea-4e63-a984-23ae45ec2fd2\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:53.110011 3773 cni.go:452] [openshift-multus/network-metrics-daemon-tn7rh 0cdef91b57d0a8ad091e82f942dc14ea6e155a328eb9aa9a8bb82e9c90f140dd network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-tn7rh 0cdef91b57d0a8ad091e82f942dc14ea6e155a328eb9aa9a8bb82e9c90f140dd network default NAD default NAD key ] I0423 19:49:53.122741 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-tn7rh, hostIfaceName: 0cdef91b57d0a8a, network: default, NAD default, SandboxID: "0cdef91b57d0a8ad091e82f942dc14ea6e155a328eb9aa9a8bb82e9c90f140dd", PCI device ID: , UID: "e4828ff2-736f-4911-bd61-e40230820864", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0423 19:49:53.176004 3773 cni.go:473] [openshift-multus/network-metrics-daemon-tn7rh 0cdef91b57d0a8ad091e82f942dc14ea6e155a328eb9aa9a8bb82e9c90f140dd network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-tn7rh 0cdef91b57d0a8ad091e82f942dc14ea6e155a328eb9aa9a8bb82e9c90f140dd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:e6:c2:1f:c3:ae\",\"name\":\"0cdef91b57d0a8a\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/68d07df6-fdd5-437f-b915-03b7527fdc38\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 19:49:53.860302 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0423 19:49:53.860416 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0423 19:49:53.919644 3773 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0423 19:49:54.010993 3773 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0423 19:49:54.014650 3773 base_network_controller_pods.go:487] [default/openshift-console/console-788b45bb7-d685x] creating logical port openshift-console_console-788b45bb7-d685x for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:54.014771 3773 kube.go:256] Updating pod openshift-console/console-788b45bb7-d685x I0423 19:49:54.020632 3773 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-6nrbl] creating logical port openshift-console_downloads-5c54f4944f-6nrbl for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:54.020730 3773 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-6nrbl I0423 19:49:54.071748 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0423 19:49:54.077558 3773 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0423 19:49:54.080089 3773 pod.go:62] [openshift-console/console-788b45bb7-d685x] pod update took 65.343915ms I0423 19:49:54.080109 3773 base_network_controller_pods.go:951] [default/openshift-console/console-788b45bb7-d685x] addLogicalPort annotation time took 65.367151ms I0423 19:49:54.081301 3773 pods.go:271] [openshift-console/console-788b45bb7-d685x] addLogicalPort took 66.668195ms, libovsdb time 850.995µs I0423 19:49:54.083734 3773 pod.go:62] [openshift-console/downloads-5c54f4944f-6nrbl] pod update took 63.026526ms I0423 19:49:54.083757 3773 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-6nrbl] addLogicalPort annotation time took 63.050999ms I0423 19:49:54.085031 3773 pods.go:271] [openshift-console/downloads-5c54f4944f-6nrbl] addLogicalPort took 64.412021ms, libovsdb time 839.164µs W0423 19:49:54.090939 3773 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-5fdb8fd55-ftc48 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-5fdb8fd55-ftc48: no pod IPs found I0423 19:49:54.264072 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0423 19:49:54.556031 3773 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0423 19:49:54.556050 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0423 19:49:54.556056 3773 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0423 19:49:54.607554 3773 cni.go:452] [openshift-console/console-788b45bb7-d685x aef80746d7e85d7f9b6fcba84ff6d8abd9db167fb0257120d932b59965e8e114 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-788b45bb7-d685x aef80746d7e85d7f9b6fcba84ff6d8abd9db167fb0257120d932b59965e8e114 network default NAD default NAD key ] I0423 19:49:54.607586 3773 cni.go:452] [openshift-console/downloads-5c54f4944f-6nrbl 41516a981e7606a9e5bf9ef558acf405645f30131b581866022b7fbeea25d5d6 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-6nrbl 41516a981e7606a9e5bf9ef558acf405645f30131b581866022b7fbeea25d5d6 network default NAD default NAD key ] I0423 19:49:54.619088 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-6nrbl, hostIfaceName: 41516a981e7606a, network: default, NAD default, SandboxID: "41516a981e7606a9e5bf9ef558acf405645f30131b581866022b7fbeea25d5d6", PCI device ID: , UID: "c0c62639-dcda-48a9-9c2f-b73dbe708480", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0423 19:49:54.619239 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-788b45bb7-d685x, hostIfaceName: aef80746d7e85d7, network: default, NAD default, SandboxID: "aef80746d7e85d7f9b6fcba84ff6d8abd9db167fb0257120d932b59965e8e114", PCI device ID: , UID: "c83e6307-1cbf-4c7f-8d34-48a01fa596d9", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0423 19:49:54.625218 3773 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pp9dv] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-pp9dv for pod on switch ip-10-0-0-253.ec2.internal I0423 19:49:54.625349 3773 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pp9dv I0423 19:49:54.650238 3773 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pp9dv] pod update took 24.915445ms I0423 19:49:54.650271 3773 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pp9dv] addLogicalPort annotation time took 24.951295ms I0423 19:49:54.651783 3773 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pp9dv] addLogicalPort took 26.578382ms, libovsdb time 1.035957ms I0423 19:49:54.675162 3773 cni.go:473] [openshift-console/downloads-5c54f4944f-6nrbl 41516a981e7606a9e5bf9ef558acf405645f30131b581866022b7fbeea25d5d6 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-6nrbl 41516a981e7606a9e5bf9ef558acf405645f30131b581866022b7fbeea25d5d6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:55:c2:76:64:37\",\"name\":\"41516a981e7606a\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1b70ca21-8409-4da6-b756-71cbf4429b35\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:54.690173 3773 cni.go:473] [openshift-console/console-788b45bb7-d685x aef80746d7e85d7f9b6fcba84ff6d8abd9db167fb0257120d932b59965e8e114 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-788b45bb7-d685x aef80746d7e85d7f9b6fcba84ff6d8abd9db167fb0257120d932b59965e8e114 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:37:f8:80:a6:7f\",\"name\":\"aef80746d7e85d7\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dd4fdd36-598e-4832-b022-f68fc16f8ec2\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:54.996834 3773 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-42.ec2.internal in network "default" I0423 19:49:54.996894 3773 node_tracker.go:235] Node ip-10-0-2-42.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-42.ec2.internal" I0423 19:49:54.996913 3773 node_tracker.go:169] Node ip-10-0-2-42.ec2.internal switch + router changed, syncing services in network "default" I0423 19:49:54.996924 3773 services_controller.go:574] Full service sync requested for network=default I0423 19:49:54.996941 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "ip-10-0-2-42.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0423 19:49:54.997120 3773 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:49:54.997362 3773 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0423 19:49:54.997382 3773 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0423 19:49:54.997389 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0423 19:49:54.997485 3773 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0423 19:49:55.048547 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "ip-10-0-2-42.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:49:55.139458 3773 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-42.ec2.internal in network "default" I0423 19:49:55.139502 3773 default_network_controller.go:1025] Node "ip-10-0-2-42.ec2.internal" in remote zone "ip-10-0-2-42.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:49:55.139527 3773 node_tracker.go:169] Node ip-10-0-2-42.ec2.internal switch + router changed, syncing services in network "default" I0423 19:49:55.139543 3773 services_controller.go:574] Full service sync requested for network=default W0423 19:49:55.139704 3773 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:49:55.139824 3773 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:49:55.140052 3773 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0423 19:49:55.140064 3773 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0423 19:49:55.140068 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default I0423 19:49:55.140484 3773 zone_ic_handler.go:195] Time taken to create transit switch: 301.814µs I0423 19:49:55.140498 3773 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-42.ec2.internal for the network default I0423 19:49:55.141562 3773 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-42.ec2.internal" on network "default" took: 1.393672ms I0423 19:49:55.228454 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-57.ec2.internal I0423 19:49:55.228489 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-57.ec2.internal: 44.736µs W0423 19:49:55.295401 3773 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0423 19:49:55.295419 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0423 19:49:55.295426 3773 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0423 19:49:55.332169 3773 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pp9dv 08e6d3370a13ba6a583d2d4a5ea975c9561f0c908f4c9b95bae2325ce878f1a2 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pp9dv 08e6d3370a13ba6a583d2d4a5ea975c9561f0c908f4c9b95bae2325ce878f1a2 network default NAD default NAD key ] I0423 19:49:55.343560 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-pp9dv, hostIfaceName: 08e6d3370a13ba6, network: default, NAD default, SandboxID: "08e6d3370a13ba6a583d2d4a5ea975c9561f0c908f4c9b95bae2325ce878f1a2", PCI device ID: , UID: "4b303bd1-fcc3-40fa-b3dc-52661cac016d", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0423 19:49:55.397889 3773 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pp9dv 08e6d3370a13ba6a583d2d4a5ea975c9561f0c908f4c9b95bae2325ce878f1a2 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pp9dv 08e6d3370a13ba6a583d2d4a5ea975c9561f0c908f4c9b95bae2325ce878f1a2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:c4:84:3e:bf:a0\",\"name\":\"08e6d3370a13ba6\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/46c7c17e-411d-461f-9962-b3e00b82a628\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:49:55.697899 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-42.ec2.internal I0423 19:49:55.697907 3773 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-42.ec2.internal in network "default" I0423 19:49:55.697923 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-42.ec2.internal: 31.585µs I0423 19:49:55.697953 3773 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-42.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0423 19:49:55.697967 3773 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-42.ec2.internal", IP: 10.0.2.42 to PMTUD blocking rules I0423 19:49:55.697981 3773 node_tracker.go:169] Node ip-10-0-2-42.ec2.internal switch + router changed, syncing services in network "default" I0423 19:49:55.697999 3773 services_controller.go:574] Full service sync requested for network=default I0423 19:49:55.697909 3773 egressfirewall.go:723] Syncing node "ip-10-0-2-42.ec2.internal" for egress firewall W0423 19:49:55.698170 3773 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:49:55.698319 3773 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0423 19:49:55.698333 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0423 19:49:55.698338 3773 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0423 19:49:55.698581 3773 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0423 19:49:55.705248 3773 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-42.ec2.internal. Updating no re-route policies I0423 19:49:55.717490 3773 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-42.ec2.internal", IP: 10.0.2.42 to PMTUD blocking rules I0423 19:49:55.717506 3773 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-42.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0423 19:49:56.534843 3773 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:49:58.255581 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:49:58.403825 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0423 19:49:58.426745 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0423 19:49:58.984017 3773 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-42.ec2.internal I0423 19:49:58.984045 3773 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-42.ec2.internal: 34.486µs W0423 19:49:59.411479 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:50:00.338724 3773 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:50:00.584420 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:50:01.570351 3773 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0423 19:50:01.570372 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0423 19:50:01.570379 3773 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0423 19:50:04.354346 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0423 19:50:04.354368 3773 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0423 19:50:04.354376 3773 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0423 19:50:04.376541 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:50:04.392433 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0423 19:50:04.392452 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0423 19:50:04.392458 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0423 19:50:06.603761 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:50:06.603787 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:50:10.399419 3773 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:50:10.399436 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:50:10.399441 3773 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:50:10.451335 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:50:10.451359 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:50:10.492912 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:50:10.492938 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:50:12.018324 3773 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-67bd86b949-vvm79 IP address from the namespace address-set, err: pod openshift-console/console-67bd86b949-vvm79: no pod IPs found W0423 19:50:13.416701 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 19:50:13.455083 3773 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:50:13.455104 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:50:14.078885 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:50:14.420674 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0423 19:50:14.519016 3773 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:50:14.519038 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:50:14.519043 3773 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:50:14.519049 3773 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:50:15.513961 3773 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0423 19:50:15.541264 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0423 19:50:17.594804 3773 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-5fdb8fd55-zb4vz IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-5fdb8fd55-zb4vz: no pod IPs found W0423 19:50:17.663512 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:50:17.663532 3773 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:50:17.663538 3773 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:50:18.682687 3773 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:50:18.682709 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0423 19:50:19.211758 3773 cni.go:452] [openshift-image-registry/image-registry-85684f4cf6-78p8f 5dc77e238cb4d79880e16fad3d7dfbfc3e22d0e6e7cb2c318b923b2873e5e1a0 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-85684f4cf6-78p8f 5dc77e238cb4d79880e16fad3d7dfbfc3e22d0e6e7cb2c318b923b2873e5e1a0 network default NAD default NAD key ] I0423 19:50:19.284367 3773 cni.go:473] [openshift-image-registry/image-registry-85684f4cf6-78p8f 5dc77e238cb4d79880e16fad3d7dfbfc3e22d0e6e7cb2c318b923b2873e5e1a0 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-85684f4cf6-78p8f 5dc77e238cb4d79880e16fad3d7dfbfc3e22d0e6e7cb2c318b923b2873e5e1a0 network default NAD default NAD key default], result "{}", err I0423 19:50:19.334450 3773 obj_retry.go:498] Detected object openshift-image-registry/image-registry-85684f4cf6-78p8f of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:50:19.334485 3773 pods.go:176] Deleting pod: openshift-image-registry/image-registry-85684f4cf6-78p8f I0423 19:50:19.335920 3773 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-85684f4cf6-78p8f, ips: 10.128.0.5 I0423 19:50:19.335954 3773 obj_retry.go:498] Detected object openshift-image-registry/image-registry-85684f4cf6-78p8f of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:50:19.504634 3773 obj_retry.go:492] Detected object openshift-image-registry/image-registry-85684f4cf6-78p8f of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:50:19.504666 3773 obj_retry.go:492] Detected object openshift-image-registry/image-registry-85684f4cf6-78p8f of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:50:19.511009 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-85684f4cf6-78p8f I0423 19:50:19.511040 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-85684f4cf6-78p8f I0423 19:50:21.196996 3773 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0423 19:50:24.550722 3773 cni.go:452] [openshift-console/console-788b45bb7-d685x aef80746d7e85d7f9b6fcba84ff6d8abd9db167fb0257120d932b59965e8e114 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-788b45bb7-d685x aef80746d7e85d7f9b6fcba84ff6d8abd9db167fb0257120d932b59965e8e114 network default NAD default NAD key ] I0423 19:50:24.619336 3773 cni.go:473] [openshift-console/console-788b45bb7-d685x aef80746d7e85d7f9b6fcba84ff6d8abd9db167fb0257120d932b59965e8e114 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-788b45bb7-d685x aef80746d7e85d7f9b6fcba84ff6d8abd9db167fb0257120d932b59965e8e114 network default NAD default NAD key default], result "{}", err I0423 19:50:24.669371 3773 obj_retry.go:498] Detected object openshift-console/console-788b45bb7-d685x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:50:24.669409 3773 pods.go:176] Deleting pod: openshift-console/console-788b45bb7-d685x I0423 19:50:24.670743 3773 pods.go:236] Attempting to release IPs for pod: openshift-console/console-788b45bb7-d685x, ips: 10.128.0.24 I0423 19:50:24.670774 3773 obj_retry.go:498] Detected object openshift-console/console-788b45bb7-d685x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:50:24.679416 3773 base_network_controller_pods.go:487] [default/openshift-console/console-5d984bf55d-2r7dh] creating logical port openshift-console_console-5d984bf55d-2r7dh for pod on switch ip-10-0-0-253.ec2.internal I0423 19:50:24.679502 3773 kube.go:256] Updating pod openshift-console/console-5d984bf55d-2r7dh I0423 19:50:24.694435 3773 pod.go:62] [openshift-console/console-5d984bf55d-2r7dh] pod update took 14.958758ms I0423 19:50:24.694456 3773 base_network_controller_pods.go:951] [default/openshift-console/console-5d984bf55d-2r7dh] addLogicalPort annotation time took 14.982646ms I0423 19:50:24.695572 3773 pods.go:271] [openshift-console/console-5d984bf55d-2r7dh] addLogicalPort took 16.166217ms, libovsdb time 778.248µs W0423 19:50:24.782893 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0423 19:50:25.038691 3773 cni.go:452] [openshift-console/console-5d984bf55d-2r7dh 33144ee701e4ef6303204d19aef750db3fadea718ee98051c53a292b7dd6e7ed network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-5d984bf55d-2r7dh 33144ee701e4ef6303204d19aef750db3fadea718ee98051c53a292b7dd6e7ed network default NAD default NAD key ] I0423 19:50:25.047989 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-5d984bf55d-2r7dh, hostIfaceName: 33144ee701e4ef6, network: default, NAD default, SandboxID: "33144ee701e4ef6303204d19aef750db3fadea718ee98051c53a292b7dd6e7ed", PCI device ID: , UID: "61b13f1c-4dd7-409c-a8ff-0ad1700f3fa9", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] W0423 19:50:25.084331 3773 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-5d984bf55d-zplxs IP address from the namespace address-set, err: pod openshift-console/console-5d984bf55d-zplxs: no pod IPs found I0423 19:50:25.100431 3773 cni.go:473] [openshift-console/console-5d984bf55d-2r7dh 33144ee701e4ef6303204d19aef750db3fadea718ee98051c53a292b7dd6e7ed network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-5d984bf55d-2r7dh 33144ee701e4ef6303204d19aef750db3fadea718ee98051c53a292b7dd6e7ed network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:e8:9f:0e:e5:87\",\"name\":\"33144ee701e4ef6\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b76edb53-bdeb-4643-8c46-43e33ef40a81\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:50:25.570024 3773 obj_retry.go:492] Detected object openshift-console/console-788b45bb7-d685x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:50:25.570052 3773 obj_retry.go:492] Detected object openshift-console/console-788b45bb7-d685x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:50:25.581971 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-788b45bb7-d685x I0423 19:50:25.581997 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-788b45bb7-d685x W0423 19:50:32.567689 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0423 19:50:32.567752 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0423 19:50:32.567687 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0423 19:50:33.575510 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0423 19:50:35.800017 3773 obj_retry.go:498] Detected object openshift-console/console-788b45bb7-cbqd9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:50:35.800645 3773 obj_retry.go:498] Detected object openshift-console/console-788b45bb7-cbqd9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:50:35.952741 3773 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390gmk6s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:50:35.953168 3773 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390gmk6s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:50:35.953192 3773 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390gmk6s of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:50:35.953201 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0423 19:50:35.953213 3773 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390gmk6s of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:50:35.953218 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0423 19:50:36.114883 3773 obj_retry.go:492] Detected object openshift-console/console-788b45bb7-cbqd9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:50:36.114907 3773 obj_retry.go:492] Detected object openshift-console/console-788b45bb7-cbqd9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:50:36.124298 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-788b45bb7-cbqd9 I0423 19:50:36.124322 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-788b45bb7-cbqd9 I0423 19:50:36.980746 3773 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390gmk6s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:50:36.980767 3773 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390gmk6s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:50:36.980787 3773 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390gmk6s of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:50:36.980796 3773 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390gmk6s of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:50:41.698651 3773 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0423 19:50:41.708593 3773 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0423 19:50:43.343977 3773 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-tpqzx] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-tpqzx for pod on switch ip-10-0-0-253.ec2.internal I0423 19:50:43.344091 3773 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-tpqzx I0423 19:50:43.360247 3773 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-tpqzx] pod update took 16.190299ms I0423 19:50:43.360268 3773 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-tpqzx] addLogicalPort annotation time took 16.217338ms I0423 19:50:43.361517 3773 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-tpqzx] addLogicalPort took 17.556168ms, libovsdb time 870.623µs I0423 19:50:43.764619 3773 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-tpqzx 66127ac4a055d6d6a409cb5ab1f9ab22080e0277a588a73ed3179aa6ad76be6e network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-tpqzx 66127ac4a055d6d6a409cb5ab1f9ab22080e0277a588a73ed3179aa6ad76be6e network default NAD default NAD key ] I0423 19:50:43.773817 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-tpqzx, hostIfaceName: 66127ac4a055d6d, network: default, NAD default, SandboxID: "66127ac4a055d6d6a409cb5ab1f9ab22080e0277a588a73ed3179aa6ad76be6e", PCI device ID: , UID: "8b7ea9c7-d5af-454e-8b59-523404d60d5b", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0423 19:50:43.826986 3773 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-tpqzx 66127ac4a055d6d6a409cb5ab1f9ab22080e0277a588a73ed3179aa6ad76be6e network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-tpqzx 66127ac4a055d6d6a409cb5ab1f9ab22080e0277a588a73ed3179aa6ad76be6e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:3c:39:d2:d1:16\",\"name\":\"66127ac4a055d6d\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/639230c5-9894-4839-ab84-4b4f606c842b\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 19:50:46.882834 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:50:46.882865 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:50:46.922168 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:50:46.922193 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:50:47.639563 3773 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0423 19:50:47.664196 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0423 19:50:47.664226 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0423 19:50:47.664257 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0423 19:50:48.651103 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0423 19:50:50.689015 3773 cni.go:452] [openshift-console/console-5d984bf55d-2r7dh 33144ee701e4ef6303204d19aef750db3fadea718ee98051c53a292b7dd6e7ed network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-5d984bf55d-2r7dh 33144ee701e4ef6303204d19aef750db3fadea718ee98051c53a292b7dd6e7ed network default NAD default NAD key ] I0423 19:50:50.764892 3773 cni.go:473] [openshift-console/console-5d984bf55d-2r7dh 33144ee701e4ef6303204d19aef750db3fadea718ee98051c53a292b7dd6e7ed network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-5d984bf55d-2r7dh 33144ee701e4ef6303204d19aef750db3fadea718ee98051c53a292b7dd6e7ed network default NAD default NAD key default], result "{}", err I0423 19:50:50.835784 3773 obj_retry.go:498] Detected object openshift-console/console-5d984bf55d-2r7dh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:50:50.835816 3773 pods.go:176] Deleting pod: openshift-console/console-5d984bf55d-2r7dh I0423 19:50:50.836991 3773 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5d984bf55d-2r7dh, ips: 10.128.0.27 I0423 19:50:50.837015 3773 obj_retry.go:498] Detected object openshift-console/console-5d984bf55d-2r7dh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:50:50.845080 3773 base_network_controller_pods.go:487] [default/openshift-console/console-7c8bb6b4ff-kwxs6] creating logical port openshift-console_console-7c8bb6b4ff-kwxs6 for pod on switch ip-10-0-0-253.ec2.internal I0423 19:50:50.845227 3773 kube.go:256] Updating pod openshift-console/console-7c8bb6b4ff-kwxs6 I0423 19:50:50.860814 3773 pod.go:62] [openshift-console/console-7c8bb6b4ff-kwxs6] pod update took 15.615542ms I0423 19:50:50.860834 3773 base_network_controller_pods.go:951] [default/openshift-console/console-7c8bb6b4ff-kwxs6] addLogicalPort annotation time took 15.639168ms I0423 19:50:50.862050 3773 pods.go:271] [openshift-console/console-7c8bb6b4ff-kwxs6] addLogicalPort took 16.983322ms, libovsdb time 782.975µs I0423 19:50:51.266458 3773 cni.go:452] [openshift-console/console-7c8bb6b4ff-kwxs6 635724659aa21af322cc498c0bc3878076022ca57ea9f9b8303f1876ee7c8474 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7c8bb6b4ff-kwxs6 635724659aa21af322cc498c0bc3878076022ca57ea9f9b8303f1876ee7c8474 network default NAD default NAD key ] I0423 19:50:51.276367 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7c8bb6b4ff-kwxs6, hostIfaceName: 635724659aa21af, network: default, NAD default, SandboxID: "635724659aa21af322cc498c0bc3878076022ca57ea9f9b8303f1876ee7c8474", PCI device ID: , UID: "a4ac5d84-72ed-42a5-9069-f468c04d94a5", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0423 19:50:51.327690 3773 cni.go:473] [openshift-console/console-7c8bb6b4ff-kwxs6 635724659aa21af322cc498c0bc3878076022ca57ea9f9b8303f1876ee7c8474 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7c8bb6b4ff-kwxs6 635724659aa21af322cc498c0bc3878076022ca57ea9f9b8303f1876ee7c8474 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:a9:8e:c8:8c:c1\",\"name\":\"635724659aa21af\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67b0531d-5854-4470-b444-a50ee3805b7a\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:50:51.672215 3773 obj_retry.go:492] Detected object openshift-console/console-5d984bf55d-2r7dh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:50:51.672242 3773 obj_retry.go:492] Detected object openshift-console/console-5d984bf55d-2r7dh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:50:51.677688 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5d984bf55d-2r7dh I0423 19:50:51.677710 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5d984bf55d-2r7dh W0423 19:50:52.668378 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0423 19:50:52.668439 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0423 19:50:52.668579 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0423 19:50:53.676348 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0423 19:50:58.650598 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0423 19:51:01.716011 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0423 19:51:07.223294 3773 cni.go:452] [openshift-ingress/router-default-84d49796ff-mkxxq c1075e841d85fc9f6733156eeb0c89360cbc9292048c68c423c13955d1a26f54 network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-84d49796ff-mkxxq c1075e841d85fc9f6733156eeb0c89360cbc9292048c68c423c13955d1a26f54 network default NAD default NAD key ] I0423 19:51:07.315910 3773 cni.go:473] [openshift-ingress/router-default-84d49796ff-mkxxq c1075e841d85fc9f6733156eeb0c89360cbc9292048c68c423c13955d1a26f54 network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-84d49796ff-mkxxq c1075e841d85fc9f6733156eeb0c89360cbc9292048c68c423c13955d1a26f54 network default NAD default NAD key default], result "{}", err I0423 19:51:07.364885 3773 obj_retry.go:498] Detected object openshift-ingress/router-default-84d49796ff-mkxxq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:51:07.364921 3773 pods.go:176] Deleting pod: openshift-ingress/router-default-84d49796ff-mkxxq I0423 19:51:07.366164 3773 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-84d49796ff-mkxxq, ips: 10.128.0.15 I0423 19:51:07.366197 3773 obj_retry.go:498] Detected object openshift-ingress/router-default-84d49796ff-mkxxq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:51:07.733604 3773 obj_retry.go:492] Detected object openshift-ingress/router-default-84d49796ff-mkxxq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:51:07.733625 3773 obj_retry.go:492] Detected object openshift-ingress/router-default-84d49796ff-mkxxq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:51:07.737263 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-84d49796ff-mkxxq I0423 19:51:07.737292 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-84d49796ff-mkxxq I0423 19:51:12.218778 3773 obj_retry.go:498] Detected object openshift-console/console-67bd86b949-gwgww of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:51:12.219352 3773 obj_retry.go:498] Detected object openshift-console/console-67bd86b949-gwgww of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:51:12.666021 3773 obj_retry.go:492] Detected object openshift-console/console-67bd86b949-gwgww of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:51:12.666067 3773 obj_retry.go:492] Detected object openshift-console/console-67bd86b949-gwgww of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:51:12.672311 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-67bd86b949-gwgww I0423 19:51:12.672348 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-67bd86b949-gwgww I0423 19:51:34.472167 3773 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0423 19:51:35.905843 3773 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616230-vs6sb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:51:35.906430 3773 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616230-vs6sb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:51:36.784006 3773 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616230-vs6sb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:51:36.784028 3773 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616230-vs6sb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:51:50.613091 3773 cni.go:452] [openshift-console/console-7c8bb6b4ff-kwxs6 635724659aa21af322cc498c0bc3878076022ca57ea9f9b8303f1876ee7c8474 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7c8bb6b4ff-kwxs6 635724659aa21af322cc498c0bc3878076022ca57ea9f9b8303f1876ee7c8474 network default NAD default NAD key ] I0423 19:51:50.692205 3773 cni.go:473] [openshift-console/console-7c8bb6b4ff-kwxs6 635724659aa21af322cc498c0bc3878076022ca57ea9f9b8303f1876ee7c8474 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7c8bb6b4ff-kwxs6 635724659aa21af322cc498c0bc3878076022ca57ea9f9b8303f1876ee7c8474 network default NAD default NAD key default], result "{}", err I0423 19:51:50.751942 3773 obj_retry.go:498] Detected object openshift-console/console-7c8bb6b4ff-kwxs6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:51:50.751979 3773 pods.go:176] Deleting pod: openshift-console/console-7c8bb6b4ff-kwxs6 I0423 19:51:50.753233 3773 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7c8bb6b4ff-kwxs6, ips: 10.128.0.29 I0423 19:51:50.753261 3773 obj_retry.go:498] Detected object openshift-console/console-7c8bb6b4ff-kwxs6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:51:50.762217 3773 base_network_controller_pods.go:487] [default/openshift-console/console-5596545ff-glx6z] creating logical port openshift-console_console-5596545ff-glx6z for pod on switch ip-10-0-0-253.ec2.internal I0423 19:51:50.762340 3773 kube.go:256] Updating pod openshift-console/console-5596545ff-glx6z I0423 19:51:50.776524 3773 pod.go:62] [openshift-console/console-5596545ff-glx6z] pod update took 14.209177ms I0423 19:51:50.776545 3773 base_network_controller_pods.go:951] [default/openshift-console/console-5596545ff-glx6z] addLogicalPort annotation time took 14.233555ms I0423 19:51:50.777684 3773 pods.go:271] [openshift-console/console-5596545ff-glx6z] addLogicalPort took 15.481414ms, libovsdb time 783.814µs I0423 19:51:50.916400 3773 obj_retry.go:492] Detected object openshift-console/console-7c8bb6b4ff-kwxs6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:51:50.916420 3773 obj_retry.go:492] Detected object openshift-console/console-7c8bb6b4ff-kwxs6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:51:50.922838 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7c8bb6b4ff-kwxs6 I0423 19:51:50.922868 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7c8bb6b4ff-kwxs6 I0423 19:51:51.120225 3773 cni.go:452] [openshift-console/console-5596545ff-glx6z 798a9d153f033cdf4baa1a8c466a361b67379dae7a8da145f533f2a0f1e94de0 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-5596545ff-glx6z 798a9d153f033cdf4baa1a8c466a361b67379dae7a8da145f533f2a0f1e94de0 network default NAD default NAD key ] I0423 19:51:51.129539 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-5596545ff-glx6z, hostIfaceName: 798a9d153f033cd, network: default, NAD default, SandboxID: "798a9d153f033cdf4baa1a8c466a361b67379dae7a8da145f533f2a0f1e94de0", PCI device ID: , UID: "77a0b56a-7262-4a9b-b131-0332ef2f9b40", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0423 19:51:51.176704 3773 cni.go:473] [openshift-console/console-5596545ff-glx6z 798a9d153f033cdf4baa1a8c466a361b67379dae7a8da145f533f2a0f1e94de0 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-5596545ff-glx6z 798a9d153f033cdf4baa1a8c466a361b67379dae7a8da145f533f2a0f1e94de0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:3b:2c:93:a8:cc\",\"name\":\"798a9d153f033cd\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a8728d54-b991-4a0d-85ac-dc22c10a0a4d\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 19:52:01.980232 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0423 19:52:02.123129 3773 obj_retry.go:498] Detected object openshift-console/console-7c8bb6b4ff-xwqmb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:52:02.123664 3773 obj_retry.go:498] Detected object openshift-console/console-7c8bb6b4ff-xwqmb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:52:02.484626 3773 obj_retry.go:492] Detected object openshift-console/console-7c8bb6b4ff-xwqmb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:52:02.484645 3773 obj_retry.go:492] Detected object openshift-console/console-7c8bb6b4ff-xwqmb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:52:02.489994 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7c8bb6b4ff-xwqmb I0423 19:52:02.490008 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7c8bb6b4ff-xwqmb I0423 19:54:30.557846 3773 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=5 I0423 19:55:03.084543 3773 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0423 19:55:03.685402 3773 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=5 I0423 19:55:10.512199 3773 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=5 I0423 19:55:10.563452 3773 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0423 19:55:41.666489 3773 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0423 19:55:58.697315 3773 namespace.go:98] [tsf] adding namespace I0423 19:55:58.698326 3773 namespace.go:102] [tsf] adding namespace took 988.245µs I0423 19:55:58.713310 3773 namespace.go:142] [tsf] updating namespace I0423 19:55:58.721744 3773 namespace.go:142] [tsf] updating namespace I0423 19:56:02.480916 3773 namespace.go:98] [tsf-tas] adding namespace I0423 19:56:02.481826 3773 namespace.go:102] [tsf-tas] adding namespace took 885.391µs I0423 19:56:02.486119 3773 namespace.go:98] [konflux-ui] adding namespace I0423 19:56:02.486202 3773 namespace.go:98] [konflux-operator] adding namespace I0423 19:56:02.486299 3773 namespace.go:98] [rhbk-operator] adding namespace I0423 19:56:02.486388 3773 namespace.go:98] [cert-manager-operator] adding namespace I0423 19:56:02.486846 3773 namespace.go:102] [konflux-ui] adding namespace took 712.099µs I0423 19:56:02.487459 3773 namespace.go:102] [konflux-operator] adding namespace took 1.23739ms I0423 19:56:02.487601 3773 namespace.go:98] [rhtpa-operator] adding namespace I0423 19:56:02.487701 3773 namespace.go:98] [tsf-keycloak] adding namespace I0423 19:56:02.488165 3773 namespace.go:102] [rhbk-operator] adding namespace took 1.84705ms I0423 19:56:02.488806 3773 namespace.go:102] [cert-manager-operator] adding namespace took 2.401415ms I0423 19:56:02.489572 3773 namespace.go:102] [rhtpa-operator] adding namespace took 1.953772ms I0423 19:56:02.490255 3773 namespace.go:102] [tsf-keycloak] adding namespace took 2.537622ms I0423 19:56:02.494713 3773 namespace.go:98] [tsf-tpa] adding namespace I0423 19:56:02.495332 3773 namespace.go:102] [tsf-tpa] adding namespace took 606.417µs I0423 19:56:02.499927 3773 namespace.go:142] [tsf-tas] updating namespace I0423 19:56:02.513710 3773 namespace.go:142] [tsf-tas] updating namespace I0423 19:56:02.522871 3773 namespace.go:142] [konflux-operator] updating namespace I0423 19:56:02.523037 3773 namespace.go:142] [cert-manager-operator] updating namespace I0423 19:56:02.532192 3773 namespace.go:142] [konflux-ui] updating namespace I0423 19:56:02.532332 3773 namespace.go:142] [rhbk-operator] updating namespace I0423 19:56:02.532375 3773 namespace.go:142] [tsf-keycloak] updating namespace I0423 19:56:02.538403 3773 namespace.go:142] [rhtpa-operator] updating namespace I0423 19:56:02.539309 3773 namespace.go:142] [tsf-tpa] updating namespace I0423 19:56:02.547529 3773 namespace.go:142] [konflux-ui] updating namespace I0423 19:56:02.548088 3773 namespace.go:142] [tsf-tas] updating namespace I0423 19:56:02.566791 3773 namespace.go:142] [konflux-ui] updating namespace I0423 19:56:02.566948 3773 namespace.go:142] [konflux-operator] updating namespace I0423 19:56:02.586820 3773 namespace.go:142] [konflux-operator] updating namespace I0423 19:56:02.586949 3773 namespace.go:142] [rhbk-operator] updating namespace I0423 19:56:02.606840 3773 namespace.go:142] [cert-manager-operator] updating namespace I0423 19:56:02.606912 3773 namespace.go:142] [rhbk-operator] updating namespace I0423 19:56:02.620199 3773 namespace.go:142] [cert-manager-operator] updating namespace I0423 19:56:02.626935 3773 namespace.go:142] [rhtpa-operator] updating namespace I0423 19:56:02.639737 3773 namespace.go:142] [rhtpa-operator] updating namespace I0423 19:56:02.647823 3773 namespace.go:142] [tsf-keycloak] updating namespace I0423 19:56:02.663184 3773 namespace.go:142] [tsf-keycloak] updating namespace I0423 19:56:02.666735 3773 namespace.go:142] [tsf-tpa] updating namespace I0423 19:56:02.690793 3773 namespace.go:142] [tsf-tpa] updating namespace I0423 19:56:03.557576 3773 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=71 I0423 19:56:03.966159 3773 namespace.go:142] [tsf-keycloak] updating namespace I0423 19:56:03.973327 3773 namespace.go:142] [tsf-tpa] updating namespace I0423 19:56:09.315736 3773 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158282xd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:56:09.316317 3773 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158282xd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:56:09.316344 3773 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158282xd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:56:09.316352 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0423 19:56:09.316362 3773 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158282xd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:56:09.316368 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0423 19:56:10.339475 3773 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158282xd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:10.339501 3773 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158282xd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:10.339517 3773 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158282xd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:10.339528 3773 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158282xd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:11.330684 3773 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbz6n8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:56:11.331229 3773 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbz6n8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:56:11.331255 3773 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbz6n8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:56:11.331262 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0423 19:56:11.331272 3773 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbz6n8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:56:11.331275 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0423 19:56:12.362813 3773 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbz6n8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:12.362835 3773 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbz6n8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:12.362845 3773 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbz6n8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:12.362854 3773 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbz6n8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:15.367848 3773 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tjlnp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:56:15.368368 3773 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tjlnp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:56:15.368398 3773 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tjlnp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:56:15.368430 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0423 19:56:15.368444 3773 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tjlnp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:56:15.368449 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0423 19:56:15.488284 3773 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b515qrf8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:56:15.488683 3773 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b515qrf8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:56:15.488707 3773 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b515qrf8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:56:15.488714 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0423 19:56:15.488724 3773 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b515qrf8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:56:15.488729 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0423 19:56:16.386924 3773 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kvtp8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:56:16.387438 3773 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kvtp8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:56:16.387464 3773 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kvtp8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:56:16.387472 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0423 19:56:16.387488 3773 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kvtp8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:56:16.387494 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0423 19:56:16.394541 3773 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tjlnp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:16.394556 3773 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tjlnp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:16.394569 3773 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tjlnp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:16.394577 3773 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tjlnp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:16.512257 3773 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b515qrf8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:16.512280 3773 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b515qrf8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:16.512295 3773 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b515qrf8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:16.512306 3773 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b515qrf8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:17.402972 3773 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136fwbp4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:56:17.403485 3773 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136fwbp4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:56:17.403510 3773 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136fwbp4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:56:17.403518 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0423 19:56:17.403527 3773 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136fwbp4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:56:17.403532 3773 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0423 19:56:17.411979 3773 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kvtp8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:17.411998 3773 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kvtp8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:17.412012 3773 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kvtp8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:17.412024 3773 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kvtp8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:18.430968 3773 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136fwbp4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:18.430991 3773 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136fwbp4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:18.431007 3773 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136fwbp4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:18.431016 3773 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136fwbp4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:56:23.734110 3773 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-cv8gn] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-5b66478c58-cv8gn for pod on switch ip-10-0-0-253.ec2.internal I0423 19:56:23.734262 3773 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-cv8gn I0423 19:56:23.752012 3773 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-cv8gn] pod update took 17.777694ms I0423 19:56:23.752036 3773 base_network_controller_pods.go:951] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-cv8gn] addLogicalPort annotation time took 17.804039ms I0423 19:56:23.753311 3773 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-cv8gn] addLogicalPort took 19.218825ms, libovsdb time 914.776µs I0423 19:56:24.094525 3773 cni.go:452] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-cv8gn c7b7bc750053b6e03899a4f20f36e9ae7c38318f96c17dfed6a1b83c0e6c6dfb network default NAD default NAD key ] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-cv8gn c7b7bc750053b6e03899a4f20f36e9ae7c38318f96c17dfed6a1b83c0e6c6dfb network default NAD default NAD key ] I0423 19:56:24.103656 3773 helper_linux.go:556] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-5b66478c58-cv8gn, hostIfaceName: c7b7bc750053b6e, network: default, NAD default, SandboxID: "c7b7bc750053b6e03899a4f20f36e9ae7c38318f96c17dfed6a1b83c0e6c6dfb", PCI device ID: , UID: "7a354270-8b06-4c6a-88cc-603d7f756b90", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0423 19:56:24.153472 3773 cni.go:473] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-cv8gn c7b7bc750053b6e03899a4f20f36e9ae7c38318f96c17dfed6a1b83c0e6c6dfb network default NAD default NAD key default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-cv8gn c7b7bc750053b6e03899a4f20f36e9ae7c38318f96c17dfed6a1b83c0e6c6dfb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:3c:43:3d:ca:30\",\"name\":\"c7b7bc750053b6e\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b63104d9-afe6-4e17-9f08-f0be5eb05d87\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:56:24.772683 3773 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0423 19:56:26.984079 3773 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-66b879f574-4dlc6] creating logical port konflux-operator_konflux-operator-controller-manager-66b879f574-4dlc6 for pod on switch ip-10-0-0-253.ec2.internal I0423 19:56:26.984208 3773 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-66b879f574-4dlc6 I0423 19:56:27.007034 3773 pod.go:62] [konflux-operator/konflux-operator-controller-manager-66b879f574-4dlc6] pod update took 22.853671ms I0423 19:56:27.007056 3773 base_network_controller_pods.go:951] [default/konflux-operator/konflux-operator-controller-manager-66b879f574-4dlc6] addLogicalPort annotation time took 22.879302ms I0423 19:56:27.008231 3773 pods.go:271] [konflux-operator/konflux-operator-controller-manager-66b879f574-4dlc6] addLogicalPort took 24.171898ms, libovsdb time 832.695µs I0423 19:56:27.232848 3773 namespace.go:98] [cert-manager] adding namespace I0423 19:56:27.233806 3773 namespace.go:102] [cert-manager] adding namespace took 932.827µs I0423 19:56:27.267377 3773 namespace.go:142] [cert-manager] updating namespace I0423 19:56:27.293406 3773 namespace.go:142] [cert-manager] updating namespace I0423 19:56:27.353243 3773 cni.go:452] [konflux-operator/konflux-operator-controller-manager-66b879f574-4dlc6 85644c93cf9b03070d0005c512b7d9fc2161a4d5e3d2396f3e1f43649f9b4753 network default NAD default NAD key ] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-66b879f574-4dlc6 85644c93cf9b03070d0005c512b7d9fc2161a4d5e3d2396f3e1f43649f9b4753 network default NAD default NAD key ] I0423 19:56:27.363255 3773 helper_linux.go:556] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-66b879f574-4dlc6, hostIfaceName: 85644c93cf9b030, network: default, NAD default, SandboxID: "85644c93cf9b03070d0005c512b7d9fc2161a4d5e3d2396f3e1f43649f9b4753", PCI device ID: , UID: "75691324-fbdf-4b57-b6e6-93c4e1d78e7c", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0423 19:56:27.412903 3773 cni.go:473] [konflux-operator/konflux-operator-controller-manager-66b879f574-4dlc6 85644c93cf9b03070d0005c512b7d9fc2161a4d5e3d2396f3e1f43649f9b4753 network default NAD default NAD key default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-66b879f574-4dlc6 85644c93cf9b03070d0005c512b7d9fc2161a4d5e3d2396f3e1f43649f9b4753 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:92:e9:86:e3:74\",\"name\":\"85644c93cf9b030\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1304856d-0656-43e1-ad6d-4c079d514292\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:56:27.673927 3773 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-fb86cd8d9-pqtml] creating logical port rhbk-operator_rhbk-operator-fb86cd8d9-pqtml for pod on switch ip-10-0-0-253.ec2.internal I0423 19:56:27.674038 3773 kube.go:256] Updating pod rhbk-operator/rhbk-operator-fb86cd8d9-pqtml I0423 19:56:27.702468 3773 pod.go:62] [rhbk-operator/rhbk-operator-fb86cd8d9-pqtml] pod update took 28.45581ms I0423 19:56:27.702498 3773 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-fb86cd8d9-pqtml] addLogicalPort annotation time took 28.489461ms I0423 19:56:27.703783 3773 pods.go:271] [rhbk-operator/rhbk-operator-fb86cd8d9-pqtml] addLogicalPort took 29.874799ms, libovsdb time 852.008µs I0423 19:56:28.037935 3773 cni.go:452] [rhbk-operator/rhbk-operator-fb86cd8d9-pqtml 3122229fe172a54c1401bf962e5670d96fa3285071ad3ffa33b7b787f56c3570 network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-pqtml 3122229fe172a54c1401bf962e5670d96fa3285071ad3ffa33b7b787f56c3570 network default NAD default NAD key ] I0423 19:56:28.047553 3773 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-fb86cd8d9-pqtml, hostIfaceName: 3122229fe172a54, network: default, NAD default, SandboxID: "3122229fe172a54c1401bf962e5670d96fa3285071ad3ffa33b7b787f56c3570", PCI device ID: , UID: "027dd319-cdec-4f86-8a14-de92b0e63735", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0423 19:56:28.095814 3773 cni.go:473] [rhbk-operator/rhbk-operator-fb86cd8d9-pqtml 3122229fe172a54c1401bf962e5670d96fa3285071ad3ffa33b7b787f56c3570 network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-pqtml 3122229fe172a54c1401bf962e5670d96fa3285071ad3ffa33b7b787f56c3570 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:cc:4d:0e:28:45\",\"name\":\"3122229fe172a54\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d849686e-8349-4be8-ae60-6bd4a662264b\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 19:56:31.155240 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0423 19:56:32.050628 3773 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-c7687] creating logical port tsf-tpa_rhtpa-operator-controller-manager-586cbb496b-c7687 for pod on switch ip-10-0-0-253.ec2.internal I0423 19:56:32.050742 3773 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-c7687 I0423 19:56:32.115461 3773 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-c7687] pod update took 64.747051ms I0423 19:56:32.115489 3773 base_network_controller_pods.go:951] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-c7687] addLogicalPort annotation time took 64.776423ms I0423 19:56:32.116885 3773 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-c7687] addLogicalPort took 66.27659ms, libovsdb time 916.472µs I0423 19:56:32.195428 3773 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 W0423 19:56:32.426031 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0423 19:56:32.426049 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0423 19:56:34.482513 3773 cni.go:452] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-c7687 673c4bd482c83633f321ab6a52e64f049ef28510ec17f5574e8a86a6d849cc29 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-c7687 673c4bd482c83633f321ab6a52e64f049ef28510ec17f5574e8a86a6d849cc29 network default NAD default NAD key ] I0423 19:56:34.493865 3773 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-586cbb496b-c7687, hostIfaceName: 673c4bd482c8363, network: default, NAD default, SandboxID: "673c4bd482c83633f321ab6a52e64f049ef28510ec17f5574e8a86a6d849cc29", PCI device ID: , UID: "94a9e829-f52c-4c50-aec0-bb799e393d4b", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0423 19:56:34.543697 3773 cni.go:473] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-c7687 673c4bd482c83633f321ab6a52e64f049ef28510ec17f5574e8a86a6d849cc29 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-c7687 673c4bd482c83633f321ab6a52e64f049ef28510ec17f5574e8a86a6d849cc29 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:7d:aa:b6:b7:dc\",\"name\":\"673c4bd482c8363\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1f180175-106b-43cf-a722-2d5881f791bf\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 19:56:34.742847 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0423 19:56:34.753196 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0423 19:56:34.766882 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0423 19:56:34.773623 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0423 19:56:37.086742 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0423 19:56:38.958226 3773 namespace.go:142] [openshift-operators] updating namespace I0423 19:56:38.968729 3773 namespace.go:142] [openshift-operators] updating namespace W0423 19:56:42.075876 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0423 19:56:45.934627 3773 namespace.go:142] [openshift-operators] updating namespace I0423 19:56:46.281099 3773 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-5864bd5898-gsvjp] creating logical port openshift-operators_tekton-operator-webhook-5864bd5898-gsvjp for pod on switch ip-10-0-0-253.ec2.internal I0423 19:56:46.281235 3773 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-5864bd5898-gsvjp I0423 19:56:46.304820 3773 pod.go:62] [openshift-operators/tekton-operator-webhook-5864bd5898-gsvjp] pod update took 23.610691ms I0423 19:56:46.304846 3773 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-5864bd5898-gsvjp] addLogicalPort annotation time took 23.640667ms I0423 19:56:46.306058 3773 pods.go:271] [openshift-operators/tekton-operator-webhook-5864bd5898-gsvjp] addLogicalPort took 24.974571ms, libovsdb time 790.749µs I0423 19:56:46.656876 3773 cni.go:452] [openshift-operators/tekton-operator-webhook-5864bd5898-gsvjp 13f708b42b91d2bf88717c32a984041673607376d5e7f1d1569aa4d2e8e7d612 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-gsvjp 13f708b42b91d2bf88717c32a984041673607376d5e7f1d1569aa4d2e8e7d612 network default NAD default NAD key ] I0423 19:56:46.666921 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-5864bd5898-gsvjp, hostIfaceName: 13f708b42b91d2b, network: default, NAD default, SandboxID: "13f708b42b91d2bf88717c32a984041673607376d5e7f1d1569aa4d2e8e7d612", PCI device ID: , UID: "fda11d22-fc9e-4a00-8e8e-d3b5ac3bd02c", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0423 19:56:46.925544 3773 cni.go:473] [openshift-operators/tekton-operator-webhook-5864bd5898-gsvjp 13f708b42b91d2bf88717c32a984041673607376d5e7f1d1569aa4d2e8e7d612 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-gsvjp 13f708b42b91d2bf88717c32a984041673607376d5e7f1d1569aa4d2e8e7d612 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:cf:1e:f7:6b:7e\",\"name\":\"13f708b42b91d2b\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/52985fa1-692b-4c29-be65-78e39cf01731\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 19:56:48.120104 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0423 19:56:51.477118 3773 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-s4mpk] creating logical port openshift-operators_rhtas-operator-controller-manager-68888cb75-s4mpk for pod on switch ip-10-0-0-253.ec2.internal I0423 19:56:51.477249 3773 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-68888cb75-s4mpk I0423 19:56:51.505459 3773 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-68888cb75-s4mpk] pod update took 28.236383ms I0423 19:56:51.505482 3773 base_network_controller_pods.go:951] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-s4mpk] addLogicalPort annotation time took 28.263522ms I0423 19:56:51.506725 3773 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-68888cb75-s4mpk] addLogicalPort took 29.624902ms, libovsdb time 861.126µs I0423 19:56:51.866291 3773 cni.go:452] [openshift-operators/rhtas-operator-controller-manager-68888cb75-s4mpk bb37ffb1fb234622dc8a683763f14ba0ae2b6db9b3806f25ab4171dfa9ea120f network default NAD default NAD key ] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-s4mpk bb37ffb1fb234622dc8a683763f14ba0ae2b6db9b3806f25ab4171dfa9ea120f network default NAD default NAD key ] I0423 19:56:51.889810 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-68888cb75-s4mpk, hostIfaceName: bb37ffb1fb23462, network: default, NAD default, SandboxID: "bb37ffb1fb234622dc8a683763f14ba0ae2b6db9b3806f25ab4171dfa9ea120f", PCI device ID: , UID: "5fb0d25b-3226-401b-8659-f33dc8201bfc", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0423 19:56:51.941175 3773 cni.go:473] [openshift-operators/rhtas-operator-controller-manager-68888cb75-s4mpk bb37ffb1fb234622dc8a683763f14ba0ae2b6db9b3806f25ab4171dfa9ea120f network default NAD default NAD key default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-s4mpk bb37ffb1fb234622dc8a683763f14ba0ae2b6db9b3806f25ab4171dfa9ea120f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:b2:ae:96:ac:77\",\"name\":\"bb37ffb1fb23462\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1e8f0c2c-b948-42b5-b18a-94482677e9b6\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:56:53.347827 3773 namespace.go:98] [trusted-artifact-signer] adding namespace I0423 19:56:53.348756 3773 namespace.go:102] [trusted-artifact-signer] adding namespace took 898.925µs I0423 19:56:53.398787 3773 namespace.go:142] [trusted-artifact-signer] updating namespace I0423 19:56:53.405930 3773 namespace.go:142] [trusted-artifact-signer] updating namespace W0423 19:56:54.168879 3773 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0423 19:57:01.171787 3773 namespace.go:98] [openshift-pipelines] adding namespace I0423 19:57:01.172658 3773 namespace.go:102] [openshift-pipelines] adding namespace took 848.432µs I0423 19:57:01.191788 3773 namespace.go:142] [openshift-pipelines] updating namespace I0423 19:57:01.874295 3773 namespace.go:142] [cert-manager] updating namespace I0423 19:57:02.080269 3773 namespace.go:142] [cert-manager-operator] updating namespace I0423 19:57:02.483112 3773 namespace.go:142] [konflux-operator] updating namespace I0423 19:57:02.683057 3773 namespace.go:142] [konflux-ui] updating namespace I0423 19:57:02.884258 3773 namespace.go:142] [open-cluster-management-2ps2c5s4oqe20tlffn1rjju2utl40p4b] updating namespace I0423 19:57:03.282185 3773 namespace.go:142] [rhbk-operator] updating namespace I0423 19:57:03.482556 3773 namespace.go:142] [rhtpa-operator] updating namespace I0423 19:57:03.682401 3773 namespace.go:142] [trusted-artifact-signer] updating namespace I0423 19:57:03.882343 3773 namespace.go:142] [tsf] updating namespace I0423 19:57:04.085844 3773 namespace.go:142] [tsf-keycloak] updating namespace I0423 19:57:04.284355 3773 namespace.go:142] [tsf-tas] updating namespace I0423 19:57:04.484906 3773 namespace.go:142] [tsf-tpa] updating namespace I0423 19:57:04.612976 3773 namespace.go:142] [cert-manager] updating namespace I0423 19:57:04.623925 3773 namespace.go:142] [cert-manager-operator] updating namespace I0423 19:57:04.631408 3773 namespace.go:142] [default] updating namespace I0423 19:57:04.642059 3773 namespace.go:142] [konflux-operator] updating namespace I0423 19:57:04.650785 3773 namespace.go:142] [konflux-ui] updating namespace I0423 19:57:04.660254 3773 namespace.go:142] [open-cluster-management-2ps2c5s4oqe20tlffn1rjju2utl40p4b] updating namespace I0423 19:57:04.667890 3773 namespace.go:142] [openshift] updating namespace I0423 19:57:04.681974 3773 namespace.go:142] [rhbk-operator] updating namespace I0423 19:57:04.693359 3773 namespace.go:142] [rhtpa-operator] updating namespace I0423 19:57:04.700967 3773 namespace.go:142] [trusted-artifact-signer] updating namespace I0423 19:57:04.711173 3773 namespace.go:142] [tsf] updating namespace I0423 19:57:04.729677 3773 namespace.go:142] [tsf-keycloak] updating namespace I0423 19:57:04.743887 3773 namespace.go:142] [tsf-tas] updating namespace I0423 19:57:04.765220 3773 namespace.go:142] [tsf-tpa] updating namespace I0423 19:57:04.867227 3773 namespace.go:142] [cert-manager] updating namespace I0423 19:57:04.967417 3773 namespace.go:142] [cert-manager-operator] updating namespace I0423 19:57:05.067336 3773 namespace.go:142] [default] updating namespace W0423 19:57:05.165108 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0423 19:57:05.169694 3773 namespace.go:142] [konflux-operator] updating namespace I0423 19:57:05.268185 3773 namespace.go:142] [konflux-ui] updating namespace I0423 19:57:05.368882 3773 namespace.go:142] [open-cluster-management-2ps2c5s4oqe20tlffn1rjju2utl40p4b] updating namespace I0423 19:57:05.471157 3773 namespace.go:142] [openshift] updating namespace I0423 19:57:05.567929 3773 namespace.go:142] [rhbk-operator] updating namespace I0423 19:57:05.687801 3773 namespace.go:142] [rhtpa-operator] updating namespace I0423 19:57:05.813544 3773 namespace.go:142] [trusted-artifact-signer] updating namespace I0423 19:57:06.201884 3773 namespace.go:142] [tsf] updating namespace I0423 19:57:06.494551 3773 namespace.go:142] [tsf-keycloak] updating namespace I0423 19:57:06.719225 3773 namespace.go:142] [tsf-tas] updating namespace I0423 19:57:06.931163 3773 namespace.go:142] [tsf-tpa] updating namespace I0423 19:57:07.210030 3773 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:57:07.210598 3773 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:57:07.235288 3773 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:57:07.235306 3773 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:57:07.240954 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0423 19:57:07.240980 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0423 19:57:07.478629 3773 namespace.go:142] [openshift-pipelines] updating namespace I0423 19:57:09.439318 3773 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0423 19:57:12.779171 3773 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-ncs6h] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-ncs6h for pod on switch ip-10-0-0-253.ec2.internal I0423 19:57:12.779268 3773 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-ncs6h I0423 19:57:12.793857 3773 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-ncs6h] pod update took 14.612324ms I0423 19:57:12.793919 3773 base_network_controller_pods.go:951] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-ncs6h] addLogicalPort annotation time took 14.679378ms I0423 19:57:12.795174 3773 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-ncs6h] addLogicalPort took 16.035441ms, libovsdb time 858.337µs I0423 19:57:15.601332 3773 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-fkvr7] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-57f84cff6f-fkvr7 for pod on switch ip-10-0-0-253.ec2.internal I0423 19:57:15.601414 3773 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-fkvr7 I0423 19:57:15.614952 3773 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-fkvr7] pod update took 13.557194ms I0423 19:57:15.614988 3773 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-fkvr7] addLogicalPort annotation time took 13.598528ms I0423 19:57:15.616453 3773 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-fkvr7] addLogicalPort took 15.142447ms, libovsdb time 1.037825ms I0423 19:57:15.962318 3773 cni.go:452] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-fkvr7 1a0268f71966adb3de944de267ba05ac83fe6f5d8eacf21e110329d7a6fab2d7 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-fkvr7 1a0268f71966adb3de944de267ba05ac83fe6f5d8eacf21e110329d7a6fab2d7 network default NAD default NAD key ] I0423 19:57:15.971840 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-57f84cff6f-fkvr7, hostIfaceName: 1a0268f71966adb, network: default, NAD default, SandboxID: "1a0268f71966adb3de944de267ba05ac83fe6f5d8eacf21e110329d7a6fab2d7", PCI device ID: , UID: "1279d084-d8f5-4eb0-bdc5-b4188120e448", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0423 19:57:16.021611 3773 cni.go:473] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-fkvr7 1a0268f71966adb3de944de267ba05ac83fe6f5d8eacf21e110329d7a6fab2d7 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-fkvr7 1a0268f71966adb3de944de267ba05ac83fe6f5d8eacf21e110329d7a6fab2d7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:ad:51:a8:01:45\",\"name\":\"1a0268f71966adb\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5eb55edf-b747-4442-b7a7-089993fa7ebf\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:57:16.557035 3773 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=157 W0423 19:57:20.654249 3773 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0423 19:57:20.654271 3773 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0423 19:57:20.654278 3773 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0423 19:57:20.654284 3773 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0423 19:57:21.152257 3773 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0423 19:57:21.152280 3773 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0423 19:57:21.152286 3773 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0423 19:57:21.210846 3773 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0423 19:57:21.210862 3773 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0423 19:57:21.210868 3773 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0423 19:57:21.245892 3773 cni.go:452] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-ncs6h 3f812df868032a0fa1d321eb94def431fb811d8c166886221ad5740917549024 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-ncs6h 3f812df868032a0fa1d321eb94def431fb811d8c166886221ad5740917549024 network default NAD default NAD key ] I0423 19:57:21.255494 3773 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-ncs6h, hostIfaceName: 3f812df868032a0, network: default, NAD default, SandboxID: "3f812df868032a0fa1d321eb94def431fb811d8c166886221ad5740917549024", PCI device ID: , UID: "56505ae9-ad89-4a1b-b8cf-df44faf1193f", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0423 19:57:21.306313 3773 cni.go:473] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-ncs6h 3f812df868032a0fa1d321eb94def431fb811d8c166886221ad5740917549024 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-ncs6h 3f812df868032a0fa1d321eb94def431fb811d8c166886221ad5740917549024 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:a8:68:67:c1:b2\",\"name\":\"3f812df868032a0\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/474f67b9-2e4c-4a6d-ace3-b68c71ac9d10\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 19:57:25.148431 3773 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0423 19:57:27.312520 3773 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0423 19:57:27.873633 3773 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 W0423 19:57:42.708909 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0423 19:57:48.329469 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0423 19:57:51.419343 3773 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:57:51.419891 3773 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:57:51.438453 3773 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:57:51.438470 3773 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:57:51.445319 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0423 19:57:51.445339 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0423 19:57:53.180951 3773 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0423 19:57:53.181727 3773 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0423 19:57:53.181759 3773 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0423 19:57:53.183901 3773 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0423 19:57:53.547917 3773 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-0-253.ec2.internal I0423 19:57:53.548040 3773 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0423 19:57:53.567830 3773 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 19.818173ms I0423 19:57:53.567859 3773 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 19.853031ms I0423 19:57:53.569315 3773 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 21.420164ms, libovsdb time 904.284µs I0423 19:57:53.909230 3773 cni.go:452] [tsf-keycloak/keycloak-0 580d2772ef5c6f46c1078f9125668869ee01dee053c71f8e679ab09678c06729 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-0 580d2772ef5c6f46c1078f9125668869ee01dee053c71f8e679ab09678c06729 network default NAD default NAD key ] I0423 19:57:53.918481 3773 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 580d2772ef5c6f4, network: default, NAD default, SandboxID: "580d2772ef5c6f46c1078f9125668869ee01dee053c71f8e679ab09678c06729", PCI device ID: , UID: "491b4f77-6104-464a-acea-822769f66bd4", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0423 19:57:53.969730 3773 cni.go:473] [tsf-keycloak/keycloak-0 580d2772ef5c6f46c1078f9125668869ee01dee053c71f8e679ab09678c06729 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-0 580d2772ef5c6f46c1078f9125668869ee01dee053c71f8e679ab09678c06729 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:1b:da:56:28:74\",\"name\":\"580d2772ef5c6f4\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/39f77a91-d7de-4308-a7c3-17b16d613cdc\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:57:59.706690 3773 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0423 19:58:00.480562 3773 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0423 19:58:00.480587 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0423 19:58:00.485218 3773 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0423 19:58:00.485243 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0423 19:58:21.952511 3773 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0423 19:58:21.952533 3773 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0423 19:58:21.952539 3773 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0423 19:58:26.135914 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0423 19:58:26.135933 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0423 19:58:26.142238 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 9000: empty IP address endpoints for service tsf-keycloak/keycloak-service W0423 19:58:26.142256 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0423 19:58:33.427555 3773 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-tkhpq] creating logical port tsf-keycloak_tsf-iam-tkhpq for pod on switch ip-10-0-0-253.ec2.internal I0423 19:58:33.427648 3773 kube.go:256] Updating pod tsf-keycloak/tsf-iam-tkhpq I0423 19:58:33.440715 3773 pod.go:62] [tsf-keycloak/tsf-iam-tkhpq] pod update took 13.092537ms I0423 19:58:33.440738 3773 base_network_controller_pods.go:951] [default/tsf-keycloak/tsf-iam-tkhpq] addLogicalPort annotation time took 13.121867ms I0423 19:58:33.442171 3773 pods.go:271] [tsf-keycloak/tsf-iam-tkhpq] addLogicalPort took 14.632568ms, libovsdb time 903.438µs I0423 19:58:33.558089 3773 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=110 I0423 19:58:33.870810 3773 cni.go:452] [tsf-keycloak/tsf-iam-tkhpq 657bf808d9afeca2d32a8eda706f0101d9ef9a264d3e8f58ad9c0e55e341d004 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/tsf-iam-tkhpq 657bf808d9afeca2d32a8eda706f0101d9ef9a264d3e8f58ad9c0e55e341d004 network default NAD default NAD key ] I0423 19:58:33.881228 3773 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-tkhpq, hostIfaceName: 657bf808d9afeca, network: default, NAD default, SandboxID: "657bf808d9afeca2d32a8eda706f0101d9ef9a264d3e8f58ad9c0e55e341d004", PCI device ID: , UID: "828874dc-5030-4d2d-8924-306e0916d017", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0423 19:58:33.936843 3773 cni.go:473] [tsf-keycloak/tsf-iam-tkhpq 657bf808d9afeca2d32a8eda706f0101d9ef9a264d3e8f58ad9c0e55e341d004 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/tsf-iam-tkhpq 657bf808d9afeca2d32a8eda706f0101d9ef9a264d3e8f58ad9c0e55e341d004 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:52:55:cc:4b:c5\",\"name\":\"657bf808d9afeca\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cc66fda8-b533-456c-83a3-12e2669e42c2\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:58:38.978521 3773 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0423 19:58:43.777364 3773 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:58:43.777978 3773 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:58:43.814108 3773 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:58:43.814152 3773 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:58:43.823166 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0423 19:58:43.823189 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0423 19:58:46.904319 3773 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-rppps-644tm] creating logical port tsf-tas_rekor-createtree-job-rppps-644tm for pod on switch ip-10-0-0-253.ec2.internal I0423 19:58:46.904445 3773 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-rppps-644tm I0423 19:58:46.929168 3773 pod.go:62] [tsf-tas/rekor-createtree-job-rppps-644tm] pod update took 24.743906ms I0423 19:58:46.929195 3773 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-rppps-644tm] addLogicalPort annotation time took 24.779647ms I0423 19:58:46.930585 3773 pods.go:271] [tsf-tas/rekor-createtree-job-rppps-644tm] addLogicalPort took 26.281511ms, libovsdb time 997.832µs I0423 19:58:47.271739 3773 cni.go:452] [tsf-tas/rekor-createtree-job-rppps-644tm b411e803f77b0c9e10a312604dbb8e846c5f547c52a0c6331aebfd5e05eff168 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-rppps-644tm b411e803f77b0c9e10a312604dbb8e846c5f547c52a0c6331aebfd5e05eff168 network default NAD default NAD key ] I0423 19:58:47.281647 3773 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-rppps-644tm, hostIfaceName: b411e803f77b0c9, network: default, NAD default, SandboxID: "b411e803f77b0c9e10a312604dbb8e846c5f547c52a0c6331aebfd5e05eff168", PCI device ID: , UID: "1654cd68-2602-416f-9ed7-a4b2361aa967", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0423 19:58:47.339356 3773 cni.go:473] [tsf-tas/rekor-createtree-job-rppps-644tm b411e803f77b0c9e10a312604dbb8e846c5f547c52a0c6331aebfd5e05eff168 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-rppps-644tm b411e803f77b0c9e10a312604dbb8e846c5f547c52a0c6331aebfd5e05eff168 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:82:11:67:58:2e\",\"name\":\"b411e803f77b0c9\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b664461c-10ea-4ab5-94d4-c151a50cc425\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 19:58:49.077438 3773 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0423 19:58:49.077522 3773 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server I0423 19:58:49.296520 3773 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0423 19:58:52.777646 3773 cni.go:452] [tsf-keycloak/tsf-iam-tkhpq 657bf808d9afeca2d32a8eda706f0101d9ef9a264d3e8f58ad9c0e55e341d004 network default NAD default NAD key ] DEL starting CNI request [tsf-keycloak/tsf-iam-tkhpq 657bf808d9afeca2d32a8eda706f0101d9ef9a264d3e8f58ad9c0e55e341d004 network default NAD default NAD key ] I0423 19:58:52.864316 3773 cni.go:473] [tsf-keycloak/tsf-iam-tkhpq 657bf808d9afeca2d32a8eda706f0101d9ef9a264d3e8f58ad9c0e55e341d004 network default NAD default NAD key default] DEL finished CNI request [tsf-keycloak/tsf-iam-tkhpq 657bf808d9afeca2d32a8eda706f0101d9ef9a264d3e8f58ad9c0e55e341d004 network default NAD default NAD key default], result "{}", err I0423 19:58:52.912721 3773 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-tkhpq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:58:52.912754 3773 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-tkhpq I0423 19:58:52.914378 3773 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-tkhpq, ips: 10.128.0.40 I0423 19:58:52.914413 3773 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-tkhpq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:58:53.794529 3773 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-tkhpq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:58:53.794561 3773 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-tkhpq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0423 19:58:55.122044 3773 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0423 19:58:55.566753 3773 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=946 W0423 19:59:00.712410 3773 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0423 19:59:02.708888 3773 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0423 19:59:03.828587 3773 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-xj5p6-qbtgf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:03.829160 3773 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-xj5p6-qbtgf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:04.849203 3773 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-xj5p6-qbtgf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:04.849224 3773 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-xj5p6-qbtgf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:05.539712 3773 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ldp2s] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-ldp2s for pod on switch ip-10-0-0-253.ec2.internal I0423 19:59:05.539805 3773 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ldp2s I0423 19:59:05.564010 3773 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ldp2s] pod update took 24.231896ms I0423 19:59:05.564030 3773 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ldp2s] addLogicalPort annotation time took 24.258246ms I0423 19:59:05.565277 3773 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ldp2s] addLogicalPort took 25.578224ms, libovsdb time 812.96µs I0423 19:59:05.900745 3773 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ldp2s 1c4228a3bfc76a140cfde81ffceda5a194c284a653e7af0c4d9ffdbc29aa6569 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ldp2s 1c4228a3bfc76a140cfde81ffceda5a194c284a653e7af0c4d9ffdbc29aa6569 network default NAD default NAD key ] I0423 19:59:05.909999 3773 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-ldp2s, hostIfaceName: 1c4228a3bfc76a1, network: default, NAD default, SandboxID: "1c4228a3bfc76a140cfde81ffceda5a194c284a653e7af0c4d9ffdbc29aa6569", PCI device ID: , UID: "86c029b5-9982-43d8-8368-ddba6efc4b1b", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0423 19:59:05.961582 3773 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ldp2s 1c4228a3bfc76a140cfde81ffceda5a194c284a653e7af0c4d9ffdbc29aa6569 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ldp2s 1c4228a3bfc76a140cfde81ffceda5a194c284a653e7af0c4d9ffdbc29aa6569 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:56:73:d1:d3:49\",\"name\":\"1c4228a3bfc76a1\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/29e8ad0c-eb60-4db2-afb7-8ab15b09d77e\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 19:59:08.855157 3773 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0423 19:59:10.737457 3773 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0423 19:59:10.737476 3773 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0423 19:59:19.860333 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0423 19:59:35.012327 3773 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:35.012877 3773 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:35.030984 3773 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:35.031005 3773 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:35.036678 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0423 19:59:35.036698 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0423 19:59:35.844377 3773 base_network_controller_pods.go:487] [default/tsf/patch-tekton-config-qnxz7] creating logical port tsf_patch-tekton-config-qnxz7 for pod on switch ip-10-0-0-253.ec2.internal I0423 19:59:35.844487 3773 kube.go:256] Updating pod tsf/patch-tekton-config-qnxz7 I0423 19:59:35.859668 3773 pod.go:62] [tsf/patch-tekton-config-qnxz7] pod update took 15.213121ms I0423 19:59:35.859693 3773 base_network_controller_pods.go:951] [default/tsf/patch-tekton-config-qnxz7] addLogicalPort annotation time took 15.243673ms I0423 19:59:35.861085 3773 pods.go:271] [tsf/patch-tekton-config-qnxz7] addLogicalPort took 16.731152ms, libovsdb time 984.349µs I0423 19:59:36.205013 3773 cni.go:452] [tsf/patch-tekton-config-qnxz7 8bfd03dd3f2bf403087fdf500407c2f566a712e2495f1fda3835dcd9fb4e268d network default NAD default NAD key ] ADD starting CNI request [tsf/patch-tekton-config-qnxz7 8bfd03dd3f2bf403087fdf500407c2f566a712e2495f1fda3835dcd9fb4e268d network default NAD default NAD key ] I0423 19:59:36.214172 3773 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: patch-tekton-config-qnxz7, hostIfaceName: 8bfd03dd3f2bf40, network: default, NAD default, SandboxID: "8bfd03dd3f2bf403087fdf500407c2f566a712e2495f1fda3835dcd9fb4e268d", PCI device ID: , UID: "e0d7d1a3-ae42-41eb-8f20-2469ead36b5f", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0423 19:59:36.265921 3773 cni.go:473] [tsf/patch-tekton-config-qnxz7 8bfd03dd3f2bf403087fdf500407c2f566a712e2495f1fda3835dcd9fb4e268d network default NAD default NAD key default] ADD finished CNI request [tsf/patch-tekton-config-qnxz7 8bfd03dd3f2bf403087fdf500407c2f566a712e2495f1fda3835dcd9fb4e268d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:c5:60:fb:29:44\",\"name\":\"8bfd03dd3f2bf40\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f983cfb2-cde3-443e-b690-c4385c46c1e9\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 19:59:41.040907 3773 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-6w8fz-z7stp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:41.041493 3773 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-6w8fz-z7stp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:41.919972 3773 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-6w8fz-z7stp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:41.919997 3773 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-6w8fz-z7stp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:43.038981 3773 cni.go:452] [tsf-tas/rekor-createtree-job-rppps-644tm b411e803f77b0c9e10a312604dbb8e846c5f547c52a0c6331aebfd5e05eff168 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-rppps-644tm b411e803f77b0c9e10a312604dbb8e846c5f547c52a0c6331aebfd5e05eff168 network default NAD default NAD key ] I0423 19:59:43.129954 3773 cni.go:473] [tsf-tas/rekor-createtree-job-rppps-644tm b411e803f77b0c9e10a312604dbb8e846c5f547c52a0c6331aebfd5e05eff168 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-rppps-644tm b411e803f77b0c9e10a312604dbb8e846c5f547c52a0c6331aebfd5e05eff168 network default NAD default NAD key default], result "{}", err I0423 19:59:43.176150 3773 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-rppps-644tm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:43.176180 3773 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-rppps-644tm I0423 19:59:43.177573 3773 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-rppps-644tm, ips: 10.128.0.41 I0423 19:59:43.177599 3773 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-rppps-644tm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:44.044319 3773 cni.go:452] [tsf/patch-tekton-config-qnxz7 8bfd03dd3f2bf403087fdf500407c2f566a712e2495f1fda3835dcd9fb4e268d network default NAD default NAD key ] DEL starting CNI request [tsf/patch-tekton-config-qnxz7 8bfd03dd3f2bf403087fdf500407c2f566a712e2495f1fda3835dcd9fb4e268d network default NAD default NAD key ] I0423 19:59:44.051493 3773 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-rppps-644tm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:44.051517 3773 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-rppps-644tm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:44.125288 3773 cni.go:473] [tsf/patch-tekton-config-qnxz7 8bfd03dd3f2bf403087fdf500407c2f566a712e2495f1fda3835dcd9fb4e268d network default NAD default NAD key default] DEL finished CNI request [tsf/patch-tekton-config-qnxz7 8bfd03dd3f2bf403087fdf500407c2f566a712e2495f1fda3835dcd9fb4e268d network default NAD default NAD key default], result "{}", err I0423 19:59:44.171308 3773 obj_retry.go:498] Detected object tsf/patch-tekton-config-qnxz7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:44.171334 3773 pods.go:176] Deleting pod: tsf/patch-tekton-config-qnxz7 I0423 19:59:44.172661 3773 pods.go:236] Attempting to release IPs for pod: tsf/patch-tekton-config-qnxz7, ips: 10.128.0.43 I0423 19:59:44.172685 3773 obj_retry.go:498] Detected object tsf/patch-tekton-config-qnxz7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:45.195981 3773 obj_retry.go:492] Detected object tsf/patch-tekton-config-qnxz7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:45.196012 3773 obj_retry.go:492] Detected object tsf/patch-tekton-config-qnxz7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:45.296410 3773 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-4q7n9] creating logical port tsf_tsf-tekton-configuration-4q7n9 for pod on switch ip-10-0-0-253.ec2.internal I0423 19:59:45.296543 3773 kube.go:256] Updating pod tsf/tsf-tekton-configuration-4q7n9 I0423 19:59:45.308609 3773 pod.go:62] [tsf/tsf-tekton-configuration-4q7n9] pod update took 12.090368ms I0423 19:59:45.308635 3773 base_network_controller_pods.go:951] [default/tsf/tsf-tekton-configuration-4q7n9] addLogicalPort annotation time took 12.11986ms I0423 19:59:45.310051 3773 pods.go:271] [tsf/tsf-tekton-configuration-4q7n9] addLogicalPort took 13.656519ms, libovsdb time 1.028168ms I0423 19:59:45.501209 3773 namespace.go:142] [openshift-pipelines] updating namespace I0423 19:59:45.657792 3773 cni.go:452] [tsf/tsf-tekton-configuration-4q7n9 e9550c9164f1fa3ed64ffe5146e76001e785035d9e98dab7926101bb4cef3c4d network default NAD default NAD key ] ADD starting CNI request [tsf/tsf-tekton-configuration-4q7n9 e9550c9164f1fa3ed64ffe5146e76001e785035d9e98dab7926101bb4cef3c4d network default NAD default NAD key ] I0423 19:59:45.667318 3773 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-4q7n9, hostIfaceName: e9550c9164f1fa3, network: default, NAD default, SandboxID: "e9550c9164f1fa3ed64ffe5146e76001e785035d9e98dab7926101bb4cef3c4d", PCI device ID: , UID: "07bd3be9-7d72-4004-be74-9348f0629aeb", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0423 19:59:45.718256 3773 cni.go:473] [tsf/tsf-tekton-configuration-4q7n9 e9550c9164f1fa3ed64ffe5146e76001e785035d9e98dab7926101bb4cef3c4d network default NAD default NAD key default] ADD finished CNI request [tsf/tsf-tekton-configuration-4q7n9 e9550c9164f1fa3ed64ffe5146e76001e785035d9e98dab7926101bb4cef3c4d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:3f:99:37:c9:e3\",\"name\":\"e9550c9164f1fa3\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/857f9b35-b87f-41d3-9f85-9a049d7d99c2\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 19:59:48.945022 3773 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0423 19:59:51.962775 3773 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0423 19:59:52.972922 3773 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0423 19:59:53.094365 3773 cni.go:452] [tsf/tsf-tekton-configuration-4q7n9 e9550c9164f1fa3ed64ffe5146e76001e785035d9e98dab7926101bb4cef3c4d network default NAD default NAD key ] DEL starting CNI request [tsf/tsf-tekton-configuration-4q7n9 e9550c9164f1fa3ed64ffe5146e76001e785035d9e98dab7926101bb4cef3c4d network default NAD default NAD key ] I0423 19:59:53.170033 3773 cni.go:473] [tsf/tsf-tekton-configuration-4q7n9 e9550c9164f1fa3ed64ffe5146e76001e785035d9e98dab7926101bb4cef3c4d network default NAD default NAD key default] DEL finished CNI request [tsf/tsf-tekton-configuration-4q7n9 e9550c9164f1fa3ed64ffe5146e76001e785035d9e98dab7926101bb4cef3c4d network default NAD default NAD key default], result "{}", err I0423 19:59:53.214080 3773 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-4q7n9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:53.214111 3773 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-4q7n9 I0423 19:59:53.215740 3773 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-4q7n9, ips: 10.128.0.44 I0423 19:59:53.215770 3773 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-4q7n9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:54.233847 3773 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-4q7n9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:54.233886 3773 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-4q7n9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:54.281752 3773 obj_retry.go:492] Detected object tsf/patch-tekton-config-qnxz7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:54.281784 3773 obj_retry.go:492] Detected object tsf/patch-tekton-config-qnxz7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:54.286851 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-qnxz7 I0423 19:59:54.286877 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-qnxz7 I0423 19:59:58.436023 3773 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:58.436622 3773 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:58.455557 3773 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:58.455576 3773 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:58.464451 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0423 19:59:58.464473 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa W0423 19:59:59.013489 3773 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0423 19:59:59.799878 3773 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-dqdjz] creating logical port tsf-tpa_create-db-dqdjz for pod on switch ip-10-0-0-253.ec2.internal I0423 19:59:59.799998 3773 kube.go:256] Updating pod tsf-tpa/create-db-dqdjz I0423 19:59:59.815532 3773 pod.go:62] [tsf-tpa/create-db-dqdjz] pod update took 15.588934ms I0423 19:59:59.815553 3773 base_network_controller_pods.go:951] [default/tsf-tpa/create-db-dqdjz] addLogicalPort annotation time took 15.614345ms I0423 19:59:59.816848 3773 pods.go:271] [tsf-tpa/create-db-dqdjz] addLogicalPort took 16.984667ms, libovsdb time 887.606µs I0423 20:00:00.159081 3773 cni.go:452] [tsf-tpa/create-db-dqdjz 36853b09de216c4f2347a324435e989e18cf9959e994155a3567fb0df64742dd network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-db-dqdjz 36853b09de216c4f2347a324435e989e18cf9959e994155a3567fb0df64742dd network default NAD default NAD key ] I0423 20:00:00.168298 3773 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-db-dqdjz, hostIfaceName: 36853b09de216c4, network: default, NAD default, SandboxID: "36853b09de216c4f2347a324435e989e18cf9959e994155a3567fb0df64742dd", PCI device ID: , UID: "7745c77b-c4ed-4838-b094-84f8b80fd89c", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0423 20:00:00.219380 3773 cni.go:473] [tsf-tpa/create-db-dqdjz 36853b09de216c4f2347a324435e989e18cf9959e994155a3567fb0df64742dd network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-db-dqdjz 36853b09de216c4f2347a324435e989e18cf9959e994155a3567fb0df64742dd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:3a:5e:64:dd:3f\",\"name\":\"36853b09de216c4\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e13f7540-1d56-49df-abd8-4ac72f777c85\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:00:04.185289 3773 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-cfspz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:04.185847 3773 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-cfspz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:05.062333 3773 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-cfspz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:05.062359 3773 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-cfspz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:06.160723 3773 cni.go:452] [tsf-tpa/create-db-dqdjz 36853b09de216c4f2347a324435e989e18cf9959e994155a3567fb0df64742dd network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-db-dqdjz 36853b09de216c4f2347a324435e989e18cf9959e994155a3567fb0df64742dd network default NAD default NAD key ] I0423 20:00:06.248518 3773 cni.go:473] [tsf-tpa/create-db-dqdjz 36853b09de216c4f2347a324435e989e18cf9959e994155a3567fb0df64742dd network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-db-dqdjz 36853b09de216c4f2347a324435e989e18cf9959e994155a3567fb0df64742dd network default NAD default NAD key default], result "{}", err I0423 20:00:06.300008 3773 obj_retry.go:498] Detected object tsf-tpa/create-db-dqdjz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:06.300039 3773 pods.go:176] Deleting pod: tsf-tpa/create-db-dqdjz I0423 20:00:06.301382 3773 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-dqdjz, ips: 10.128.0.45 I0423 20:00:06.301411 3773 obj_retry.go:498] Detected object tsf-tpa/create-db-dqdjz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:07.323352 3773 obj_retry.go:492] Detected object tsf-tpa/create-db-dqdjz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:07.323372 3773 obj_retry.go:492] Detected object tsf-tpa/create-db-dqdjz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:07.429603 3773 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-p85md] creating logical port tsf-tpa_migrate-db-p85md for pod on switch ip-10-0-0-253.ec2.internal I0423 20:00:07.429691 3773 kube.go:256] Updating pod tsf-tpa/migrate-db-p85md I0423 20:00:07.441147 3773 pod.go:62] [tsf-tpa/migrate-db-p85md] pod update took 11.461211ms I0423 20:00:07.441171 3773 base_network_controller_pods.go:951] [default/tsf-tpa/migrate-db-p85md] addLogicalPort annotation time took 11.502996ms I0423 20:00:07.442436 3773 pods.go:271] [tsf-tpa/migrate-db-p85md] addLogicalPort took 12.848878ms, libovsdb time 867.839µs I0423 20:00:07.791393 3773 cni.go:452] [tsf-tpa/migrate-db-p85md 80415376b42fa928677504dc71417d902739262b05b4e3dfe14b017fd0e9b391 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/migrate-db-p85md 80415376b42fa928677504dc71417d902739262b05b4e3dfe14b017fd0e9b391 network default NAD default NAD key ] I0423 20:00:07.801680 3773 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-p85md, hostIfaceName: 80415376b42fa92, network: default, NAD default, SandboxID: "80415376b42fa928677504dc71417d902739262b05b4e3dfe14b017fd0e9b391", PCI device ID: , UID: "c726a287-65ce-4f65-9321-17a351cba9ad", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0423 20:00:07.852286 3773 cni.go:473] [tsf-tpa/migrate-db-p85md 80415376b42fa928677504dc71417d902739262b05b4e3dfe14b017fd0e9b391 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/migrate-db-p85md 80415376b42fa928677504dc71417d902739262b05b4e3dfe14b017fd0e9b391 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:b3:e7:2b:90:fe\",\"name\":\"80415376b42fa92\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/46e80dc1-90e4-479a-8e8b-acfa3fa97d4f\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:00:10.184710 3773 cni.go:452] [tsf-tpa/migrate-db-p85md 80415376b42fa928677504dc71417d902739262b05b4e3dfe14b017fd0e9b391 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/migrate-db-p85md 80415376b42fa928677504dc71417d902739262b05b4e3dfe14b017fd0e9b391 network default NAD default NAD key ] I0423 20:00:10.274561 3773 cni.go:473] [tsf-tpa/migrate-db-p85md 80415376b42fa928677504dc71417d902739262b05b4e3dfe14b017fd0e9b391 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/migrate-db-p85md 80415376b42fa928677504dc71417d902739262b05b4e3dfe14b017fd0e9b391 network default NAD default NAD key default], result "{}", err I0423 20:00:10.324429 3773 obj_retry.go:498] Detected object tsf-tpa/migrate-db-p85md of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:10.324457 3773 pods.go:176] Deleting pod: tsf-tpa/migrate-db-p85md I0423 20:00:10.325899 3773 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-p85md, ips: 10.128.0.46 I0423 20:00:10.325926 3773 obj_retry.go:498] Detected object tsf-tpa/migrate-db-p85md of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:10.462709 3773 obj_retry.go:492] Detected object tsf-tpa/migrate-db-p85md of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:10.462735 3773 obj_retry.go:492] Detected object tsf-tpa/migrate-db-p85md of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:13.579857 3773 base_network_controller_pods.go:487] [default/tsf-tpa/server-6c45987ff-fj94n] creating logical port tsf-tpa_server-6c45987ff-fj94n for pod on switch ip-10-0-0-253.ec2.internal I0423 20:00:13.579965 3773 kube.go:256] Updating pod tsf-tpa/server-6c45987ff-fj94n I0423 20:00:13.594295 3773 pod.go:62] [tsf-tpa/server-6c45987ff-fj94n] pod update took 14.356723ms I0423 20:00:13.594326 3773 base_network_controller_pods.go:951] [default/tsf-tpa/server-6c45987ff-fj94n] addLogicalPort annotation time took 14.391627ms I0423 20:00:13.595774 3773 pods.go:271] [tsf-tpa/server-6c45987ff-fj94n] addLogicalPort took 15.935068ms, libovsdb time 922.742µs I0423 20:00:13.630714 3773 base_network_controller_pods.go:487] [default/tsf-tpa/importer-86bcdf477b-tc4j7] creating logical port tsf-tpa_importer-86bcdf477b-tc4j7 for pod on switch ip-10-0-0-253.ec2.internal I0423 20:00:13.630827 3773 kube.go:256] Updating pod tsf-tpa/importer-86bcdf477b-tc4j7 I0423 20:00:13.648538 3773 pod.go:62] [tsf-tpa/importer-86bcdf477b-tc4j7] pod update took 17.739576ms I0423 20:00:13.648566 3773 base_network_controller_pods.go:951] [default/tsf-tpa/importer-86bcdf477b-tc4j7] addLogicalPort annotation time took 17.774481ms I0423 20:00:13.649811 3773 pods.go:271] [tsf-tpa/importer-86bcdf477b-tc4j7] addLogicalPort took 19.11453ms, libovsdb time 864.372µs I0423 20:00:15.698529 3773 obj_retry.go:498] Detected object tsf-tpa/create-importers-nx22w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:15.699115 3773 obj_retry.go:498] Detected object tsf-tpa/create-importers-nx22w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:16.722846 3773 obj_retry.go:492] Detected object tsf-tpa/create-importers-nx22w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:16.722869 3773 obj_retry.go:492] Detected object tsf-tpa/create-importers-nx22w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:24.954251 3773 cni.go:452] [tsf-tpa/server-6c45987ff-fj94n 210ced9bbe526de479730b5b9b8581f114fcf62f1753aa7230fe2b1cd030e6ed network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-6c45987ff-fj94n 210ced9bbe526de479730b5b9b8581f114fcf62f1753aa7230fe2b1cd030e6ed network default NAD default NAD key ] I0423 20:00:24.964163 3773 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: server-6c45987ff-fj94n, hostIfaceName: 210ced9bbe526de, network: default, NAD default, SandboxID: "210ced9bbe526de479730b5b9b8581f114fcf62f1753aa7230fe2b1cd030e6ed", PCI device ID: , UID: "13727e91-43ff-4044-a4bd-e15e5d63cdad", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0423 20:00:24.975492 3773 cni.go:452] [tsf-tpa/importer-86bcdf477b-tc4j7 4ccfc8ed2b7ba895eae33dd00ecb94ba934493a3cb69bb685a832b44691ea56c network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/importer-86bcdf477b-tc4j7 4ccfc8ed2b7ba895eae33dd00ecb94ba934493a3cb69bb685a832b44691ea56c network default NAD default NAD key ] I0423 20:00:24.985261 3773 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: importer-86bcdf477b-tc4j7, hostIfaceName: 4ccfc8ed2b7ba89, network: default, NAD default, SandboxID: "4ccfc8ed2b7ba895eae33dd00ecb94ba934493a3cb69bb685a832b44691ea56c", PCI device ID: , UID: "2e606d7b-b5f2-49cb-a57f-1e21f89ba19f", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0423 20:00:25.022151 3773 cni.go:473] [tsf-tpa/server-6c45987ff-fj94n 210ced9bbe526de479730b5b9b8581f114fcf62f1753aa7230fe2b1cd030e6ed network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-6c45987ff-fj94n 210ced9bbe526de479730b5b9b8581f114fcf62f1753aa7230fe2b1cd030e6ed network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:dd:0c:5d:b2:32\",\"name\":\"210ced9bbe526de\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f59117e3-3e87-4383-b2ba-625ab0d87c8f\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:00:25.047836 3773 cni.go:473] [tsf-tpa/importer-86bcdf477b-tc4j7 4ccfc8ed2b7ba895eae33dd00ecb94ba934493a3cb69bb685a832b44691ea56c network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/importer-86bcdf477b-tc4j7 4ccfc8ed2b7ba895eae33dd00ecb94ba934493a3cb69bb685a832b44691ea56c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:3e:f6:c6:33:32\",\"name\":\"4ccfc8ed2b7ba89\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cf5b0ee3-4cef-4de4-b551-b85d371ce6e0\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 20:00:25.261929 3773 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0423 20:00:28.798878 3773 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-jr2f6-knqbv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:28.799456 3773 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-jr2f6-knqbv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:29.666149 3773 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-jr2f6-knqbv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:29.666177 3773 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-jr2f6-knqbv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0423 20:00:36.297307 3773 util.go:959] Failed to build node endpoints for node ip-10-0-0-253.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server W0423 20:00:37.705579 3773 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0423 20:00:52.468112 3773 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:52.468659 3773 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:52.507574 3773 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:52.507598 3773 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:52.513468 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0423 20:00:52.513492 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0423 20:00:53.209313 3773 namespace.go:98] [image-controller] adding namespace I0423 20:00:53.209463 3773 namespace.go:98] [default-tenant] adding namespace I0423 20:00:53.210203 3773 namespace.go:102] [image-controller] adding namespace took 870.273µs I0423 20:00:53.210797 3773 namespace.go:102] [default-tenant] adding namespace took 1.318065ms I0423 20:00:53.212841 3773 namespace.go:98] [integration-service] adding namespace I0423 20:00:53.213486 3773 namespace.go:102] [integration-service] adding namespace took 629.59µs I0423 20:00:53.215871 3773 namespace.go:98] [build-service] adding namespace I0423 20:00:53.216555 3773 namespace.go:102] [build-service] adding namespace took 667.644µs I0423 20:00:53.241896 3773 namespace.go:142] [image-controller] updating namespace I0423 20:00:53.268027 3773 namespace.go:142] [image-controller] updating namespace I0423 20:00:53.278103 3773 namespace.go:142] [default-tenant] updating namespace I0423 20:00:53.321213 3773 namespace.go:142] [integration-service] updating namespace I0423 20:00:53.324858 3773 namespace.go:142] [default-tenant] updating namespace I0423 20:00:53.345898 3773 namespace.go:142] [build-service] updating namespace I0423 20:00:53.355006 3773 namespace.go:142] [integration-service] updating namespace I0423 20:00:53.374359 3773 namespace.go:142] [build-service] updating namespace I0423 20:00:53.439307 3773 namespace.go:142] [build-service] updating namespace I0423 20:00:53.456676 3773 namespace.go:142] [integration-service] updating namespace I0423 20:00:53.479055 3773 namespace.go:98] [release-service] adding namespace I0423 20:00:53.479903 3773 namespace.go:102] [release-service] adding namespace took 828.453µs I0423 20:00:53.502292 3773 namespace.go:142] [release-service] updating namespace I0423 20:00:53.522996 3773 namespace.go:142] [release-service] updating namespace I0423 20:00:53.571101 3773 namespace.go:142] [konflux-ui] updating namespace I0423 20:00:53.620009 3773 namespace.go:98] [konflux-info] adding namespace I0423 20:00:53.620796 3773 namespace.go:102] [konflux-info] adding namespace took 767.375µs I0423 20:00:53.652898 3773 namespace.go:142] [konflux-info] updating namespace I0423 20:00:53.733044 3773 namespace.go:142] [konflux-info] updating namespace I0423 20:00:53.836003 3773 namespace.go:98] [namespace-lister] adding namespace I0423 20:00:53.836822 3773 namespace.go:102] [namespace-lister] adding namespace took 790.636µs I0423 20:00:53.936231 3773 namespace.go:142] [namespace-lister] updating namespace I0423 20:00:53.982297 3773 namespace.go:98] [enterprise-contract-service] adding namespace I0423 20:00:53.983245 3773 namespace.go:102] [enterprise-contract-service] adding namespace took 922.932µs I0423 20:00:54.001990 3773 namespace.go:142] [namespace-lister] updating namespace I0423 20:00:54.100052 3773 namespace.go:142] [enterprise-contract-service] updating namespace I0423 20:00:54.175002 3773 namespace.go:142] [enterprise-contract-service] updating namespace I0423 20:00:54.211179 3773 namespace.go:142] [image-controller] updating namespace I0423 20:00:54.641453 3773 namespace.go:142] [default-tenant] updating namespace I0423 20:00:54.917516 3773 namespace.go:98] [segment-bridge] adding namespace I0423 20:00:54.917549 3773 namespace.go:98] [konflux-cli] adding namespace I0423 20:00:54.918537 3773 namespace.go:102] [segment-bridge] adding namespace took 894.88µs I0423 20:00:54.919170 3773 namespace.go:102] [konflux-cli] adding namespace took 1.440409ms I0423 20:00:55.313009 3773 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0423 20:00:55.313677 3773 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0423 20:00:55.313695 3773 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0423 20:00:55.316412 3773 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0423 20:00:55.316447 3773 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0423 20:00:55.316459 3773 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0423 20:00:55.316991 3773 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0423 20:00:55.434223 3773 namespace.go:142] [build-service] updating namespace I0423 20:00:55.434218 3773 namespace.go:142] [segment-bridge] updating namespace I0423 20:00:55.633861 3773 namespace.go:142] [segment-bridge] updating namespace I0423 20:00:55.633907 3773 namespace.go:142] [konflux-cli] updating namespace I0423 20:00:55.711312 3773 namespace.go:142] [konflux-cli] updating namespace I0423 20:00:55.798060 3773 namespace.go:142] [default-tenant] updating namespace I0423 20:00:56.001547 3773 namespace.go:142] [enterprise-contract-service] updating namespace I0423 20:00:56.140194 3773 namespace.go:142] [image-controller] updating namespace I0423 20:00:56.370005 3773 namespace.go:142] [integration-service] updating namespace I0423 20:00:56.569327 3773 namespace.go:142] [konflux-info] updating namespace I0423 20:00:56.766603 3773 namespace.go:142] [namespace-lister] updating namespace I0423 20:00:57.000068 3773 namespace.go:142] [release-service] updating namespace I0423 20:00:57.125584 3773 namespace.go:142] [build-service] updating namespace I0423 20:00:57.161079 3773 namespace.go:142] [default-tenant] updating namespace I0423 20:00:57.205282 3773 namespace.go:142] [enterprise-contract-service] updating namespace I0423 20:00:57.223380 3773 namespace.go:142] [image-controller] updating namespace I0423 20:00:57.239046 3773 namespace.go:142] [integration-service] updating namespace I0423 20:00:57.268021 3773 namespace.go:142] [konflux-info] updating namespace I0423 20:00:57.297603 3773 namespace.go:142] [namespace-lister] updating namespace I0423 20:00:57.327933 3773 namespace.go:142] [release-service] updating namespace I0423 20:00:57.440359 3773 namespace.go:142] [build-service] updating namespace I0423 20:00:57.521265 3773 namespace.go:142] [default-tenant] updating namespace I0423 20:00:57.592016 3773 namespace.go:142] [enterprise-contract-service] updating namespace I0423 20:00:57.649822 3773 namespace.go:142] [image-controller] updating namespace I0423 20:00:57.716292 3773 namespace.go:142] [integration-service] updating namespace I0423 20:00:57.800945 3773 namespace.go:142] [konflux-info] updating namespace I0423 20:00:57.895579 3773 namespace.go:142] [namespace-lister] updating namespace I0423 20:00:57.954117 3773 namespace.go:142] [release-service] updating namespace I0423 20:00:59.281250 3773 namespace.go:142] [konflux-cli] updating namespace I0423 20:00:59.483418 3773 namespace.go:142] [segment-bridge] updating namespace I0423 20:00:59.589510 3773 namespace.go:142] [konflux-cli] updating namespace I0423 20:00:59.606493 3773 namespace.go:142] [segment-bridge] updating namespace I0423 20:00:59.705779 3773 namespace.go:142] [konflux-cli] updating namespace I0423 20:00:59.817102 3773 namespace.go:142] [segment-bridge] updating namespace W0423 20:01:00.848999 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0423 20:01:02.408576 3773 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0423 20:01:04.875451 3773 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0423 20:01:04.879969 3773 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0423 20:01:05.413465 3773 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0423 20:01:05.413524 3773 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0423 20:01:06.896282 3773 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0423 20:01:06.919665 3773 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0423 20:01:09.443004 3773 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0423 20:01:09.443023 3773 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0423 20:01:09.929186 3773 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0423 20:01:09.929208 3773 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0423 20:01:09.950594 3773 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0423 20:01:09.952303 3773 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service I0423 20:01:16.735165 3773 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-chqd5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:01:16.735691 3773 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-chqd5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:01:16.976176 3773 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-chqd5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:01:16.976200 3773 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-chqd5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:01:16.979920 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-chqd5 I0423 20:01:16.979947 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-chqd5 I0423 20:01:25.561486 3773 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0423 20:01:39.567039 3773 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=162 I0423 20:01:49.670028 3773 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0423 20:01:59.215708 3773 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=5 I0423 20:02:23.351474 3773 obj_retry.go:498] Detected object konflux-ui/proxy-79fb68b488-8bwn9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:02:23.352174 3773 obj_retry.go:498] Detected object konflux-ui/proxy-79fb68b488-8bwn9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:02:23.673379 3773 obj_retry.go:492] Detected object konflux-ui/proxy-79fb68b488-8bwn9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:02:23.673409 3773 obj_retry.go:492] Detected object konflux-ui/proxy-79fb68b488-8bwn9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:02:23.676541 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-79fb68b488-8bwn9 I0423 20:02:23.676568 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-79fb68b488-8bwn9 I0423 20:02:25.021809 3773 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:02:25.022387 3773 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:02:25.041908 3773 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:02:25.041935 3773 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:02:25.044593 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0423 20:02:25.044619 3773 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0423 20:02:39.560211 3773 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=121 I0423 20:03:06.753105 3773 namespace.go:98] [default-managed-tenant-ddc59] adding namespace I0423 20:03:06.753998 3773 namespace.go:102] [default-managed-tenant-ddc59] adding namespace took 869.581µs I0423 20:03:06.794653 3773 namespace.go:142] [default-managed-tenant-ddc59] updating namespace I0423 20:03:06.815694 3773 namespace.go:142] [default-managed-tenant-ddc59] updating namespace I0423 20:03:07.798644 3773 namespace.go:142] [default-managed-tenant-ddc59] updating namespace I0423 20:03:07.930973 3773 namespace.go:142] [default-managed-tenant-ddc59] updating namespace I0423 20:03:08.030653 3773 namespace.go:142] [default-managed-tenant-ddc59] updating namespace I0423 20:03:32.088178 3773 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0423 20:03:49.515971 3773 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0423 20:03:53.879124 3773 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7