+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-98.ec2.internal ++ [[ -n ip-10-0-1-98.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-98.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260318-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260318-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260318-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0403 00:41:27.639025450 - disable conntrack on geneve port' I0403 00:41:27.639025450 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0403 00:41:27.676934444 - starting ovnkube-node + echo 'I0403 00:41:27.676934444 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-98.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-98.ec2.internal --init-node ip-10-0-1-98.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-98.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0403 00:41:27.713306 4001 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0403 00:41:27.713366 4001 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-5e22b1149e.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0403 00:41:27.714829 4001 kube.go:423] Waiting for certificate I0403 00:41:27.714859 4001 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0403 00:41:27.714905 4001 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0403 00:41:27.716075 4001 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0403 00:41:27.739525 4001 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0403 00:41:27.739541 4001 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0403 00:41:27.741370 4001 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0403 00:41:27.743350 4001 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-zghg5" I0403 00:41:27.751408 4001 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-zghg5" I0403 00:41:27.751487 4001 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0403 00:41:28.715656 4001 kube.go:430] Certificate found I0403 00:41:28.716213 4001 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0403 00:41:28.717077 4001 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0403 00:41:28.757580 4001 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-04 00:36:27 +0000 UTC" deadline="2026-04-03 19:15:08.949512122 +0000 UTC" I0403 00:41:28.757644 4001 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="18h33m40.191881107s" I0403 00:41:28.790858 4001 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0403 00:41:28.799964 4001 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0403 00:41:28.805689 4001 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0403 00:41:28.805833 4001 factory.go:537] Starting watch factory I0403 00:41:28.805956 4001 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.805997 4001 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.806263 4001 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.806287 4001 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.806370 4001 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.806393 4001 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.806596 4001 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.806611 4001 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.806660 4001 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.806682 4001 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.806779 4001 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.806796 4001 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.812960 4001 controller_manager.go:368] Starting the ovnkube controller I0403 00:41:28.812974 4001 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-98.ec2.internal I0403 00:41:28.813030 4001 controller_manager.go:393] NBDB zone sync took: 47.435µs I0403 00:41:28.813041 4001 factory.go:537] Starting watch factory I0403 00:41:28.822483 4001 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.822587 4001 ovn_northd.go:121] Registering OVN northd metrics I0403 00:41:28.822671 4001 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.822685 4001 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0403 00:41:28.824571 4001 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.824905 4001 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.825955 4001 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.828840 4001 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0403 00:41:28.828872 4001 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0403 00:41:28.829491 4001 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0403 00:41:28.845761 4001 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0403 00:41:28.911909 4001 factory.go:1931] *v1.Node informer cache synced successfully I0403 00:41:28.911928 4001 factory.go:1931] *v1.Service informer cache synced successfully I0403 00:41:28.911932 4001 factory.go:1931] *v1.EndpointSlice informer cache synced successfully I0403 00:41:28.911935 4001 factory.go:1931] *v1.Pod informer cache synced successfully I0403 00:41:28.911938 4001 factory.go:1931] *v1.NetworkPolicy informer cache synced successfully I0403 00:41:28.911941 4001 factory.go:1931] *v1.Namespace informer cache synced successfully I0403 00:41:28.912034 4001 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0403 00:41:28.912034 4001 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0403 00:41:28.912050 4001 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0403 00:41:28.912055 4001 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0403 00:41:28.914706 4001 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0403 00:41:28.914723 4001 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0403 00:41:28.917895 4001 factory.go:1931] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0403 00:41:28.917906 4001 factory.go:1931] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0403 00:41:28.917959 4001 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.917970 4001 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.918100 4001 factory.go:1931] *v1.Node informer cache synced successfully I0403 00:41:28.918114 4001 factory.go:1931] *v1.Service informer cache synced successfully I0403 00:41:28.918119 4001 factory.go:1931] *v1.EndpointSlice informer cache synced successfully I0403 00:41:28.918125 4001 factory.go:1931] *v1.Pod informer cache synced successfully I0403 00:41:28.918129 4001 factory.go:1931] *v1.NetworkPolicy informer cache synced successfully I0403 00:41:28.918133 4001 factory.go:1931] *v1.Namespace informer cache synced successfully I0403 00:41:28.920291 4001 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.923241 4001 factory.go:1931] *v1.EgressIP informer cache synced successfully I0403 00:41:28.923247 4001 factory.go:1931] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0403 00:41:28.923259 4001 factory.go:1931] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0403 00:41:28.923772 4001 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.923787 4001 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.926294 4001 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.929289 4001 factory.go:1931] *v1.EgressIP informer cache synced successfully I0403 00:41:28.929303 4001 factory.go:1931] *v1.EgressFirewall informer cache synced successfully I0403 00:41:28.929356 4001 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.929374 4001 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.931723 4001 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.934538 4001 factory.go:1931] *v1.EgressFirewall informer cache synced successfully I0403 00:41:28.934549 4001 factory.go:1931] *v1.EgressQoS informer cache synced successfully I0403 00:41:28.934596 4001 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.934607 4001 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.936769 4001 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.939666 4001 factory.go:1931] *v1.EgressQoS informer cache synced successfully I0403 00:41:28.939680 4001 factory.go:1931] *v1.EgressService informer cache synced successfully I0403 00:41:28.939732 4001 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.939744 4001 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.941983 4001 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.944881 4001 factory.go:1931] *v1.EgressService informer cache synced successfully I0403 00:41:28.944895 4001 factory.go:1931] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0403 00:41:28.945137 4001 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0403 00:41:28.945159 4001 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0403 00:41:28.948260 4001 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0403 00:41:28.950188 4001 factory.go:1931] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0403 00:41:28.950206 4001 factory.go:1931] *v1.NetworkAttachmentDefinition informer cache synced successfully I0403 00:41:28.950260 4001 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.950271 4001 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.950318 4001 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.950336 4001 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.952657 4001 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.952697 4001 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0403 00:41:28.955858 4001 factory.go:1931] *v1.UserDefinedNetwork informer cache synced successfully I0403 00:41:28.955868 4001 factory.go:1931] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0403 00:41:28.955869 4001 factory.go:1931] *v1.NetworkAttachmentDefinition informer cache synced successfully I0403 00:41:28.955875 4001 factory.go:649] Watch Factory start up complete, took: 150.04645ms I0403 00:41:28.955940 4001 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0403 00:41:28.961694 4001 factory.go:1931] *v1.UserDefinedNetwork informer cache synced successfully I0403 00:41:28.961704 4001 factory.go:1931] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0403 00:41:28.961711 4001 factory.go:649] Watch Factory start up complete, took: 148.668887ms I0403 00:41:28.961718 4001 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-1-98.ec2.internal" zone I0403 00:41:28.986741 4001 default_node_network_controller.go:709] Initializing the default node network controller I0403 00:41:29.055655 4001 udn_isolation.go:101] Starting UDN host isolation manager I0403 00:41:29.061854 4001 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0403 00:41:29.124150 4001 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0403 00:41:29.124229 4001 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0403 00:41:29.124242 4001 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0403 00:41:29.155914 4001 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0403 00:41:29.155979 4001 default_node_network_controller.go:808] Node ip-10-0-1-98.ec2.internal ready for ovn initialization with subnet 10.131.0.0/23 I0403 00:41:29.160823 4001 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.98"] k8s.ovn.org/zone-name:ip-10-0-1-98.ec2.internal] on node ip-10-0-1-98.ec2.internal I0403 00:41:29.183326 4001 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0403 00:41:29.189327 4001 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0403 00:41:29.189488 4001 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0403 00:41:29.189653 4001 gateway_init.go:263] Preparing Gateway I0403 00:41:29.189661 4001 gateway_shared_intf.go:1658] Creating new gateway I0403 00:41:29.212045 4001 controller_manager.go:427] Waiting for node in zone sync took: 250.316354ms I0403 00:41:29.216373 4001 controller_manager.go:608] Switching to transit router for layer2 networks I0403 00:41:29.216390 4001 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-98.ec2.internal I0403 00:41:29.233692 4001 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0403 00:41:29.239863 4001 controller_manager.go:288] SCTP support detected in OVN I0403 00:41:29.240426 4001 services_controller.go:69] Creating services controller for network=default I0403 00:41:29.240531 4001 udn_enabled_svc.go:80] Waiting for service informer to sync I0403 00:41:29.240603 4001 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0403 00:41:29.240634 4001 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0403 00:41:29.240647 4001 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0403 00:41:29.240655 4001 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0403 00:41:29.240851 4001 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0403 00:41:29.240885 4001 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0403 00:41:29.240914 4001 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 21.182µs I0403 00:41:29.240918 4001 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 42.195µs I0403 00:41:29.240923 4001 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 37.807µs I0403 00:41:29.240930 4001 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0403 00:41:29.240938 4001 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0403 00:41:29.240946 4001 default_network_controller.go:321] Starting the default network controller I0403 00:41:29.241036 4001 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 13.77µs I0403 00:41:29.241141 4001 udn_enabled_svc.go:95] Performing full resync I0403 00:41:29.241885 4001 udn_enabled_svc.go:99] Waiting for handler to sync I0403 00:41:29.241899 4001 udn_enabled_svc.go:104] Starting worker I0403 00:41:29.250031 4001 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0403 00:41:29.250127 4001 repair.go:33] Syncing exgw routes took 83.894µs I0403 00:41:29.250157 4001 default_network_controller.go:407] Starting all the Watchers... I0403 00:41:29.250375 4001 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0403 00:41:29.250380 4001 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0403 00:41:29.250385 4001 namespace.go:98] [openshift-operators] adding namespace I0403 00:41:29.250377 4001 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0403 00:41:29.250397 4001 namespace.go:98] [openshift-logging] adding namespace I0403 00:41:29.250381 4001 namespace.go:98] [openshift-console] adding namespace I0403 00:41:29.250397 4001 namespace.go:98] [openshift-console-user-settings] adding namespace I0403 00:41:29.250404 4001 namespace.go:98] [openshift-apiserver-operator] adding namespace I0403 00:41:29.250410 4001 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0403 00:41:29.250415 4001 namespace.go:98] [openshift-route-controller-manager] adding namespace I0403 00:41:29.250419 4001 namespace.go:98] [openshift-config-managed] adding namespace I0403 00:41:29.250425 4001 namespace.go:98] [kube-public] adding namespace I0403 00:41:29.250430 4001 namespace.go:98] [openshift-authentication-operator] adding namespace I0403 00:41:29.250441 4001 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0403 00:41:29.250445 4001 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0403 00:41:29.251101 4001 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 693.099µs I0403 00:41:29.251127 4001 namespace.go:98] [openshift-config] adding namespace I0403 00:41:29.251720 4001 namespace.go:102] [openshift-config] adding namespace took 585.713µs I0403 00:41:29.251737 4001 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0403 00:41:29.252302 4001 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 557.958µs I0403 00:41:29.252317 4001 namespace.go:98] [openshift-backplane-cee] adding namespace I0403 00:41:29.252910 4001 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 2.516901ms I0403 00:41:29.252928 4001 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0403 00:41:29.253543 4001 namespace.go:102] [openshift-operators] adding namespace took 3.146887ms I0403 00:41:29.253556 4001 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0403 00:41:29.254105 4001 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 3.705067ms I0403 00:41:29.254116 4001 namespace.go:98] [openshift-dns] adding namespace I0403 00:41:29.254755 4001 namespace.go:102] [openshift-logging] adding namespace took 4.351777ms I0403 00:41:29.254766 4001 namespace.go:98] [openshift-backplane-cse] adding namespace I0403 00:41:29.255371 4001 namespace.go:102] [openshift-console] adding namespace took 4.961846ms I0403 00:41:29.255400 4001 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0403 00:41:29.255999 4001 namespace.go:102] [openshift-console-user-settings] adding namespace took 5.588527ms I0403 00:41:29.256016 4001 namespace.go:98] [openshift-controller-manager] adding namespace I0403 00:41:29.256561 4001 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.149023ms I0403 00:41:29.256577 4001 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0403 00:41:29.257009 4001 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.588595ms I0403 00:41:29.257024 4001 namespace.go:98] [open-cluster-management-2peb9ob26umtfq43icvb8p97pncv0c79] adding namespace I0403 00:41:29.257474 4001 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.050048ms I0403 00:41:29.257488 4001 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0403 00:41:29.257891 4001 namespace.go:102] [openshift-config-managed] adding namespace took 7.464828ms I0403 00:41:29.257904 4001 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0403 00:41:29.258346 4001 namespace.go:102] [kube-public] adding namespace took 7.915829ms I0403 00:41:29.258358 4001 namespace.go:98] [openshift-kube-apiserver] adding namespace I0403 00:41:29.258769 4001 namespace.go:102] [openshift-authentication-operator] adding namespace took 8.3287ms I0403 00:41:29.258782 4001 namespace.go:98] [openshift-network-operator] adding namespace I0403 00:41:29.259214 4001 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 8.764928ms I0403 00:41:29.259229 4001 namespace.go:98] [openshift-node] adding namespace I0403 00:41:29.259624 4001 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 9.169875ms I0403 00:41:29.259642 4001 namespace.go:98] [openshift-image-registry] adding namespace I0403 00:41:29.260109 4001 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.784866ms I0403 00:41:29.260127 4001 namespace.go:98] [openshift-backplane-mobb] adding namespace I0403 00:41:29.260508 4001 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.574283ms I0403 00:41:29.260517 4001 namespace.go:98] [openshift-backplane-csm] adding namespace I0403 00:41:29.260876 4001 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 7.315758ms I0403 00:41:29.260887 4001 namespace.go:98] [openshift-network-node-identity] adding namespace I0403 00:41:29.261268 4001 namespace.go:102] [openshift-dns] adding namespace took 7.145867ms I0403 00:41:29.261280 4001 namespace.go:98] [openshift-authentication] adding namespace I0403 00:41:29.261691 4001 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.918278ms I0403 00:41:29.261705 4001 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0403 00:41:29.262078 4001 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.671115ms I0403 00:41:29.262110 4001 namespace.go:98] [openshift-ingress] adding namespace I0403 00:41:29.262475 4001 namespace.go:102] [openshift-controller-manager] adding namespace took 6.45398ms I0403 00:41:29.262487 4001 namespace.go:98] [openshift-multus] adding namespace I0403 00:41:29.262880 4001 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.297744ms I0403 00:41:29.262891 4001 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0403 00:41:29.263281 4001 namespace.go:102] [open-cluster-management-2peb9ob26umtfq43icvb8p97pncv0c79] adding namespace took 6.250846ms I0403 00:41:29.263294 4001 namespace.go:98] [openshift-machine-config-operator] adding namespace I0403 00:41:29.263648 4001 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.155833ms I0403 00:41:29.263658 4001 namespace.go:98] [openshift-customer-monitoring] adding namespace I0403 00:41:29.264037 4001 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.128532ms I0403 00:41:29.264049 4001 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0403 00:41:29.264445 4001 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.08293ms I0403 00:41:29.264457 4001 namespace.go:98] [openshift-apiserver] adding namespace I0403 00:41:29.264898 4001 namespace.go:102] [openshift-network-operator] adding namespace took 6.111922ms I0403 00:41:29.264908 4001 namespace.go:98] [openshift-network-diagnostics] adding namespace I0403 00:41:29.265351 4001 namespace.go:102] [openshift-node] adding namespace took 6.116732ms I0403 00:41:29.265364 4001 namespace.go:98] [kube-node-lease] adding namespace I0403 00:41:29.265766 4001 namespace.go:102] [openshift-image-registry] adding namespace took 6.118285ms I0403 00:41:29.265781 4001 namespace.go:98] [dedicated-admin] adding namespace I0403 00:41:29.266139 4001 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.007892ms I0403 00:41:29.266636 4001 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.114285ms I0403 00:41:29.266647 4001 namespace.go:98] [openshift-must-gather-operator] adding namespace I0403 00:41:29.267015 4001 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.122443ms I0403 00:41:29.267027 4001 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0403 00:41:29.267421 4001 namespace.go:102] [openshift-authentication] adding namespace took 6.136389ms I0403 00:41:29.267434 4001 namespace.go:98] [openshift-kube-scheduler] adding namespace I0403 00:41:29.267787 4001 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.076482ms I0403 00:41:29.267799 4001 namespace.go:98] [openshift-cluster-version] adding namespace W0403 00:41:29.268135 4001 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-psb2l: pod openshift-multus/network-metrics-daemon-psb2l: no pod IPs found I0403 00:41:29.268180 4001 namespace.go:102] [openshift-ingress] adding namespace took 6.064965ms I0403 00:41:29.268189 4001 namespace.go:98] [openshift-package-operator] adding namespace I0403 00:41:29.268638 4001 namespace.go:102] [openshift-multus] adding namespace took 6.146578ms I0403 00:41:29.268651 4001 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0403 00:41:29.269014 4001 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.118301ms I0403 00:41:29.269024 4001 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0403 00:41:29.269422 4001 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.12318ms I0403 00:41:29.269434 4001 namespace.go:98] [openshift] adding namespace I0403 00:41:29.269777 4001 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.114529ms I0403 00:41:29.269788 4001 namespace.go:98] [openshift-monitoring] adding namespace I0403 00:41:29.270193 4001 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.139257ms I0403 00:41:29.270205 4001 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0403 00:41:29.270544 4001 namespace.go:102] [openshift-apiserver] adding namespace took 6.082977ms I0403 00:41:29.270555 4001 namespace.go:98] [openshift-etcd] adding namespace W0403 00:41:29.270572 4001 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-xd4g6: pod openshift-network-diagnostics/network-check-target-xd4g6: no pod IPs found I0403 00:41:29.271024 4001 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.110572ms I0403 00:41:29.271037 4001 namespace.go:98] [openshift-dns-operator] adding namespace I0403 00:41:29.271419 4001 namespace.go:102] [kube-node-lease] adding namespace took 6.050481ms I0403 00:41:29.271432 4001 namespace.go:98] [default] adding namespace I0403 00:41:29.271815 4001 namespace.go:102] [dedicated-admin] adding namespace took 6.03002ms I0403 00:41:29.271827 4001 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0403 00:41:29.272161 4001 namespace.go:102] [openshift-must-gather-operator] adding namespace took 5.510226ms I0403 00:41:29.272172 4001 namespace.go:98] [openshift-insights] adding namespace I0403 00:41:29.272676 4001 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 5.643965ms I0403 00:41:29.272689 4001 namespace.go:98] [openshift-network-console] adding namespace I0403 00:41:29.273130 4001 namespace.go:102] [openshift-kube-scheduler] adding namespace took 5.689749ms I0403 00:41:29.273146 4001 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0403 00:41:29.273570 4001 namespace.go:102] [openshift-cluster-version] adding namespace took 5.76661ms I0403 00:41:29.273583 4001 namespace.go:98] [openshift-backplane-srep] adding namespace I0403 00:41:29.273938 4001 namespace.go:102] [openshift-package-operator] adding namespace took 5.743628ms I0403 00:41:29.273952 4001 namespace.go:98] [openshift-machine-api] adding namespace I0403 00:41:29.274381 4001 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 5.723551ms I0403 00:41:29.274396 4001 namespace.go:98] [openshift-backplane-tam] adding namespace I0403 00:41:29.274794 4001 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 5.764826ms I0403 00:41:29.274806 4001 namespace.go:98] [openshift-host-network] adding namespace I0403 00:41:29.275230 4001 namespace.go:102] [openshift] adding namespace took 5.7915ms I0403 00:41:29.275240 4001 namespace.go:98] [openshift-console-operator] adding namespace I0403 00:41:29.275576 4001 namespace.go:102] [openshift-monitoring] adding namespace took 5.784162ms I0403 00:41:29.275588 4001 namespace.go:98] [kube-system] adding namespace I0403 00:41:29.276037 4001 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 5.825814ms I0403 00:41:29.276053 4001 namespace.go:98] [openshift-marketplace] adding namespace I0403 00:41:29.276534 4001 namespace.go:102] [openshift-etcd] adding namespace took 5.974312ms I0403 00:41:29.276545 4001 namespace.go:98] [openshift-operators-redhat] adding namespace I0403 00:41:29.276962 4001 namespace.go:102] [openshift-dns-operator] adding namespace took 5.91966ms I0403 00:41:29.276975 4001 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0403 00:41:29.277411 4001 namespace.go:102] [default] adding namespace took 5.972317ms I0403 00:41:29.277433 4001 namespace.go:98] [openshift-ingress-canary] adding namespace I0403 00:41:29.277837 4001 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.005796ms I0403 00:41:29.277851 4001 namespace.go:98] [openshift-ingress-operator] adding namespace I0403 00:41:29.278311 4001 namespace.go:102] [openshift-insights] adding namespace took 6.131234ms I0403 00:41:29.278789 4001 namespace.go:102] [openshift-network-console] adding namespace took 6.093775ms I0403 00:41:29.278807 4001 namespace.go:98] [openshift-config-operator] adding namespace I0403 00:41:29.279215 4001 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.061724ms I0403 00:41:29.279585 4001 namespace.go:102] [openshift-backplane-srep] adding namespace took 5.997251ms I0403 00:41:29.279602 4001 namespace.go:98] [openshift-backplane] adding namespace I0403 00:41:29.280101 4001 namespace.go:102] [openshift-machine-api] adding namespace took 6.1429ms I0403 00:41:29.280119 4001 namespace.go:98] [openshift-service-ca-operator] adding namespace I0403 00:41:29.280447 4001 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.046448ms I0403 00:41:29.280870 4001 iptables.go:108] Creating table: filter chain: FORWARD I0403 00:41:29.280931 4001 namespace.go:102] [openshift-host-network] adding namespace took 6.117779ms W0403 00:41:29.281393 4001 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-czv2x: pod kube-system/global-pull-secret-syncer-czv2x: no pod IPs found I0403 00:41:29.281443 4001 namespace.go:102] [openshift-console-operator] adding namespace took 6.196838ms I0403 00:41:29.281908 4001 namespace.go:102] [kube-system] adding namespace took 6.314572ms I0403 00:41:29.281921 4001 namespace.go:98] [openshift-infra] adding namespace I0403 00:41:29.282355 4001 namespace.go:102] [openshift-marketplace] adding namespace took 6.297091ms I0403 00:41:29.282770 4001 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.219974ms I0403 00:41:29.283221 4001 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.23932ms I0403 00:41:29.283717 4001 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.277732ms I0403 00:41:29.284120 4001 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.261596ms I0403 00:41:29.284669 4001 namespace.go:102] [openshift-config-operator] adding namespace took 5.855754ms I0403 00:41:29.285067 4001 namespace.go:102] [openshift-backplane] adding namespace took 5.457987ms I0403 00:41:29.285545 4001 namespace.go:102] [openshift-service-ca-operator] adding namespace took 5.420572ms I0403 00:41:29.285904 4001 namespace.go:102] [openshift-infra] adding namespace took 3.977652ms I0403 00:41:29.286328 4001 zone_ic_handler.go:186] Time taken to create transit switch: 312.027µs I0403 00:41:29.286406 4001 master.go:532] Adding or Updating local node "ip-10-0-1-98.ec2.internal" for network "default" I0403 00:41:29.286865 4001 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-61.ec2.internal for the network default I0403 00:41:29.286983 4001 iptables.go:108] Creating table: filter chain: OUTPUT I0403 00:41:29.287862 4001 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-61.ec2.internal" on network "default" took: 1.008518ms I0403 00:41:29.290218 4001 default_network_controller.go:806] Node add failed for ip-10-0-1-98.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-98.ec2.internal, error getting gateway config for node ip-10-0-1-98.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-98.ec2.internal", failed to update chassis to local for local node ip-10-0-1-98.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-98.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-98.ec2.internal] E0403 00:41:29.290269 4001 obj_retry.go:541] Failed to create *v1.Node ip-10-0-1-98.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-98.ec2.internal, error getting gateway config for node ip-10-0-1-98.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-98.ec2.internal", failed to update chassis to local for local node ip-10-0-1-98.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-98.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-98.ec2.internal] I0403 00:41:29.290326 4001 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0403 00:41:29.290351 4001 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0403 00:41:29.290344 4001 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-1-98.ec2.internal", UID:"d0ba06ae-8f2c-4056-9b3f-e22ffda51187", APIVersion:"v1", ResourceVersion:"16222", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-98.ec2.internal, error getting gateway config for node ip-10-0-1-98.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-98.ec2.internal", failed to update chassis to local for local node ip-10-0-1-98.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-98.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-98.ec2.internal] I0403 00:41:29.290365 4001 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0403 00:41:29.290380 4001 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-98.ec2.internal in network "default" I0403 00:41:29.290409 4001 node_tracker.go:235] Node ip-10-0-1-98.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-98.ec2.internal" I0403 00:41:29.290422 4001 node_tracker.go:169] Node ip-10-0-1-98.ec2.internal switch + router changed, syncing services in network "default" I0403 00:41:29.290430 4001 services_controller.go:574] Full service sync requested for network=default I0403 00:41:29.290438 4001 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-61.ec2.internal in network "default" I0403 00:41:29.290470 4001 node_tracker.go:169] Node ip-10-0-2-61.ec2.internal switch + router changed, syncing services in network "default" I0403 00:41:29.290477 4001 services_controller.go:574] Full service sync requested for network=default I0403 00:41:29.293304 4001 gateway_shared_intf.go:1787] Gateway Creation Complete I0403 00:41:29.293320 4001 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-98.ec2.internal","mac-address":"0a:ff:d9:c4:72:0d","ip-addresses":["10.0.1.98/24"],"ip-address":"10.0.1.98/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:66cf23b7-f7e1-49ea-85bb-19b23d5efa71 k8s.ovn.org/node-encap-ips:["10.0.1.98"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.98/24"} k8s.ovn.org/zone-name:ip-10-0-1-98.ec2.internal] on node ip-10-0-1-98.ec2.internal I0403 00:41:29.311261 4001 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-98.ec2.internal in network "default" I0403 00:41:29.311300 4001 master.go:532] Adding or Updating local node "ip-10-0-1-98.ec2.internal" for network "default" I0403 00:41:29.311326 4001 node_tracker.go:169] Node ip-10-0-1-98.ec2.internal switch + router changed, syncing services in network "default" I0403 00:41:29.311381 4001 services_controller.go:574] Full service sync requested for network=default I0403 00:41:29.314709 4001 default_node_network_controller.go:518] Node connection status = connected W0403 00:41:29.315993 4001 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-czv2x: pod kube-system/global-pull-secret-syncer-czv2x: no pod IPs found W0403 00:41:29.316011 4001 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-xd4g6: pod openshift-network-diagnostics/network-check-target-xd4g6: no pod IPs found W0403 00:41:29.316021 4001 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-psb2l: pod openshift-multus/network-metrics-daemon-psb2l: no pod IPs found I0403 00:41:29.316914 4001 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-1-98.ec2.internal for the network default I0403 00:41:29.391198 4001 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0403 00:41:29.391226 4001 services_controller.go:208] Setting up event handlers for services for network=default I0403 00:41:29.391317 4001 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0403 00:41:29.391360 4001 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0403 00:41:29.391375 4001 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0403 00:41:29.491617 4001 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0403 00:41:29.491638 4001 repair.go:60] Starting repairing loop for services I0403 00:41:29.491718 4001 repair.go:131] Deleted 0 stale service LBs I0403 00:41:29.491737 4001 repair.go:137] Deleted 0 stale Chassis Template Vars I0403 00:41:29.491754 4001 repair.go:62] Finished repairing loop for services: 118.484µs I0403 00:41:29.491778 4001 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0403 00:41:29.491788 4001 services_controller.go:254] Starting workers for network=default I0403 00:41:29.492141 4001 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-czv2x] creating logical port kube-system_global-pull-secret-syncer-czv2x for pod on switch ip-10-0-1-98.ec2.internal I0403 00:41:29.492189 4001 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-psb2l] creating logical port openshift-multus_network-metrics-daemon-psb2l for pod on switch ip-10-0-1-98.ec2.internal I0403 00:41:29.492207 4001 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-xd4g6] creating logical port openshift-network-diagnostics_network-check-target-xd4g6 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:41:29.492301 4001 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-czv2x I0403 00:41:29.492317 4001 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-psb2l I0403 00:41:29.492350 4001 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-xd4g6 I0403 00:41:29.508529 4001 pod.go:62] [openshift-network-diagnostics/network-check-target-xd4g6] pod update took 16.207955ms I0403 00:41:29.508546 4001 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-xd4g6] addLogicalPort annotation time took 16.232531ms I0403 00:41:29.509448 4001 pods.go:271] [openshift-network-diagnostics/network-check-target-xd4g6] addLogicalPort took 17.249691ms, libovsdb time 709.791µs I0403 00:41:29.512436 4001 pod.go:62] [openshift-multus/network-metrics-daemon-psb2l] pod update took 20.139394ms I0403 00:41:29.512457 4001 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-psb2l] addLogicalPort annotation time took 20.162149ms I0403 00:41:29.512842 4001 pod.go:62] [kube-system/global-pull-secret-syncer-czv2x] pod update took 20.615456ms I0403 00:41:29.512857 4001 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-czv2x] addLogicalPort annotation time took 20.636406ms I0403 00:41:29.513382 4001 pods.go:271] [openshift-multus/network-metrics-daemon-psb2l] addLogicalPort took 21.243737ms, libovsdb time 676.107µs I0403 00:41:29.513619 4001 pods.go:271] [kube-system/global-pull-secret-syncer-czv2x] addLogicalPort took 21.497631ms, libovsdb time 593.971µs I0403 00:41:29.513982 4001 admin_network_policy_controller.go:227] Starting controller default-network-controller I0403 00:41:29.514004 4001 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0403 00:41:29.514017 4001 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0403 00:41:29.514021 4001 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0403 00:41:29.514232 4001 repair.go:29] Repairing admin network policies took 206.086µs I0403 00:41:29.514329 4001 repair.go:92] Repairing baseline admin network policies took 88.499µs I0403 00:41:29.514661 4001 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0403 00:41:29.514684 4001 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0403 00:41:29.516718 4001 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0403 00:41:29.516742 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0403 00:41:29.516756 4001 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0403 00:41:29.518374 4001 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0403 00:41:29.518402 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0403 00:41:29.518418 4001 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0403 00:41:29.519969 4001 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0403 00:41:29.519988 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0403 00:41:29.520427 4001 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0403 00:41:29.520443 4001 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0403 00:41:29.522027 4001 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0403 00:41:29.522051 4001 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0403 00:41:29.522065 4001 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0403 00:41:29.523838 4001 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0403 00:41:29.523858 4001 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0403 00:41:29.523870 4001 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0403 00:41:29.524821 4001 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0403 00:41:29.524855 4001 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0403 00:41:29.524876 4001 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0403 00:41:29.526485 4001 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0403 00:41:29.526503 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0403 00:41:29.526512 4001 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0403 00:41:29.526981 4001 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0403 00:41:29.527004 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0403 00:41:29.527036 4001 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0403 00:41:29.527728 4001 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0403 00:41:29.527749 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0403 00:41:29.527757 4001 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0403 00:41:29.528246 4001 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0403 00:41:29.528270 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0403 00:41:29.528279 4001 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0403 00:41:29.528734 4001 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0403 00:41:29.528755 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0403 00:41:29.528765 4001 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0403 00:41:29.529219 4001 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0403 00:41:29.529243 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0403 00:41:29.529252 4001 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0403 00:41:29.529802 4001 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0403 00:41:29.529820 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0403 00:41:29.529827 4001 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0403 00:41:29.530285 4001 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0403 00:41:29.530308 4001 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0403 00:41:29.530321 4001 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0403 00:41:29.530821 4001 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0403 00:41:29.530839 4001 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0403 00:41:29.530848 4001 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0403 00:41:29.532632 4001 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0403 00:41:29.532649 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0403 00:41:29.532657 4001 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0403 00:41:29.533122 4001 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0403 00:41:29.533145 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0403 00:41:29.533155 4001 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0403 00:41:29.533644 4001 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0403 00:41:29.533667 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0403 00:41:29.533677 4001 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0403 00:41:29.533972 4001 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0403 00:41:29.533991 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0403 00:41:29.533998 4001 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0403 00:41:29.535187 4001 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0403 00:41:29.535216 4001 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0403 00:41:29.535227 4001 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0403 00:41:29.536645 4001 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0403 00:41:29.536666 4001 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0403 00:41:29.536676 4001 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0403 00:41:29.536939 4001 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0403 00:41:29.536963 4001 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0403 00:41:29.536974 4001 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0403 00:41:29.538056 4001 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0403 00:41:29.538077 4001 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0403 00:41:29.538104 4001 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0403 00:41:29.538389 4001 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0403 00:41:29.538407 4001 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0403 00:41:29.538415 4001 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0403 00:41:29.539890 4001 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0403 00:41:29.539907 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0403 00:41:29.539913 4001 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0403 00:41:29.540171 4001 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0403 00:41:29.540193 4001 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0403 00:41:29.540202 4001 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0403 00:41:29.540659 4001 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0403 00:41:29.540674 4001 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0403 00:41:29.540692 4001 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0403 00:41:29.541641 4001 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0403 00:41:29.541660 4001 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0403 00:41:29.541671 4001 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0403 00:41:29.542155 4001 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0403 00:41:29.546315 4001 egressfirewall.go:319] Starting EgressFirewall controller I0403 00:41:29.546346 4001 controller.go:133] Adding controller egress-firewall-controller event handlers I0403 00:41:29.546346 4001 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0403 00:41:29.546366 4001 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0403 00:41:29.546370 4001 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0403 00:41:29.546378 4001 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0403 00:41:29.546380 4001 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0403 00:41:29.546631 4001 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0403 00:41:29.546645 4001 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0403 00:41:29.546654 4001 egressqos.go:193] Setting up event handlers for EgressQoS I0403 00:41:29.546728 4001 egressqos.go:245] Starting EgressQoS Controller I0403 00:41:29.546732 4001 egressfirewall.go:628] Syncing node "ip-10-0-1-98.ec2.internal" for egress firewall I0403 00:41:29.546737 4001 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0403 00:41:29.546742 4001 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0403 00:41:29.546744 4001 egressfirewall.go:628] Syncing node "ip-10-0-2-61.ec2.internal" for egress firewall I0403 00:41:29.546748 4001 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0403 00:41:29.546752 4001 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0403 00:41:29.546758 4001 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0403 00:41:29.546763 4001 shared_informer.go:356] "Caches are synced" controller="egressqos" I0403 00:41:29.546766 4001 egressqos.go:259] Repairing EgressQoSes I0403 00:41:29.546770 4001 egressqos.go:399] Starting repairing loop for egressqos I0403 00:41:29.546832 4001 egressqos.go:401] Finished repairing loop for egressqos: 61.665µs I0403 00:41:29.546847 4001 egressservice_zone.go:132] Setting up event handlers for Egress Services I0403 00:41:29.546911 4001 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-98.ec2.internal I0403 00:41:29.546948 4001 egressservice_zone.go:210] Starting Egress Services Controller I0403 00:41:29.546957 4001 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0403 00:41:29.546963 4001 shared_informer.go:356] "Caches are synced" controller="egressservices" I0403 00:41:29.546969 4001 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0403 00:41:29.546975 4001 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0403 00:41:29.546980 4001 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0403 00:41:29.546979 4001 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-98.ec2.internal : 69.796µs I0403 00:41:29.546985 4001 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0403 00:41:29.546991 4001 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0403 00:41:29.546995 4001 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0403 00:41:29.546999 4001 egressservice_zone.go:228] Repairing Egress Services I0403 00:41:29.547268 4001 master_controller.go:88] Starting Admin Policy Based Route Controller I0403 00:41:29.547277 4001 external_controller.go:278] Starting Admin Policy Based Route Controller I0403 00:41:29.547341 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-98.ec2.internal I0403 00:41:29.547351 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-98.ec2.internal: 11.135µs I0403 00:41:29.547356 4001 default_network_controller.go:568] Completing all the Watchers took 297.190787ms I0403 00:41:29.547365 4001 default_network_controller.go:572] Starting unidling controllers I0403 00:41:29.547371 4001 unidle.go:46] Registering OVN SB ControllerEvent handler I0403 00:41:29.547379 4001 unidle.go:63] Populating Initial ContollerEvent events I0403 00:41:29.547393 4001 unidle.go:79] Setting up event handlers for services I0403 00:41:29.547358 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-61.ec2.internal I0403 00:41:29.547420 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-61.ec2.internal: 62.447µs I0403 00:41:29.547449 4001 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0403 00:41:29.815872 4001 default_node_network_controller.go:518] Node connection status = connected I0403 00:41:30.315316 4001 default_node_network_controller.go:518] Node connection status = connected I0403 00:41:30.760664 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-61.ec2.internal I0403 00:41:30.760684 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-61.ec2.internal: 26.831µs I0403 00:41:30.760663 4001 egressfirewall.go:628] Syncing node "ip-10-0-2-61.ec2.internal" for egress firewall I0403 00:41:30.760758 4001 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-61.ec2.internal in network "default" I0403 00:41:30.760809 4001 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-61.ec2.internal. Updating no re-route policies I0403 00:41:30.760812 4001 node_tracker.go:169] Node ip-10-0-2-61.ec2.internal switch + router changed, syncing services in network "default" I0403 00:41:30.760907 4001 services_controller.go:574] Full service sync requested for network=default I0403 00:41:30.815789 4001 default_node_network_controller.go:518] Node connection status = connected I0403 00:41:31.165342 4001 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-96.ec2.internal in network "default" I0403 00:41:31.165368 4001 node_tracker.go:220] Node ip-10-0-0-96.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0403 00:41:31.165445 4001 egressfirewall.go:628] Syncing node "ip-10-0-0-96.ec2.internal" for egress firewall I0403 00:41:31.165458 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-96.ec2.internal I0403 00:41:31.165470 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-96.ec2.internal: 15.512µs E0403 00:41:31.165491 4001 default_network_controller.go:816] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-96.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0403 00:41:31.165507 4001 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-96.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-96.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-96.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-96.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-96.ec2.internal, error parsing annotation for node ip-10-0-0-96.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0403 00:41:31.175271 4001 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-96.ec2.internal annotation: unexpected end of JSON input I0403 00:41:31.175310 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0403 00:41:31.175352 4001 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-96.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0403 00:41:31.187656 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-96.ec2.internal I0403 00:41:31.187675 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-96.ec2.internal: 21.421µs I0403 00:41:31.187656 4001 egressfirewall.go:628] Syncing node "ip-10-0-0-96.ec2.internal" for egress firewall E0403 00:41:31.187715 4001 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-96.ec2.internal annotation: unexpected end of JSON input I0403 00:41:31.187743 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0403 00:41:31.187769 4001 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-96.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0403 00:41:31.190594 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-61.ec2.internal I0403 00:41:31.190601 4001 egressfirewall.go:628] Syncing node "ip-10-0-2-61.ec2.internal" for egress firewall I0403 00:41:31.190611 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-61.ec2.internal: 16.996µs E0403 00:41:31.203416 4001 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-96.ec2.internal annotation: unexpected end of JSON input E0403 00:41:31.203444 4001 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-96.ec2.internal annotation: unexpected end of JSON input I0403 00:41:31.203461 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0403 00:41:31.203523 4001 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-96.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-96.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-96.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-96.ec2.internal" I0403 00:41:31.216700 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0403 00:41:31.315962 4001 default_node_network_controller.go:518] Node connection status = connected I0403 00:41:31.415537 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-96.ec2.internal I0403 00:41:31.415554 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-96.ec2.internal: 22.916µs I0403 00:41:31.415536 4001 egressfirewall.go:628] Syncing node "ip-10-0-0-96.ec2.internal" for egress firewall I0403 00:41:31.415616 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:41:31.429773 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:41:31.612718 4001 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0403 00:41:31.690647 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:41:31.816146 4001 default_node_network_controller.go:518] Node connection status = connected I0403 00:41:31.833385 4001 bridgeconfig.go:537] Gateway is ready I0403 00:41:31.833402 4001 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0403 00:41:31.844590 4001 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.98. Updating OVN encap IP. I0403 00:41:31.849629 4001 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.98 - it is already configured I0403 00:41:31.849642 4001 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.98/24:{} 172.20.0.1/32:{}]. Updating annotations. I0403 00:41:31.849873 4001 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.98/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-98.ec2.internal","mac-address":"0a:ff:d9:c4:72:0d","ip-addresses":["10.0.1.98/24"],"ip-address":"10.0.1.98/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:66cf23b7-f7e1-49ea-85bb-19b23d5efa71 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.98/24"}] on node ip-10-0-1-98.ec2.internal I0403 00:41:31.864482 4001 egressfirewall.go:628] Syncing node "ip-10-0-1-98.ec2.internal" for egress firewall I0403 00:41:31.864484 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-98.ec2.internal I0403 00:41:31.864509 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-98.ec2.internal: 29.096µs I0403 00:41:31.864523 4001 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-98.ec2.internal in network "default" I0403 00:41:31.864550 4001 master.go:532] Adding or Updating local node "ip-10-0-1-98.ec2.internal" for network "default" I0403 00:41:31.864579 4001 node_tracker.go:169] Node ip-10-0-1-98.ec2.internal switch + router changed, syncing services in network "default" I0403 00:41:31.864596 4001 services_controller.go:574] Full service sync requested for network=default I0403 00:41:31.864813 4001 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.1.98 I0403 00:41:31.864906 4001 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-98.ec2.internal I0403 00:41:31.868900 4001 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-98.ec2.internal. Updating no re-route policies I0403 00:41:31.868982 4001 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-7fstf I0403 00:41:31.868995 4001 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-fpx9x I0403 00:41:31.869009 4001 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-fpx9x I0403 00:41:31.869007 4001 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-czv2x I0403 00:41:31.869019 4001 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-czv2x I0403 00:41:31.869030 4001 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-9hj66 I0403 00:41:31.869054 4001 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-9hj66 I0403 00:41:31.869055 4001 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-kbnsx I0403 00:41:31.869062 4001 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-fpx9x after 0 failed attempt(s) I0403 00:41:31.869067 4001 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-q86p9 I0403 00:41:31.869068 4001 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-xd4g6 I0403 00:41:31.869074 4001 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-kbnsx I0403 00:41:31.869075 4001 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-q86p9 I0403 00:41:31.868998 4001 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-7fstf I0403 00:41:31.869097 4001 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-xd4g6 I0403 00:41:31.869100 4001 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-9hj66 after 0 failed attempt(s) I0403 00:41:31.869103 4001 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-psb2l I0403 00:41:31.869113 4001 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-7fstf after 0 failed attempt(s) I0403 00:41:31.869115 4001 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-q86p9 after 0 failed attempt(s) I0403 00:41:31.869118 4001 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-psb2l I0403 00:41:31.869118 4001 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-kbnsx after 0 failed attempt(s) I0403 00:41:31.869054 4001 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-czv2x] creating logical port kube-system_global-pull-secret-syncer-czv2x for pod on switch ip-10-0-1-98.ec2.internal I0403 00:41:31.869130 4001 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-xd4g6] creating logical port openshift-network-diagnostics_network-check-target-xd4g6 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:41:31.869147 4001 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-psb2l] creating logical port openshift-multus_network-metrics-daemon-psb2l for pod on switch ip-10-0-1-98.ec2.internal I0403 00:41:31.869169 4001 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-98.ec2.internal I0403 00:41:31.869180 4001 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-7w8kb I0403 00:41:31.869183 4001 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-98.ec2.internal I0403 00:41:31.869193 4001 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-7w8kb I0403 00:41:31.868998 4001 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-98.ec2.internal I0403 00:41:31.869206 4001 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-98.ec2.internal after 0 failed attempt(s) I0403 00:41:31.869209 4001 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-98.ec2.internal I0403 00:41:31.869055 4001 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-sjq2p I0403 00:41:31.868983 4001 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-w2btj I0403 00:41:31.869229 4001 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-w2btj I0403 00:41:31.869233 4001 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-7w8kb after 0 failed attempt(s) I0403 00:41:31.869236 4001 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-98.ec2.internal after 0 failed attempt(s) I0403 00:41:31.869078 4001 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-v4ggs I0403 00:41:31.869252 4001 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-w2btj after 0 failed attempt(s) I0403 00:41:31.869257 4001 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-v4ggs I0403 00:41:31.869219 4001 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-sjq2p I0403 00:41:31.869281 4001 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-v4ggs after 0 failed attempt(s) I0403 00:41:31.869282 4001 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-sjq2p after 0 failed attempt(s) I0403 00:41:31.869828 4001 pods.go:271] [kube-system/global-pull-secret-syncer-czv2x] addLogicalPort took 785.218µs, libovsdb time 254.643µs I0403 00:41:31.869841 4001 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-czv2x after 0 failed attempt(s) I0403 00:41:31.869918 4001 pods.go:271] [openshift-network-diagnostics/network-check-target-xd4g6] addLogicalPort took 795.822µs, libovsdb time 252.646µs I0403 00:41:31.869930 4001 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-xd4g6 after 0 failed attempt(s) I0403 00:41:31.869984 4001 pods.go:271] [openshift-multus/network-metrics-daemon-psb2l] addLogicalPort took 848.22µs, libovsdb time 310.59µs I0403 00:41:31.869992 4001 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-psb2l after 0 failed attempt(s) I0403 00:41:31.881253 4001 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0403 00:41:31.892316 4001 iptables.go:108] Creating table: mangle chain: OUTPUT I0403 00:41:31.895366 4001 iptables.go:108] Creating table: nat chain: OUTPUT I0403 00:41:31.929037 4001 iptables.go:108] Creating table: nat chain: PREROUTING I0403 00:41:31.987586 4001 iptables.go:108] Creating table: filter chain: FORWARD I0403 00:41:32.003000 4001 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0403 00:41:32.003118 4001 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0403 00:41:32.003163 4001 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0403 00:41:32.003235 4001 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0403 00:41:32.003281 4001 gateway_init.go:325] Gateway and management port readiness took 2.692483609s I0403 00:41:32.003307 4001 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0403 00:41:32.003343 4001 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0403 00:41:32.003356 4001 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0403 00:41:32.003365 4001 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0403 00:41:32.003418 4001 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0403 00:41:32.003508 4001 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 52.273µs I0403 00:41:32.003716 4001 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0403 00:41:32.003738 4001 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 14.01µs I0403 00:41:32.003747 4001 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 30.333µs I0403 00:41:32.003752 4001 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0403 00:41:32.003760 4001 nad_controller.go:173] [node-nad-controller NAD controller]: started I0403 00:41:32.003766 4001 default_node_network_controller.go:898] Starting the default node network controller I0403 00:41:32.003770 4001 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0403 00:41:32.003847 4001 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 12.113µs I0403 00:41:32.003910 4001 default_node_network_controller.go:1449] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0403 00:41:32.003918 4001 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0403 00:41:32.003939 4001 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a0b1f0 0xc0002ddd50 0xc0002dddc0 0xc0002dde30 0xc0002ddea0 0xc0002ddf10 0xc0002ec000 0xc000a23680 0xc0002ec070 0xc0002ec0e0 0xc000bd2190 0xc00641f260 0xc0002ec150 map[0x2f8f0c0:0xc0023931a0 0x2f8f600:0xc004b84de0 0x2f90080:0xc0012bc780 0x2f90320:0xc002393140 0x2f90b00:0xc002393200 0x2f90da0:0xc002393260 0x2f91040:0xc004b84d80 0x2f91820:0xc004b84e40 0x2f91ac0:0xc004b84ea0 0x2fbc3e0:0xc001427020 0x2fbc780:0xc001427080 0x2fbd260:0xc000a7b920 0x2fbebc0:0xc001b32000 0x2fbf6a0:0xc0012bc720 0x2fc0520:0xc001426fc0] 0xc000a24700 0} I0403 00:41:32.003989 4001 gateway.go:151] Starting gateway service sync I0403 00:41:32.004396 4001 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0403 00:41:32.006361 4001 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0403 00:41:32.017726 4001 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0403 00:41:32.018797 4001 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0403 00:41:32.019923 4001 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0403 00:41:32.053263 4001 gateway.go:168] Gateway service sync done. Time taken: 49.264914ms I0403 00:41:32.053853 4001 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31764 I0403 00:41:32.053986 4001 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a0b1f0 0xc0002ddd50 0xc0002dddc0 0xc0002dde30 0xc0002ddea0 0xc0002ddf10 0xc0002ec000 0xc000a23680 0xc0002ec070 0xc0002ec0e0 0xc000bd2190 0xc00641f260 0xc0002ec150 map[0x2f8f0c0:0xc0023931a0 0x2f8f600:0xc004b84de0 0x2f90080:0xc0012bc780 0x2f90320:0xc002393140 0x2f90b00:0xc002393200 0x2f90da0:0xc002393260 0x2f91040:0xc004b84d80 0x2f91820:0xc004b84e40 0x2f91ac0:0xc004b84ea0 0x2fbc3e0:0xc001427020 0x2fbc780:0xc001427080 0x2fbd260:0xc000a7b920 0x2fbebc0:0xc001b32000 0x2fbf6a0:0xc0012bc720 0x2fc0520:0xc001426fc0] 0xc000a24700 0} I0403 00:41:32.054438 4001 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a0b1f0 0xc0002ddd50 0xc0002dddc0 0xc0002dde30 0xc0002ddea0 0xc0002ddf10 0xc0002ec000 0xc000a23680 0xc0002ec070 0xc0002ec0e0 0xc000bd2190 0xc00641f260 0xc0002ec150 map[0x2f8f0c0:0xc0023931a0 0x2f8f600:0xc004b84de0 0x2f90080:0xc0012bc780 0x2f90320:0xc002393140 0x2f90b00:0xc002393200 0x2f90da0:0xc002393260 0x2f91040:0xc004b84d80 0x2f91820:0xc004b84e40 0x2f91ac0:0xc004b84ea0 0x2fbc3e0:0xc001427020 0x2fbc780:0xc001427080 0x2fbd260:0xc000a7b920 0x2fbebc0:0xc001b32000 0x2fbf6a0:0xc0012bc720 0x2fc0520:0xc001426fc0] 0xc000a24700 0} I0403 00:41:32.054472 4001 gateway.go:519] Reconciling gateway with updates I0403 00:41:32.059284 4001 gateway.go:326] Spawning Conntrack Rule Check Thread I0403 00:41:32.059357 4001 default_node_network_controller.go:943] Gateway and management port readiness took 55.43327ms I0403 00:41:32.059391 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0403 00:41:32.059409 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0403 00:41:32.059415 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0403 00:41:32.059430 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0403 00:41:32.059443 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0403 00:41:32.059443 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0403 00:41:32.059455 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0403 00:41:32.059457 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0403 00:41:32.059456 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0403 00:41:32.059466 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0403 00:41:32.059473 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0403 00:41:32.059483 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0403 00:41:32.059481 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0403 00:41:32.059494 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0403 00:41:32.059500 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0403 00:41:32.059501 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0403 00:41:32.059500 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0403 00:41:32.059513 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0403 00:41:32.059517 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0403 00:41:32.059517 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0403 00:41:32.059520 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0403 00:41:32.059527 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0403 00:41:32.059539 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0403 00:41:32.059543 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0403 00:41:32.059547 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0403 00:41:32.059560 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0403 00:41:32.059525 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0403 00:41:32.059574 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0403 00:41:32.059579 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0403 00:41:32.059582 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0403 00:41:32.059590 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0403 00:41:32.059596 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0403 00:41:32.059601 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0403 00:41:32.059606 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0403 00:41:32.059608 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0403 00:41:32.059531 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0403 00:41:32.059620 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0403 00:41:32.059622 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0403 00:41:32.059599 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0403 00:41:32.059633 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0403 00:41:32.059642 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0403 00:41:32.059625 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0403 00:41:32.059648 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0403 00:41:32.059586 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0403 00:41:32.059487 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0403 00:41:32.059684 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0403 00:41:32.059688 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0403 00:41:32.059694 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0403 00:41:32.059508 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0403 00:41:32.059703 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0403 00:41:32.059706 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0403 00:41:32.059592 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0403 00:41:32.059712 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0403 00:41:32.059718 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0403 00:41:32.059721 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0403 00:41:32.059723 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0403 00:41:32.059731 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0403 00:41:32.059736 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0403 00:41:32.059743 4001 port_claim.go:71] Svc openshift-ingress/router-default: port 32417 is already open, no action needed I0403 00:41:32.059749 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0403 00:41:32.059751 4001 port_claim.go:71] Svc openshift-ingress/router-default: port 30261 is already open, no action needed I0403 00:41:32.059753 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0403 00:41:32.059761 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0403 00:41:32.059762 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0403 00:41:32.059777 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0403 00:41:32.059787 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0403 00:41:32.059518 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0403 00:41:32.059804 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0403 00:41:32.059807 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0403 00:41:32.059820 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0403 00:41:32.059828 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0403 00:41:32.059842 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0403 00:41:32.059864 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0403 00:41:32.059878 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0403 00:41:32.059680 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0403 00:41:32.059909 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0403 00:41:32.059922 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0403 00:41:32.059580 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0403 00:41:32.059643 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0403 00:41:32.059920 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0403 00:41:32.059503 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0403 00:41:32.059499 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0403 00:41:32.060011 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0403 00:41:32.060019 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0403 00:41:32.059532 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0403 00:41:32.060023 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0403 00:41:32.060032 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0403 00:41:32.059680 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0403 00:41:32.059489 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0403 00:41:32.060048 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0403 00:41:32.059708 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0403 00:41:32.060058 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0403 00:41:32.059694 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0403 00:41:32.059712 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0403 00:41:32.060111 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0403 00:41:32.059739 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0403 00:41:32.059895 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0403 00:41:32.060218 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0403 00:41:32.059591 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0403 00:41:32.060283 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0403 00:41:32.060097 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0403 00:41:32.059724 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0403 00:41:32.060302 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0403 00:41:32.060309 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0403 00:41:32.060850 4001 node_ip_handler_linux.go:160] Node IP manager is running W0403 00:41:32.095818 4001 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.131.0.1 on link ovn-k8s-mp0 with MTU: 0 W0403 00:41:32.096025 4001 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.131.0.1 on link ovn-k8s-mp0 with MTU: 0 W0403 00:41:32.096209 4001 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:83:00:01/10.131.0.1) on link ovn-k8s-mp0 I0403 00:41:32.104589 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0403 00:41:32.104617 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0403 00:41:32.104624 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0403 00:41:32.104631 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0403 00:41:32.143931 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0403 00:41:32.143965 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0403 00:41:32.143973 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0403 00:41:32.143979 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0403 00:41:32.143985 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0403 00:41:32.143992 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0403 00:41:32.144020 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0403 00:41:32.144029 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0403 00:41:32.144035 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0403 00:41:32.144041 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0403 00:41:32.182643 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0403 00:41:32.182653 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0403 00:41:32.182671 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0403 00:41:32.182683 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0403 00:41:32.182690 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0403 00:41:32.182697 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0403 00:41:32.182705 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0403 00:41:32.182727 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0403 00:41:32.182739 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0403 00:41:32.182747 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0403 00:41:32.182752 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0403 00:41:32.182758 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0403 00:41:32.182765 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0403 00:41:32.182771 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0403 00:41:32.182778 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0403 00:41:32.182802 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0403 00:41:33.636606 4001 default_node_network_controller.go:1369] Starting node controller node sync W0403 00:41:33.636648 4001 egressip_healthcheck.go:76] Health checking using insecure connection I0403 00:41:33.636711 4001 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.131.0.2:9107 I0403 00:41:33.648468 4001 default_node_network_controller.go:1411] Node controller node sync done. Time taken: 11.841861ms I0403 00:41:33.648584 4001 default_node_network_controller.go:1282] Adding remote node "ip-10-0-2-61.ec2.internal", IP: 10.0.2.61 to PMTUD blocking rules I0403 00:41:33.648596 4001 default_node_network_controller.go:1282] Adding remote node "ip-10-0-2-61.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0403 00:41:33.654552 4001 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0403 00:41:33.654819 4001 egressservice_node.go:93] Setting up event handlers for Egress Services I0403 00:41:33.654920 4001 egressservice_node.go:183] Starting Egress Services Controller I0403 00:41:33.654935 4001 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0403 00:41:33.654944 4001 shared_informer.go:356] "Caches are synced" controller="egressservices" I0403 00:41:33.654950 4001 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0403 00:41:33.654955 4001 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0403 00:41:33.654960 4001 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0403 00:41:33.654965 4001 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0403 00:41:33.654969 4001 egressservice_node.go:197] Repairing Egress Services I0403 00:41:33.665305 4001 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0403 00:41:33.665316 4001 external_controller.go:278] Starting Admin Policy Based Route Controller I0403 00:41:33.665456 4001 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0403 00:41:33.665468 4001 link_network_manager.go:117] Link manager is running I0403 00:41:33.665477 4001 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0403 00:41:33.665554 4001 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0403 00:41:33.665882 4001 vrf_manager.go:128] VRF manager is running I0403 00:41:33.666027 4001 node_controller_manager.go:437] Removing flows to drop GARP I0403 00:41:33.666039 4001 gateway.go:519] Reconciling gateway with updates I0403 00:41:33.683980 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0403 00:41:33.683998 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0403 00:41:33.684009 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0403 00:41:33.684008 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0403 00:41:33.684028 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0403 00:41:33.684031 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0403 00:41:33.684039 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0403 00:41:33.684049 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0403 00:41:33.684047 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0403 00:41:33.684054 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0403 00:41:33.684058 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0403 00:41:33.684056 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0403 00:41:33.684073 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0403 00:41:33.684079 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0403 00:41:33.684036 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0403 00:41:33.684108 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0403 00:41:33.684107 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0403 00:41:33.684113 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0403 00:41:33.684120 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0403 00:41:33.684130 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0403 00:41:33.684133 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0403 00:41:33.684134 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0403 00:41:33.684120 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0403 00:41:33.684142 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0403 00:41:33.684147 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0403 00:41:33.684151 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0403 00:41:33.684136 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0403 00:41:33.684157 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0403 00:41:33.684160 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0403 00:41:33.684163 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0403 00:41:33.684183 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0403 00:41:33.684185 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0403 00:41:33.684191 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0403 00:41:33.684194 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0403 00:41:33.684197 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0403 00:41:33.684159 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0403 00:41:33.684200 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0403 00:41:33.684200 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0403 00:41:33.684217 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0403 00:41:33.684120 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0403 00:41:33.684222 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0403 00:41:33.684185 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0403 00:41:33.684233 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0403 00:41:33.684240 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0403 00:41:33.684241 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0403 00:41:33.684246 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0403 00:41:33.684166 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0403 00:41:33.683981 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0403 00:41:33.684264 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0403 00:41:33.684267 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0403 00:41:33.684272 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0403 00:41:33.684274 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0403 00:41:33.684193 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0403 00:41:33.684281 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0403 00:41:33.684284 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0403 00:41:33.684290 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0403 00:41:33.684292 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0403 00:41:33.684298 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0403 00:41:33.684217 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0403 00:41:33.684290 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0403 00:41:33.684320 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0403 00:41:33.684122 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0403 00:41:33.684331 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0403 00:41:33.684170 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0403 00:41:33.684171 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0403 00:41:33.684342 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0403 00:41:33.684216 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0403 00:41:33.684352 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0403 00:41:33.684359 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0403 00:41:33.684364 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0403 00:41:33.684286 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0403 00:41:33.684371 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0403 00:41:33.684228 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0403 00:41:33.684376 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0403 00:41:33.684380 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0403 00:41:33.684381 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0403 00:41:33.684386 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0403 00:41:33.684387 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0403 00:41:33.684391 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0403 00:41:33.684395 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0403 00:41:33.684395 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0403 00:41:33.684398 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0403 00:41:33.684399 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0403 00:41:33.684404 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0403 00:41:33.684407 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0403 00:41:33.684408 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0403 00:41:33.684413 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0403 00:41:33.684417 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0403 00:41:33.684417 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0403 00:41:33.684419 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0403 00:41:33.684380 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0403 00:41:33.684426 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0403 00:41:33.684428 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0403 00:41:33.684434 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0403 00:41:33.684449 4001 port_claim.go:71] Svc openshift-ingress/router-default: port 32417 is already open, no action needed I0403 00:41:33.684455 4001 port_claim.go:71] Svc openshift-ingress/router-default: port 30261 is already open, no action needed I0403 00:41:33.684137 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0403 00:41:33.684199 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0403 00:41:33.684492 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0403 00:41:33.684336 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0403 00:41:33.684366 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0403 00:41:33.684255 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0403 00:41:33.684181 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0403 00:41:33.684579 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0403 00:41:33.684389 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0403 00:41:33.684614 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0403 00:41:33.684393 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0403 00:41:33.684621 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0403 00:41:33.684403 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0403 00:41:33.684405 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0403 00:41:33.684694 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0403 00:41:33.684404 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0403 00:41:33.684727 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0403 00:41:33.684419 4001 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0403 00:41:33.684761 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0403 00:41:33.684768 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0403 00:41:33.684423 4001 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0403 00:41:33.684777 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0403 00:41:33.705861 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0403 00:41:33.705878 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0403 00:41:33.705885 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0403 00:41:33.705891 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0403 00:41:33.705897 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0403 00:41:33.705902 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0403 00:41:33.705908 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0403 00:41:33.744427 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0403 00:41:33.744433 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0403 00:41:33.744441 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0403 00:41:33.744451 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0403 00:41:33.744456 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0403 00:41:33.744464 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0403 00:41:33.744473 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0403 00:41:33.782068 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0403 00:41:33.782103 4001 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0403 00:41:34.745185 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-61.ec2.internal I0403 00:41:34.745218 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-61.ec2.internal: 37.78µs I0403 00:41:36.006331 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-98.ec2.internal I0403 00:41:36.006367 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-98.ec2.internal: 45.201µs I0403 00:41:36.032099 4001 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7f6f7df889-vr9sm] creating logical port openshift-image-registry_image-registry-7f6f7df889-vr9sm for pod on switch ip-10-0-1-98.ec2.internal I0403 00:41:36.032223 4001 kube.go:256] Updating pod openshift-image-registry/image-registry-7f6f7df889-vr9sm I0403 00:41:36.049570 4001 pod.go:62] [openshift-image-registry/image-registry-7f6f7df889-vr9sm] pod update took 17.371992ms I0403 00:41:36.049595 4001 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-7f6f7df889-vr9sm] addLogicalPort annotation time took 17.40139ms I0403 00:41:36.050902 4001 pods.go:271] [openshift-image-registry/image-registry-7f6f7df889-vr9sm] addLogicalPort took 18.839951ms, libovsdb time 1.00105ms I0403 00:41:36.058294 4001 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-kg72g] creating logical port openshift-dns_dns-default-kg72g for pod on switch ip-10-0-1-98.ec2.internal I0403 00:41:36.058407 4001 kube.go:256] Updating pod openshift-dns/dns-default-kg72g I0403 00:41:36.074269 4001 pod.go:62] [openshift-dns/dns-default-kg72g] pod update took 15.843615ms I0403 00:41:36.074293 4001 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-kg72g] addLogicalPort annotation time took 15.911401ms I0403 00:41:36.075533 4001 pods.go:271] [openshift-dns/dns-default-kg72g] addLogicalPort took 17.256551ms, libovsdb time 749.676µs I0403 00:41:36.075853 4001 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-ncpnk] creating logical port openshift-ingress-canary_ingress-canary-ncpnk for pod on switch ip-10-0-1-98.ec2.internal I0403 00:41:36.075923 4001 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-ncpnk I0403 00:41:36.088842 4001 pod.go:62] [openshift-ingress-canary/ingress-canary-ncpnk] pod update took 12.944527ms I0403 00:41:36.088859 4001 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-ncpnk] addLogicalPort annotation time took 12.96466ms I0403 00:41:36.089705 4001 pods.go:271] [openshift-ingress-canary/ingress-canary-ncpnk] addLogicalPort took 13.86009ms, libovsdb time 646.051µs I0403 00:41:36.448594 4001 cni.go:403] [openshift-ingress-canary/ingress-canary-ncpnk d6a187d46d88b8b2e997c3b93452784b09b2e432515aa094fd669d25261addee network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-ncpnk d6a187d46d88b8b2e997c3b93452784b09b2e432515aa094fd669d25261addee network default NAD default] I0403 00:41:36.481068 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-ncpnk, hostIfaceName: d6a187d46d88b8b, network: default, NAD default, SandboxID: "d6a187d46d88b8b2e997c3b93452784b09b2e432515aa094fd669d25261addee", PCI device ID: , UID: "c6cb7ad9-0923-4494-9f1e-60f08773b76f", MAC: 0a:58:0a:83:00:08, IPs: [10.131.0.8/23] I0403 00:41:36.539207 4001 cni.go:424] [openshift-ingress-canary/ingress-canary-ncpnk d6a187d46d88b8b2e997c3b93452784b09b2e432515aa094fd669d25261addee network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-ncpnk d6a187d46d88b8b2e997c3b93452784b09b2e432515aa094fd669d25261addee network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:8b:6d:91:38:7c\",\"name\":\"d6a187d46d88b8b\"},{\"mac\":\"0a:58:0a:83:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/41382fa0-351c-4506-a4d0-63004d88244c\"}],\"ips\":[{\"address\":\"10.131.0.8/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:41:39.125729 4001 cni.go:403] [kube-system/global-pull-secret-syncer-czv2x f71c1de898cbaafe5cf583797d924a04b335c6e8d0a3dbe1dd7612876f641f84 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-czv2x f71c1de898cbaafe5cf583797d924a04b335c6e8d0a3dbe1dd7612876f641f84 network default NAD default] I0403 00:41:39.135382 4001 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-czv2x, hostIfaceName: f71c1de898cbaaf, network: default, NAD default, SandboxID: "f71c1de898cbaafe5cf583797d924a04b335c6e8d0a3dbe1dd7612876f641f84", PCI device ID: , UID: "275c8e9d-6633-46ae-a9d8-b928eb99e55c", MAC: 0a:58:0a:83:00:03, IPs: [10.131.0.3/23] I0403 00:41:39.180197 4001 cni.go:424] [kube-system/global-pull-secret-syncer-czv2x f71c1de898cbaafe5cf583797d924a04b335c6e8d0a3dbe1dd7612876f641f84 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-czv2x f71c1de898cbaafe5cf583797d924a04b335c6e8d0a3dbe1dd7612876f641f84 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:15:66:65:3d:9a\",\"name\":\"f71c1de898cbaaf\"},{\"mac\":\"0a:58:0a:83:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0d319d5c-6a36-4c63-8a31-230182fd4d41\"}],\"ips\":[{\"address\":\"10.131.0.3/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:41:41.346749 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0403 00:41:42.102546 4001 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0403 00:41:46.131153 4001 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0403 00:41:47.090895 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:41:47.090921 4001 egressfirewall.go:628] Syncing node "ip-10-0-0-96.ec2.internal" for egress firewall I0403 00:41:47.090929 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-96.ec2.internal I0403 00:41:47.090944 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-96.ec2.internal: 18.356µs I0403 00:41:47.720566 4001 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js] creating logical port openshift-deployment-validation-operator_ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js for pod on switch ip-10-0-1-98.ec2.internal I0403 00:41:47.720655 4001 kube.go:256] Updating pod openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js I0403 00:41:47.733993 4001 pod.go:62] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js] pod update took 13.361101ms I0403 00:41:47.734014 4001 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js] addLogicalPort annotation time took 13.389479ms I0403 00:41:47.735176 4001 pods.go:271] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js] addLogicalPort took 14.632544ms, libovsdb time 877.946µs I0403 00:41:48.060901 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:41:48.076694 4001 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js af4c132a7426a0f7055ee7de385773ac2f420fd2f8dc7badaf01523d98e3e640 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js af4c132a7426a0f7055ee7de385773ac2f420fd2f8dc7badaf01523d98e3e640 network default NAD default] I0403 00:41:48.086144 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js, hostIfaceName: af4c132a7426a0f, network: default, NAD default, SandboxID: "af4c132a7426a0f7055ee7de385773ac2f420fd2f8dc7badaf01523d98e3e640", PCI device ID: , UID: "9638f5e1-c973-41b1-8ea6-ded09991b489", MAC: 0a:58:0a:83:00:09, IPs: [10.131.0.9/23] I0403 00:41:48.129766 4001 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js af4c132a7426a0f7055ee7de385773ac2f420fd2f8dc7badaf01523d98e3e640 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js af4c132a7426a0f7055ee7de385773ac2f420fd2f8dc7badaf01523d98e3e640 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:3c:d9:a4:50:60\",\"name\":\"af4c132a7426a0f\"},{\"mac\":\"0a:58:0a:83:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a9aa063a-57af-4e51-9d93-eb7dea6d1a06\"}],\"ips\":[{\"address\":\"10.131.0.9/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:41:54.348042 4001 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-96.ec2.internal in network "default" I0403 00:41:54.348097 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "ip-10-0-0-96.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:41:54.348101 4001 node_tracker.go:235] Node ip-10-0-0-96.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-96.ec2.internal" I0403 00:41:54.348168 4001 node_tracker.go:169] Node ip-10-0-0-96.ec2.internal switch + router changed, syncing services in network "default" I0403 00:41:54.348178 4001 services_controller.go:574] Full service sync requested for network=default W0403 00:41:54.348345 4001 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0403 00:41:54.378123 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "ip-10-0-0-96.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:41:54.506469 4001 default_network_controller.go:1008] Node "ip-10-0-0-96.ec2.internal" in remote zone "ip-10-0-0-96.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0403 00:41:54.506494 4001 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-96.ec2.internal in network "default" I0403 00:41:54.506545 4001 node_tracker.go:169] Node ip-10-0-0-96.ec2.internal switch + router changed, syncing services in network "default" I0403 00:41:54.506561 4001 services_controller.go:574] Full service sync requested for network=default W0403 00:41:54.506717 4001 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0403 00:41:54.507172 4001 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-96.ec2.internal for the network default I0403 00:41:54.508447 4001 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-96.ec2.internal" on network "default" took: 1.294413ms I0403 00:41:55.068439 4001 egressfirewall.go:628] Syncing node "ip-10-0-0-96.ec2.internal" for egress firewall I0403 00:41:55.068465 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-96.ec2.internal I0403 00:41:55.068479 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-96.ec2.internal: 15.998µs I0403 00:41:55.068500 4001 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-96.ec2.internal in network "default" I0403 00:41:55.068556 4001 node_tracker.go:169] Node ip-10-0-0-96.ec2.internal switch + router changed, syncing services in network "default" I0403 00:41:55.068574 4001 services_controller.go:574] Full service sync requested for network=default I0403 00:41:55.068607 4001 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-96.ec2.internal", IP: 10.0.0.96 to PMTUD blocking rules I0403 00:41:55.068623 4001 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-96.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0403 00:41:55.068822 4001 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0403 00:41:55.074747 4001 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-96.ec2.internal. Updating no re-route policies I0403 00:41:55.085128 4001 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-96.ec2.internal", IP: 10.0.0.96 to PMTUD blocking rules I0403 00:41:55.085142 4001 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-96.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0403 00:41:59.223860 4001 cni.go:403] [openshift-network-diagnostics/network-check-target-xd4g6 d33943972e40da3d1181f161b45e2d4fe1345dbe826575d5e6376033c942712a network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-xd4g6 d33943972e40da3d1181f161b45e2d4fe1345dbe826575d5e6376033c942712a network default NAD default] I0403 00:41:59.234098 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-xd4g6, hostIfaceName: d33943972e40da3, network: default, NAD default, SandboxID: "d33943972e40da3d1181f161b45e2d4fe1345dbe826575d5e6376033c942712a", PCI device ID: , UID: "5f715562-5702-45d3-a294-b635e80f1b08", MAC: 0a:58:0a:83:00:05, IPs: [10.131.0.5/23] I0403 00:41:59.281477 4001 cni.go:424] [openshift-network-diagnostics/network-check-target-xd4g6 d33943972e40da3d1181f161b45e2d4fe1345dbe826575d5e6376033c942712a network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-xd4g6 d33943972e40da3d1181f161b45e2d4fe1345dbe826575d5e6376033c942712a network default NAD default], result "{\"interfaces\":[{\"mac\":\"be:05:1a:6a:38:33\",\"name\":\"d33943972e40da3\"},{\"mac\":\"0a:58:0a:83:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/715671e5-c900-49ef-b5bc-ff45356b9e5e\"}],\"ips\":[{\"address\":\"10.131.0.5/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:41:59.428095 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-96.ec2.internal I0403 00:41:59.428131 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-96.ec2.internal: 58.296µs W0403 00:42:04.482765 4001 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0403 00:42:05.495456 4001 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js af4c132a7426a0f7055ee7de385773ac2f420fd2f8dc7badaf01523d98e3e640 network default NAD default] DEL starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js af4c132a7426a0f7055ee7de385773ac2f420fd2f8dc7badaf01523d98e3e640 network default NAD default] I0403 00:42:05.566078 4001 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js af4c132a7426a0f7055ee7de385773ac2f420fd2f8dc7badaf01523d98e3e640 network default NAD default] DEL finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js af4c132a7426a0f7055ee7de385773ac2f420fd2f8dc7badaf01523d98e3e640 network default NAD default], result "{}", err I0403 00:42:05.604360 4001 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:42:05.604393 4001 pods.go:176] Deleting pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js I0403 00:42:05.605723 4001 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js, ips: 10.131.0.9 I0403 00:42:05.605749 4001 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:42:05.605770 4001 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:42:05.605780 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0403 00:42:05.605965 4001 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:42:05.605981 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0403 00:42:06.628770 4001 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:42:06.628794 4001 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:42:06.628808 4001 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:42:06.628817 4001 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390d48js of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:42:07.788176 4001 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0403 00:42:07.797390 4001 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0403 00:42:08.255499 4001 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0403 00:42:18.124764 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0403 00:42:18.124807 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0403 00:42:35.489298 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0403 00:42:56.977819 4001 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-65b957665b-z62rh] creating logical port openshift-monitoring_cluster-monitoring-operator-65b957665b-z62rh for pod on switch ip-10-0-1-98.ec2.internal I0403 00:42:56.977905 4001 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-65b957665b-z62rh I0403 00:42:56.978243 4001 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-zvhzf] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-75b48889dc-zvhzf for pod on switch ip-10-0-1-98.ec2.internal I0403 00:42:56.978336 4001 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-zvhzf I0403 00:42:56.982919 4001 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6f4dc9bddb-dmzk9] creating logical port openshift-insights_insights-operator-6f4dc9bddb-dmzk9 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:42:56.982993 4001 kube.go:256] Updating pod openshift-insights/insights-operator-6f4dc9bddb-dmzk9 I0403 00:42:56.995555 4001 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-65b957665b-z62rh] pod update took 17.673666ms I0403 00:42:56.995578 4001 base_network_controller_pods.go:937] [default/openshift-monitoring/cluster-monitoring-operator-65b957665b-z62rh] addLogicalPort annotation time took 17.696906ms I0403 00:42:56.996953 4001 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-65b957665b-z62rh] addLogicalPort took 19.14641ms, libovsdb time 1.090504ms I0403 00:42:56.997017 4001 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-bh6rr] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-98c9d9db4-bh6rr for pod on switch ip-10-0-1-98.ec2.internal I0403 00:42:56.997115 4001 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-bh6rr I0403 00:42:56.998475 4001 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-zvhzf] pod update took 20.164094ms I0403 00:42:56.998497 4001 base_network_controller_pods.go:937] [default/openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-zvhzf] addLogicalPort annotation time took 20.190652ms I0403 00:42:56.999553 4001 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-zvhzf] addLogicalPort took 21.320431ms, libovsdb time 765.87µs I0403 00:42:56.999597 4001 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-vkxsq] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-d4cbcbb94-vkxsq for pod on switch ip-10-0-1-98.ec2.internal I0403 00:42:56.999646 4001 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-vkxsq I0403 00:42:57.001963 4001 pod.go:62] [openshift-insights/insights-operator-6f4dc9bddb-dmzk9] pod update took 18.986711ms I0403 00:42:57.001987 4001 base_network_controller_pods.go:937] [default/openshift-insights/insights-operator-6f4dc9bddb-dmzk9] addLogicalPort annotation time took 19.012129ms I0403 00:42:57.002996 4001 pods.go:271] [openshift-insights/insights-operator-6f4dc9bddb-dmzk9] addLogicalPort took 20.087755ms, libovsdb time 712.835µs I0403 00:42:57.011658 4001 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-bh6rr] pod update took 14.589733ms I0403 00:42:57.011677 4001 base_network_controller_pods.go:937] [default/openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-bh6rr] addLogicalPort annotation time took 14.612775ms I0403 00:42:57.012548 4001 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-bh6rr] addLogicalPort took 15.541307ms, libovsdb time 618.972µs I0403 00:42:57.017464 4001 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-vkxsq] pod update took 17.830389ms I0403 00:42:57.017482 4001 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-vkxsq] addLogicalPort annotation time took 17.84932ms I0403 00:42:57.018277 4001 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-vkxsq] addLogicalPort took 18.685325ms, libovsdb time 579.986µs I0403 00:42:57.070211 4001 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-6465cb84c-6vzfb] creating logical port openshift-network-console_networking-console-plugin-6465cb84c-6vzfb for pod on switch ip-10-0-1-98.ec2.internal I0403 00:42:57.070327 4001 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-6465cb84c-6vzfb I0403 00:42:57.073190 4001 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-5bff8f5d77-8pvzw] creating logical port openshift-network-diagnostics_network-check-source-5bff8f5d77-8pvzw for pod on switch ip-10-0-1-98.ec2.internal I0403 00:42:57.073274 4001 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-5bff8f5d77-8pvzw I0403 00:42:57.082566 4001 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5965f78f96-shgl4] creating logical port openshift-ingress_router-default-5965f78f96-shgl4 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:42:57.082646 4001 kube.go:256] Updating pod openshift-ingress/router-default-5965f78f96-shgl4 I0403 00:42:57.086099 4001 pod.go:62] [openshift-network-console/networking-console-plugin-6465cb84c-6vzfb] pod update took 15.778471ms I0403 00:42:57.086126 4001 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-6465cb84c-6vzfb] addLogicalPort annotation time took 15.827174ms I0403 00:42:57.087295 4001 pods.go:271] [openshift-network-console/networking-console-plugin-6465cb84c-6vzfb] addLogicalPort took 17.094283ms, libovsdb time 811.693µs I0403 00:42:57.092141 4001 pod.go:62] [openshift-network-diagnostics/network-check-source-5bff8f5d77-8pvzw] pod update took 18.883585ms I0403 00:42:57.092159 4001 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-source-5bff8f5d77-8pvzw] addLogicalPort annotation time took 18.903557ms I0403 00:42:57.093227 4001 pods.go:271] [openshift-network-diagnostics/network-check-source-5bff8f5d77-8pvzw] addLogicalPort took 20.044261ms, libovsdb time 762.139µs I0403 00:42:57.093275 4001 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5cf556f5bc-qkqjp] creating logical port openshift-ingress_router-default-5cf556f5bc-qkqjp for pod on switch ip-10-0-1-98.ec2.internal I0403 00:42:57.093329 4001 kube.go:256] Updating pod openshift-ingress/router-default-5cf556f5bc-qkqjp I0403 00:42:57.108473 4001 pod.go:62] [openshift-ingress/router-default-5965f78f96-shgl4] pod update took 25.857479ms I0403 00:42:57.108493 4001 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-5965f78f96-shgl4] addLogicalPort annotation time took 25.878825ms I0403 00:42:57.109560 4001 pods.go:271] [openshift-ingress/router-default-5965f78f96-shgl4] addLogicalPort took 27.006422ms, libovsdb time 772.22µs I0403 00:42:57.114099 4001 pod.go:62] [openshift-ingress/router-default-5cf556f5bc-qkqjp] pod update took 20.771198ms I0403 00:42:57.114134 4001 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-5cf556f5bc-qkqjp] addLogicalPort annotation time took 20.821998ms I0403 00:42:57.115110 4001 pods.go:271] [openshift-ingress/router-default-5cf556f5bc-qkqjp] addLogicalPort took 21.841118ms, libovsdb time 607.599µs I0403 00:42:57.346014 4001 cni.go:403] [openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-bh6rr eccef88d4d32df9b3110e929bde40fac80da48844b8e0182cd948b4ba91587e2 network default NAD default] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-bh6rr eccef88d4d32df9b3110e929bde40fac80da48844b8e0182cd948b4ba91587e2 network default NAD default] I0403 00:42:57.356426 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-98c9d9db4-bh6rr, hostIfaceName: eccef88d4d32df9, network: default, NAD default, SandboxID: "eccef88d4d32df9b3110e929bde40fac80da48844b8e0182cd948b4ba91587e2", PCI device ID: , UID: "cb06f578-6974-4098-87f7-3d464c27144f", MAC: 0a:58:0a:83:00:0d, IPs: [10.131.0.13/23] I0403 00:42:57.363737 4001 cni.go:403] [openshift-insights/insights-operator-6f4dc9bddb-dmzk9 ed58e5cdaa388b7c2eee863bebdf6496ec962bf6b38b33c733b8ffcfe766b23f network default NAD default] ADD starting CNI request [openshift-insights/insights-operator-6f4dc9bddb-dmzk9 ed58e5cdaa388b7c2eee863bebdf6496ec962bf6b38b33c733b8ffcfe766b23f network default NAD default] I0403 00:42:57.363863 4001 cni.go:403] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-vkxsq 35d59e85a1645cddc8dc6c2a31c636205366811c7ec75a1b50cd217d0cbd0f07 network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-vkxsq 35d59e85a1645cddc8dc6c2a31c636205366811c7ec75a1b50cd217d0cbd0f07 network default NAD default] I0403 00:42:57.377981 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-d4cbcbb94-vkxsq, hostIfaceName: 35d59e85a1645cd, network: default, NAD default, SandboxID: "35d59e85a1645cddc8dc6c2a31c636205366811c7ec75a1b50cd217d0cbd0f07", PCI device ID: , UID: "9fff42c1-cddf-42c7-92f3-12f8b15d8b27", MAC: 0a:58:0a:83:00:0e, IPs: [10.131.0.14/23] I0403 00:42:57.377996 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6f4dc9bddb-dmzk9, hostIfaceName: ed58e5cdaa388b7, network: default, NAD default, SandboxID: "ed58e5cdaa388b7c2eee863bebdf6496ec962bf6b38b33c733b8ffcfe766b23f", PCI device ID: , UID: "d371f774-606d-43e3-8db7-efe8028cdbea", MAC: 0a:58:0a:83:00:0c, IPs: [10.131.0.12/23] I0403 00:42:57.406716 4001 cni.go:424] [openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-bh6rr eccef88d4d32df9b3110e929bde40fac80da48844b8e0182cd948b4ba91587e2 network default NAD default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-98c9d9db4-bh6rr eccef88d4d32df9b3110e929bde40fac80da48844b8e0182cd948b4ba91587e2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:00:6e:68:90:24\",\"name\":\"eccef88d4d32df9\"},{\"mac\":\"0a:58:0a:83:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/49094448-3bac-4583-b70b-ced4eed41b81\"}],\"ips\":[{\"address\":\"10.131.0.13/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:42:57.427997 4001 cni.go:424] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-vkxsq 35d59e85a1645cddc8dc6c2a31c636205366811c7ec75a1b50cd217d0cbd0f07 network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-d4cbcbb94-vkxsq 35d59e85a1645cddc8dc6c2a31c636205366811c7ec75a1b50cd217d0cbd0f07 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:d9:70:2b:63:3e\",\"name\":\"35d59e85a1645cd\"},{\"mac\":\"0a:58:0a:83:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1f49e86c-8f15-46b2-81b6-2380b68c8b0c\"}],\"ips\":[{\"address\":\"10.131.0.14/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:42:57.441199 4001 cni.go:424] [openshift-insights/insights-operator-6f4dc9bddb-dmzk9 ed58e5cdaa388b7c2eee863bebdf6496ec962bf6b38b33c733b8ffcfe766b23f network default NAD default] ADD finished CNI request [openshift-insights/insights-operator-6f4dc9bddb-dmzk9 ed58e5cdaa388b7c2eee863bebdf6496ec962bf6b38b33c733b8ffcfe766b23f network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:95:e0:29:b3:79\",\"name\":\"ed58e5cdaa388b7\"},{\"mac\":\"0a:58:0a:83:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e2945025-f2b5-4e52-8937-3f952629348d\"}],\"ips\":[{\"address\":\"10.131.0.12/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:42:57.443070 4001 cni.go:403] [openshift-network-diagnostics/network-check-source-5bff8f5d77-8pvzw b14f3c32f8e42b21217fb188680c362467f1df27fdcaeaaae6ac84c78c09bea1 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-source-5bff8f5d77-8pvzw b14f3c32f8e42b21217fb188680c362467f1df27fdcaeaaae6ac84c78c09bea1 network default NAD default] I0403 00:42:57.457341 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-5bff8f5d77-8pvzw, hostIfaceName: b14f3c32f8e42b2, network: default, NAD default, SandboxID: "b14f3c32f8e42b21217fb188680c362467f1df27fdcaeaaae6ac84c78c09bea1", PCI device ID: , UID: "d969967f-9d50-4177-8cba-01fcae98d1d3", MAC: 0a:58:0a:83:00:10, IPs: [10.131.0.16/23] I0403 00:42:57.506225 4001 cni.go:424] [openshift-network-diagnostics/network-check-source-5bff8f5d77-8pvzw b14f3c32f8e42b21217fb188680c362467f1df27fdcaeaaae6ac84c78c09bea1 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-5bff8f5d77-8pvzw b14f3c32f8e42b21217fb188680c362467f1df27fdcaeaaae6ac84c78c09bea1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:53:09:1d:d2:f8\",\"name\":\"b14f3c32f8e42b2\"},{\"mac\":\"0a:58:0a:83:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/868364dc-444f-4d53-a5a2-5fe6b37db87e\"}],\"ips\":[{\"address\":\"10.131.0.16/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err W0403 00:43:00.320063 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0403 00:43:01.492784 4001 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0403 00:43:01.493729 4001 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 916.268µs I0403 00:43:01.515026 4001 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0403 00:43:01.890171 4001 namespace.go:98] [openshift-service-ca] adding namespace I0403 00:43:01.890991 4001 namespace.go:102] [openshift-service-ca] adding namespace took 790.745µs I0403 00:43:01.907175 4001 namespace.go:142] [openshift-service-ca] updating namespace W0403 00:43:09.184518 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0403 00:43:12.948371 4001 cni.go:403] [openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-zvhzf 39ed1b256c718663a2f741ea866d4618421714686f9305d78ec8817a731c8f1c network default NAD default] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-zvhzf 39ed1b256c718663a2f741ea866d4618421714686f9305d78ec8817a731c8f1c network default NAD default] I0403 00:43:12.957921 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-75b48889dc-zvhzf, hostIfaceName: 39ed1b256c71866, network: default, NAD default, SandboxID: "39ed1b256c718663a2f741ea866d4618421714686f9305d78ec8817a731c8f1c", PCI device ID: , UID: "8155715f-6f80-4eb6-9738-14b4868452e1", MAC: 0a:58:0a:83:00:0b, IPs: [10.131.0.11/23] I0403 00:43:13.003208 4001 cni.go:424] [openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-zvhzf 39ed1b256c718663a2f741ea866d4618421714686f9305d78ec8817a731c8f1c network default NAD default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-75b48889dc-zvhzf 39ed1b256c718663a2f741ea866d4618421714686f9305d78ec8817a731c8f1c network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:92:e2:ef:a5:91\",\"name\":\"39ed1b256c71866\"},{\"mac\":\"0a:58:0a:83:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0c3ec00c-c903-4963-a941-10ce66f503ae\"}],\"ips\":[{\"address\":\"10.131.0.11/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:43:23.483245 4001 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-bns25] creating logical port openshift-insights_insights-runtime-extractor-bns25 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:43:23.483352 4001 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-bns25 I0403 00:43:23.504077 4001 pod.go:62] [openshift-insights/insights-runtime-extractor-bns25] pod update took 20.753659ms I0403 00:43:23.504114 4001 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-bns25] addLogicalPort annotation time took 20.795836ms I0403 00:43:23.505142 4001 pods.go:271] [openshift-insights/insights-runtime-extractor-bns25] addLogicalPort took 21.913828ms, libovsdb time 756.969µs I0403 00:43:23.841715 4001 cni.go:403] [openshift-insights/insights-runtime-extractor-bns25 346408f780dc1577e5ec89395b6b251d2ffbe9c4dfa506b17e74d71744021d18 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-bns25 346408f780dc1577e5ec89395b6b251d2ffbe9c4dfa506b17e74d71744021d18 network default NAD default] I0403 00:43:23.850699 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-bns25, hostIfaceName: 346408f780dc157, network: default, NAD default, SandboxID: "346408f780dc1577e5ec89395b6b251d2ffbe9c4dfa506b17e74d71744021d18", PCI device ID: , UID: "4c788531-e28c-4527-a8b8-e4929702a68b", MAC: 0a:58:0a:83:00:13, IPs: [10.131.0.19/23] I0403 00:43:23.897836 4001 cni.go:424] [openshift-insights/insights-runtime-extractor-bns25 346408f780dc1577e5ec89395b6b251d2ffbe9c4dfa506b17e74d71744021d18 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-bns25 346408f780dc1577e5ec89395b6b251d2ffbe9c4dfa506b17e74d71744021d18 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:0a:ec:2f:f6:f9\",\"name\":\"346408f780dc157\"},{\"mac\":\"0a:58:0a:83:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e15560a0-a6ed-496e-ae2b-a99270c10496\"}],\"ips\":[{\"address\":\"10.131.0.19/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:43:24.753072 4001 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7f6f7df889-vr9sm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:43:24.753127 4001 pods.go:176] Deleting pod: openshift-image-registry/image-registry-7f6f7df889-vr9sm I0403 00:43:24.754384 4001 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-7f6f7df889-vr9sm, ips: 10.131.0.6 I0403 00:43:24.754410 4001 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7f6f7df889-vr9sm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:43:24.771352 4001 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-cf8f9c488-rv7mw] creating logical port openshift-image-registry_image-registry-cf8f9c488-rv7mw for pod on switch ip-10-0-1-98.ec2.internal I0403 00:43:24.771432 4001 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7f6f7df889-vr9sm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:43:24.771454 4001 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7f6f7df889-vr9sm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:43:24.771464 4001 kube.go:256] Updating pod openshift-image-registry/image-registry-cf8f9c488-rv7mw I0403 00:43:24.779410 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7f6f7df889-vr9sm I0403 00:43:24.779436 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7f6f7df889-vr9sm I0403 00:43:24.786266 4001 pod.go:62] [openshift-image-registry/image-registry-cf8f9c488-rv7mw] pod update took 14.84084ms I0403 00:43:24.786286 4001 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-cf8f9c488-rv7mw] addLogicalPort annotation time took 14.867857ms I0403 00:43:24.787378 4001 pods.go:271] [openshift-image-registry/image-registry-cf8f9c488-rv7mw] addLogicalPort took 16.045258ms, libovsdb time 777.52µs I0403 00:43:25.126676 4001 cni.go:403] [openshift-image-registry/image-registry-cf8f9c488-rv7mw 5d7b6d573f750a37bc6627e1da597b3adfc3202e862021e069b94f7ccdff9aac network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-cf8f9c488-rv7mw 5d7b6d573f750a37bc6627e1da597b3adfc3202e862021e069b94f7ccdff9aac network default NAD default] I0403 00:43:25.136128 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-cf8f9c488-rv7mw, hostIfaceName: 5d7b6d573f750a3, network: default, NAD default, SandboxID: "5d7b6d573f750a37bc6627e1da597b3adfc3202e862021e069b94f7ccdff9aac", PCI device ID: , UID: "c539f09a-99e1-4ce4-9540-875efdd00272", MAC: 0a:58:0a:83:00:14, IPs: [10.131.0.20/23] I0403 00:43:25.180534 4001 cni.go:424] [openshift-image-registry/image-registry-cf8f9c488-rv7mw 5d7b6d573f750a37bc6627e1da597b3adfc3202e862021e069b94f7ccdff9aac network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-cf8f9c488-rv7mw 5d7b6d573f750a37bc6627e1da597b3adfc3202e862021e069b94f7ccdff9aac network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:c7:15:a0:d8:38\",\"name\":\"5d7b6d573f750a3\"},{\"mac\":\"0a:58:0a:83:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/225bd785-44e1-43f3-bf65-8e0ef0c75ce3\"}],\"ips\":[{\"address\":\"10.131.0.20/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:43:25.398394 4001 obj_retry.go:464] Detected object openshift-image-registry/image-registry-54ff97fc46-kngvd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:43:25.398963 4001 obj_retry.go:464] Detected object openshift-image-registry/image-registry-54ff97fc46-kngvd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:43:25.412373 4001 obj_retry.go:458] Detected object openshift-image-registry/image-registry-54ff97fc46-kngvd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:43:25.412395 4001 obj_retry.go:458] Detected object openshift-image-registry/image-registry-54ff97fc46-kngvd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:43:25.419229 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-54ff97fc46-kngvd I0403 00:43:25.419252 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-54ff97fc46-kngvd W0403 00:43:25.743803 4001 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0403 00:43:26.402698 4001 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0403 00:43:29.138940 4001 cni.go:403] [openshift-monitoring/cluster-monitoring-operator-65b957665b-z62rh b4b999431f72d27f96199c5a6212df31308b4d88225a4d986b7f115ce665b8e4 network default NAD default] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-65b957665b-z62rh b4b999431f72d27f96199c5a6212df31308b4d88225a4d986b7f115ce665b8e4 network default NAD default] I0403 00:43:29.148168 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-65b957665b-z62rh, hostIfaceName: b4b999431f72d27, network: default, NAD default, SandboxID: "b4b999431f72d27f96199c5a6212df31308b4d88225a4d986b7f115ce665b8e4", PCI device ID: , UID: "5ea31eef-482c-45bd-a900-d5da21690111", MAC: 0a:58:0a:83:00:0a, IPs: [10.131.0.10/23] I0403 00:43:29.194459 4001 cni.go:424] [openshift-monitoring/cluster-monitoring-operator-65b957665b-z62rh b4b999431f72d27f96199c5a6212df31308b4d88225a4d986b7f115ce665b8e4 network default NAD default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-65b957665b-z62rh b4b999431f72d27f96199c5a6212df31308b4d88225a4d986b7f115ce665b8e4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:c0:1b:a2:ee:91\",\"name\":\"b4b999431f72d27\"},{\"mac\":\"0a:58:0a:83:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d35ef833-2897-4ddf-9525-eb174c41e625\"}],\"ips\":[{\"address\":\"10.131.0.10/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:43:29.239430 4001 cni.go:403] [openshift-network-console/networking-console-plugin-6465cb84c-6vzfb d4e726f8839f88af1d4294d75b0e3b93605f0e87591250b5673b93f352f9618e network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-6465cb84c-6vzfb d4e726f8839f88af1d4294d75b0e3b93605f0e87591250b5673b93f352f9618e network default NAD default] I0403 00:43:29.244175 4001 cni.go:403] [openshift-ingress/router-default-5cf556f5bc-qkqjp 09bf1629b194d8fef38bab7a2583201f2e677cebdbe5a085fe67f8955110da34 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-5cf556f5bc-qkqjp 09bf1629b194d8fef38bab7a2583201f2e677cebdbe5a085fe67f8955110da34 network default NAD default] I0403 00:43:29.249733 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-6465cb84c-6vzfb, hostIfaceName: d4e726f8839f88a, network: default, NAD default, SandboxID: "d4e726f8839f88af1d4294d75b0e3b93605f0e87591250b5673b93f352f9618e", PCI device ID: , UID: "5cd3447c-3227-46ee-b4f3-1ea107b4fef9", MAC: 0a:58:0a:83:00:0f, IPs: [10.131.0.15/23] I0403 00:43:29.255898 4001 cni.go:403] [openshift-ingress/router-default-5965f78f96-shgl4 3bd2ab51e0eb1b91c9033a85ce1eaa5e51ab86e6272612c2a52a8f7dc1cfe09e network default NAD default] ADD starting CNI request [openshift-ingress/router-default-5965f78f96-shgl4 3bd2ab51e0eb1b91c9033a85ce1eaa5e51ab86e6272612c2a52a8f7dc1cfe09e network default NAD default] I0403 00:43:29.256359 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5cf556f5bc-qkqjp, hostIfaceName: 09bf1629b194d8f, network: default, NAD default, SandboxID: "09bf1629b194d8fef38bab7a2583201f2e677cebdbe5a085fe67f8955110da34", PCI device ID: , UID: "0612fbf8-db30-4693-9027-d80e83f778f0", MAC: 0a:58:0a:83:00:12, IPs: [10.131.0.18/23] I0403 00:43:29.266487 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5965f78f96-shgl4, hostIfaceName: 3bd2ab51e0eb1b9, network: default, NAD default, SandboxID: "3bd2ab51e0eb1b91c9033a85ce1eaa5e51ab86e6272612c2a52a8f7dc1cfe09e", PCI device ID: , UID: "5df50982-7a4a-427e-beea-58fcf9878774", MAC: 0a:58:0a:83:00:11, IPs: [10.131.0.17/23] I0403 00:43:29.297786 4001 cni.go:424] [openshift-network-console/networking-console-plugin-6465cb84c-6vzfb d4e726f8839f88af1d4294d75b0e3b93605f0e87591250b5673b93f352f9618e network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-6465cb84c-6vzfb d4e726f8839f88af1d4294d75b0e3b93605f0e87591250b5673b93f352f9618e network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:6d:b6:5a:f6:95\",\"name\":\"d4e726f8839f88a\"},{\"mac\":\"0a:58:0a:83:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/46da5037-f434-4835-8a61-c946795bc0e7\"}],\"ips\":[{\"address\":\"10.131.0.15/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:43:29.523126 4001 cni.go:424] [openshift-ingress/router-default-5cf556f5bc-qkqjp 09bf1629b194d8fef38bab7a2583201f2e677cebdbe5a085fe67f8955110da34 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-5cf556f5bc-qkqjp 09bf1629b194d8fef38bab7a2583201f2e677cebdbe5a085fe67f8955110da34 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:a3:47:fa:c5:3a\",\"name\":\"09bf1629b194d8f\"},{\"mac\":\"0a:58:0a:83:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7b3b22cc-ceb1-4248-ae85-61e053ec8bf9\"}],\"ips\":[{\"address\":\"10.131.0.18/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:43:29.523129 4001 cni.go:424] [openshift-ingress/router-default-5965f78f96-shgl4 3bd2ab51e0eb1b91c9033a85ce1eaa5e51ab86e6272612c2a52a8f7dc1cfe09e network default NAD default] ADD finished CNI request [openshift-ingress/router-default-5965f78f96-shgl4 3bd2ab51e0eb1b91c9033a85ce1eaa5e51ab86e6272612c2a52a8f7dc1cfe09e network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:b0:93:87:cb:4c\",\"name\":\"3bd2ab51e0eb1b9\"},{\"mac\":\"0a:58:0a:83:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2418fa11-7ef0-427d-8b78-51e91158b735\"}],\"ips\":[{\"address\":\"10.131.0.17/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err W0403 00:43:29.755271 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0403 00:43:29.755290 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0403 00:43:29.755986 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0403 00:43:29.756000 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0403 00:43:29.756009 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0403 00:43:29.776885 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0403 00:43:29.776904 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0403 00:43:29.777910 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0403 00:43:29.777921 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0403 00:43:29.777926 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0403 00:43:30.420808 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0403 00:43:30.420825 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0403 00:43:30.425405 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0403 00:43:30.425422 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0403 00:43:30.425428 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0403 00:43:30.789440 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0403 00:43:30.789466 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0403 00:43:30.884949 4001 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0403 00:43:30.897745 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0403 00:43:30.897767 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0403 00:43:30.897776 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default I0403 00:43:31.006903 4001 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0403 00:43:32.209692 4001 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-r6j9l] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-7464b9f894-r6j9l for pod on switch ip-10-0-1-98.ec2.internal I0403 00:43:32.209780 4001 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-r6j9l I0403 00:43:32.221984 4001 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-r6j9l] pod update took 12.222955ms I0403 00:43:32.222013 4001 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-r6j9l] addLogicalPort annotation time took 12.25563ms I0403 00:43:32.223413 4001 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-r6j9l] addLogicalPort took 13.736564ms, libovsdb time 980.522µs I0403 00:43:32.567051 4001 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-r6j9l 678a4f88de8c90834dbe8d54faea8174d9dc035c52b05fc801e7f203116d890a network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-r6j9l 678a4f88de8c90834dbe8d54faea8174d9dc035c52b05fc801e7f203116d890a network default NAD default] I0403 00:43:32.583514 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-7464b9f894-r6j9l, hostIfaceName: 678a4f88de8c908, network: default, NAD default, SandboxID: "678a4f88de8c90834dbe8d54faea8174d9dc035c52b05fc801e7f203116d890a", PCI device ID: , UID: "6220d369-1246-4e08-9455-376d972e2cb2", MAC: 0a:58:0a:83:00:15, IPs: [10.131.0.21/23] I0403 00:43:32.632852 4001 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-r6j9l 678a4f88de8c90834dbe8d54faea8174d9dc035c52b05fc801e7f203116d890a network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-r6j9l 678a4f88de8c90834dbe8d54faea8174d9dc035c52b05fc801e7f203116d890a network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:09:e3:08:4d:4d\",\"name\":\"678a4f88de8c908\"},{\"mac\":\"0a:58:0a:83:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/40efaf61-ad90-4d2e-bee7-6880e1db22b5\"}],\"ips\":[{\"address\":\"10.131.0.21/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err W0403 00:43:34.471479 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0403 00:43:34.800526 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0403 00:43:40.302237 4001 base_network_controller_pods.go:487] [default/openshift-monitoring/kube-state-metrics-794f95bd5b-k77gs] creating logical port openshift-monitoring_kube-state-metrics-794f95bd5b-k77gs for pod on switch ip-10-0-1-98.ec2.internal I0403 00:43:40.302328 4001 kube.go:256] Updating pod openshift-monitoring/kube-state-metrics-794f95bd5b-k77gs I0403 00:43:40.318961 4001 pod.go:62] [openshift-monitoring/kube-state-metrics-794f95bd5b-k77gs] pod update took 16.654211ms I0403 00:43:40.318989 4001 base_network_controller_pods.go:937] [default/openshift-monitoring/kube-state-metrics-794f95bd5b-k77gs] addLogicalPort annotation time took 16.690394ms I0403 00:43:40.320150 4001 pods.go:271] [openshift-monitoring/kube-state-metrics-794f95bd5b-k77gs] addLogicalPort took 17.930201ms, libovsdb time 843.659µs I0403 00:43:40.663782 4001 cni.go:403] [openshift-monitoring/kube-state-metrics-794f95bd5b-k77gs 3722ed816cc1f253627f68d470489d0fdca2f71688e17bb1ebee2af2f01ae384 network default NAD default] ADD starting CNI request [openshift-monitoring/kube-state-metrics-794f95bd5b-k77gs 3722ed816cc1f253627f68d470489d0fdca2f71688e17bb1ebee2af2f01ae384 network default NAD default] I0403 00:43:40.673351 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: kube-state-metrics-794f95bd5b-k77gs, hostIfaceName: 3722ed816cc1f25, network: default, NAD default, SandboxID: "3722ed816cc1f253627f68d470489d0fdca2f71688e17bb1ebee2af2f01ae384", PCI device ID: , UID: "c5ca5c5f-9ba4-49be-bff7-6c2b9156419f", MAC: 0a:58:0a:83:00:16, IPs: [10.131.0.22/23] I0403 00:43:40.723876 4001 cni.go:424] [openshift-monitoring/kube-state-metrics-794f95bd5b-k77gs 3722ed816cc1f253627f68d470489d0fdca2f71688e17bb1ebee2af2f01ae384 network default NAD default] ADD finished CNI request [openshift-monitoring/kube-state-metrics-794f95bd5b-k77gs 3722ed816cc1f253627f68d470489d0fdca2f71688e17bb1ebee2af2f01ae384 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:a8:91:85:79:8a\",\"name\":\"3722ed816cc1f25\"},{\"mac\":\"0a:58:0a:83:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/65b40efc-2c9e-40d6-8ed5-e54951d9665a\"}],\"ips\":[{\"address\":\"10.131.0.22/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:43:41.055514 4001 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:43:41.055630 4001 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0403 00:43:41.095118 4001 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 39.525289ms I0403 00:43:41.095143 4001 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 39.555527ms I0403 00:43:41.096416 4001 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 40.918443ms, libovsdb time 939.3µs I0403 00:43:42.173004 4001 cni.go:403] [openshift-monitoring/alertmanager-main-0 6ea0498045cd26c85478b6362f0b02c1c89eb22cc23fb19cafb04c27543bb396 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 6ea0498045cd26c85478b6362f0b02c1c89eb22cc23fb19cafb04c27543bb396 network default NAD default] I0403 00:43:42.183371 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 6ea0498045cd26c, network: default, NAD default, SandboxID: "6ea0498045cd26c85478b6362f0b02c1c89eb22cc23fb19cafb04c27543bb396", PCI device ID: , UID: "ff2cbe82-b5ff-4cbe-9b40-42369aaee1df", MAC: 0a:58:0a:83:00:17, IPs: [10.131.0.23/23] I0403 00:43:42.234345 4001 cni.go:424] [openshift-monitoring/alertmanager-main-0 6ea0498045cd26c85478b6362f0b02c1c89eb22cc23fb19cafb04c27543bb396 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 6ea0498045cd26c85478b6362f0b02c1c89eb22cc23fb19cafb04c27543bb396 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:ce:44:b4:30:b0\",\"name\":\"6ea0498045cd26c\"},{\"mac\":\"0a:58:0a:83:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e39d5d3c-0d70-4531-8d50-45dc08c9266f\"}],\"ips\":[{\"address\":\"10.131.0.23/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:43:42.904368 4001 base_network_controller_pods.go:487] [default/openshift-console/downloads-744b9bb959-c8ns7] creating logical port openshift-console_downloads-744b9bb959-c8ns7 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:43:42.904458 4001 kube.go:256] Updating pod openshift-console/downloads-744b9bb959-c8ns7 I0403 00:43:42.918007 4001 pod.go:62] [openshift-console/downloads-744b9bb959-c8ns7] pod update took 13.572047ms I0403 00:43:42.918038 4001 base_network_controller_pods.go:937] [default/openshift-console/downloads-744b9bb959-c8ns7] addLogicalPort annotation time took 13.609234ms I0403 00:43:42.919690 4001 pods.go:271] [openshift-console/downloads-744b9bb959-c8ns7] addLogicalPort took 15.337105ms, libovsdb time 1.15822ms I0403 00:43:43.020724 4001 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-6854ff8d7-xnw2d] creating logical port openshift-monitoring_thanos-querier-6854ff8d7-xnw2d for pod on switch ip-10-0-1-98.ec2.internal I0403 00:43:43.020855 4001 kube.go:256] Updating pod openshift-monitoring/thanos-querier-6854ff8d7-xnw2d I0403 00:43:43.040927 4001 pod.go:62] [openshift-monitoring/thanos-querier-6854ff8d7-xnw2d] pod update took 20.097656ms I0403 00:43:43.040957 4001 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-6854ff8d7-xnw2d] addLogicalPort annotation time took 20.132579ms I0403 00:43:43.042202 4001 pods.go:271] [openshift-monitoring/thanos-querier-6854ff8d7-xnw2d] addLogicalPort took 21.498052ms, libovsdb time 868.93µs I0403 00:43:43.268030 4001 cni.go:403] [openshift-console/downloads-744b9bb959-c8ns7 64278722908b126a20a08d40fd68e997b653c4f86be0aafd3cdb0c78b1f0923d network default NAD default] ADD starting CNI request [openshift-console/downloads-744b9bb959-c8ns7 64278722908b126a20a08d40fd68e997b653c4f86be0aafd3cdb0c78b1f0923d network default NAD default] I0403 00:43:43.278002 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-744b9bb959-c8ns7, hostIfaceName: 64278722908b126, network: default, NAD default, SandboxID: "64278722908b126a20a08d40fd68e997b653c4f86be0aafd3cdb0c78b1f0923d", PCI device ID: , UID: "16cf49da-cda6-4521-9a42-a4d66ad81801", MAC: 0a:58:0a:83:00:18, IPs: [10.131.0.24/23] I0403 00:43:43.327919 4001 cni.go:424] [openshift-console/downloads-744b9bb959-c8ns7 64278722908b126a20a08d40fd68e997b653c4f86be0aafd3cdb0c78b1f0923d network default NAD default] ADD finished CNI request [openshift-console/downloads-744b9bb959-c8ns7 64278722908b126a20a08d40fd68e997b653c4f86be0aafd3cdb0c78b1f0923d network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:31:92:53:39:7c\",\"name\":\"64278722908b126\"},{\"mac\":\"0a:58:0a:83:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/af5cd50e-24ef-44ba-b94f-d0942e5c4aad\"}],\"ips\":[{\"address\":\"10.131.0.24/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:43:43.382479 4001 cni.go:403] [openshift-monitoring/thanos-querier-6854ff8d7-xnw2d 61a4f79a12e2e77d8fb8d3ac2bd99d4a35b54ee8a71c8b3acd14da9e2a05e86e network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-6854ff8d7-xnw2d 61a4f79a12e2e77d8fb8d3ac2bd99d4a35b54ee8a71c8b3acd14da9e2a05e86e network default NAD default] I0403 00:43:43.391893 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-6854ff8d7-xnw2d, hostIfaceName: 61a4f79a12e2e77, network: default, NAD default, SandboxID: "61a4f79a12e2e77d8fb8d3ac2bd99d4a35b54ee8a71c8b3acd14da9e2a05e86e", PCI device ID: , UID: "355eff62-5d19-4f44-83ba-cfb13f718259", MAC: 0a:58:0a:83:00:19, IPs: [10.131.0.25/23] I0403 00:43:43.441836 4001 cni.go:424] [openshift-monitoring/thanos-querier-6854ff8d7-xnw2d 61a4f79a12e2e77d8fb8d3ac2bd99d4a35b54ee8a71c8b3acd14da9e2a05e86e network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-6854ff8d7-xnw2d 61a4f79a12e2e77d8fb8d3ac2bd99d4a35b54ee8a71c8b3acd14da9e2a05e86e network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:4e:7a:96:2f:82\",\"name\":\"61a4f79a12e2e77\"},{\"mac\":\"0a:58:0a:83:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/547710a3-bdd3-40a8-b941-77292ef6ee4d\"}],\"ips\":[{\"address\":\"10.131.0.25/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err W0403 00:43:43.552637 4001 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:43:43.552656 4001 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:43:43.552661 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:43:43.835346 4001 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:43:43.835365 4001 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:43:43.835371 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0403 00:43:44.030292 4001 cni.go:403] [openshift-dns/dns-default-kg72g e907de79c963385d269c7e7724f530660aa42cffe1c5adb442c9651dfb4979e5 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-kg72g e907de79c963385d269c7e7724f530660aa42cffe1c5adb442c9651dfb4979e5 network default NAD default] I0403 00:43:44.040513 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-kg72g, hostIfaceName: e907de79c963385, network: default, NAD default, SandboxID: "e907de79c963385d269c7e7724f530660aa42cffe1c5adb442c9651dfb4979e5", PCI device ID: , UID: "4e92c5bf-720c-48a5-9d1a-76d7a3f9b2e9", MAC: 0a:58:0a:83:00:07, IPs: [10.131.0.7/23] I0403 00:43:44.098237 4001 cni.go:424] [openshift-dns/dns-default-kg72g e907de79c963385d269c7e7724f530660aa42cffe1c5adb442c9651dfb4979e5 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-kg72g e907de79c963385d269c7e7724f530660aa42cffe1c5adb442c9651dfb4979e5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:53:87:de:62:45\",\"name\":\"e907de79c963385\"},{\"mac\":\"0a:58:0a:83:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/232e82c8-fc1f-4145-8873-6dfacd5ce8f2\"}],\"ips\":[{\"address\":\"10.131.0.7/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:43:46.083625 4001 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:43:46.083761 4001 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0403 00:43:46.135616 4001 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 51.904027ms I0403 00:43:46.135639 4001 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 51.935718ms I0403 00:43:46.137109 4001 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 53.481742ms, libovsdb time 1.091258ms I0403 00:43:46.450924 4001 cni.go:403] [openshift-monitoring/prometheus-k8s-0 678c4ccef61e6caaffe7b0b7119c77a24d94a0d731f7dae9f84634083bfbe480 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 678c4ccef61e6caaffe7b0b7119c77a24d94a0d731f7dae9f84634083bfbe480 network default NAD default] I0403 00:43:46.463061 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 678c4ccef61e6ca, network: default, NAD default, SandboxID: "678c4ccef61e6caaffe7b0b7119c77a24d94a0d731f7dae9f84634083bfbe480", PCI device ID: , UID: "b55971df-164c-4d4f-9c57-ca396bebcc26", MAC: 0a:58:0a:83:00:1a, IPs: [10.131.0.26/23] W0403 00:43:46.485387 4001 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:46.485416 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:46.485424 4001 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0403 00:43:46.515676 4001 cni.go:424] [openshift-monitoring/prometheus-k8s-0 678c4ccef61e6caaffe7b0b7119c77a24d94a0d731f7dae9f84634083bfbe480 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 678c4ccef61e6caaffe7b0b7119c77a24d94a0d731f7dae9f84634083bfbe480 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:5e:4a:00:13:6c\",\"name\":\"678c4ccef61e6ca\"},{\"mac\":\"0a:58:0a:83:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3697e270-50b5-43a3-8063-54edb62b3f2e\"}],\"ips\":[{\"address\":\"10.131.0.26/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err W0403 00:43:46.845280 4001 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:46.845300 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:46.845306 4001 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:46.880341 4001 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:43:46.880364 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:43:47.497108 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0403 00:43:47.546027 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0403 00:43:47.573445 4001 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:43:47.573473 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:43:47.584518 4001 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:47.584541 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:47.584549 4001 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:47.584554 4001 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:47.630041 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0403 00:43:47.892968 4001 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:47.892987 4001 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:47.892994 4001 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:47.893000 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier I0403 00:43:51.786107 4001 base_network_controller_pods.go:487] [default/openshift-console/console-794fbf75b9-mddt4] creating logical port openshift-console_console-794fbf75b9-mddt4 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:43:51.786253 4001 kube.go:256] Updating pod openshift-console/console-794fbf75b9-mddt4 I0403 00:43:51.805372 4001 pod.go:62] [openshift-console/console-794fbf75b9-mddt4] pod update took 19.139817ms I0403 00:43:51.805395 4001 base_network_controller_pods.go:937] [default/openshift-console/console-794fbf75b9-mddt4] addLogicalPort annotation time took 19.172257ms I0403 00:43:51.806804 4001 pods.go:271] [openshift-console/console-794fbf75b9-mddt4] addLogicalPort took 20.729333ms, libovsdb time 1.020073ms I0403 00:43:52.160508 4001 cni.go:403] [openshift-console/console-794fbf75b9-mddt4 dcdb18d047f8ffee04481bf9dd9967804dd5d2210eec8e4c247ed55800186772 network default NAD default] ADD starting CNI request [openshift-console/console-794fbf75b9-mddt4 dcdb18d047f8ffee04481bf9dd9967804dd5d2210eec8e4c247ed55800186772 network default NAD default] I0403 00:43:52.171230 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-794fbf75b9-mddt4, hostIfaceName: dcdb18d047f8ffe, network: default, NAD default, SandboxID: "dcdb18d047f8ffee04481bf9dd9967804dd5d2210eec8e4c247ed55800186772", PCI device ID: , UID: "ccb4a98a-cf43-4640-b0c8-da20850a92ec", MAC: 0a:58:0a:83:00:1b, IPs: [10.131.0.27/23] I0403 00:43:52.223541 4001 cni.go:424] [openshift-console/console-794fbf75b9-mddt4 dcdb18d047f8ffee04481bf9dd9967804dd5d2210eec8e4c247ed55800186772 network default NAD default] ADD finished CNI request [openshift-console/console-794fbf75b9-mddt4 dcdb18d047f8ffee04481bf9dd9967804dd5d2210eec8e4c247ed55800186772 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:e7:88:f3:43:b5\",\"name\":\"dcdb18d047f8ffe\"},{\"mac\":\"0a:58:0a:83:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/290e6493-9b55-4460-bdae-905b7c9cefff\"}],\"ips\":[{\"address\":\"10.131.0.27/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err W0403 00:43:53.527488 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:53.527519 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:53.527527 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:53.527536 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:53.527560 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:53.527567 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:53.527575 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:53.527581 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:54.890802 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:54.890829 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:54.890836 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:54.890841 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0403 00:43:55.548561 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0403 00:43:56.530013 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:56.530038 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:56.530045 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:56.530055 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:56.530060 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:56.530064 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:56.867398 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:56.867420 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:56.867427 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0403 00:43:57.414321 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0403 00:43:57.414349 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0403 00:43:58.603479 4001 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7f6f7df889-6rm24 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:43:58.604158 4001 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7f6f7df889-6rm24 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:43:58.613377 4001 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7f6f7df889-6rm24 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:43:58.613402 4001 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7f6f7df889-6rm24 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:43:58.620416 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7f6f7df889-6rm24 I0403 00:43:58.620439 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7f6f7df889-6rm24 I0403 00:44:00.199117 4001 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0403 00:44:00.612302 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0403 00:44:00.995061 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0403 00:44:01.037061 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0403 00:44:02.581530 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:44:02.581555 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:44:06.962276 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0403 00:44:06.962335 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0403 00:44:06.962380 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0403 00:44:07.978008 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0403 00:44:10.697871 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:44:10.697895 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:44:10.731448 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:44:10.731471 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0403 00:44:12.062055 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:44:12.062074 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:44:12.062079 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:44:15.126341 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0403 00:44:26.103058 4001 cni.go:403] [openshift-console/console-794fbf75b9-mddt4 dcdb18d047f8ffee04481bf9dd9967804dd5d2210eec8e4c247ed55800186772 network default NAD default] DEL starting CNI request [openshift-console/console-794fbf75b9-mddt4 dcdb18d047f8ffee04481bf9dd9967804dd5d2210eec8e4c247ed55800186772 network default NAD default] I0403 00:44:26.187978 4001 cni.go:424] [openshift-console/console-794fbf75b9-mddt4 dcdb18d047f8ffee04481bf9dd9967804dd5d2210eec8e4c247ed55800186772 network default NAD default] DEL finished CNI request [openshift-console/console-794fbf75b9-mddt4 dcdb18d047f8ffee04481bf9dd9967804dd5d2210eec8e4c247ed55800186772 network default NAD default], result "{}", err I0403 00:44:26.250111 4001 obj_retry.go:464] Detected object openshift-console/console-794fbf75b9-mddt4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:44:26.250150 4001 pods.go:176] Deleting pod: openshift-console/console-794fbf75b9-mddt4 I0403 00:44:26.251577 4001 pods.go:236] Attempting to release IPs for pod: openshift-console/console-794fbf75b9-mddt4, ips: 10.131.0.27 I0403 00:44:26.251613 4001 obj_retry.go:464] Detected object openshift-console/console-794fbf75b9-mddt4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:44:26.258167 4001 base_network_controller_pods.go:487] [default/openshift-console/console-66974c5f69-pvgpq] creating logical port openshift-console_console-66974c5f69-pvgpq for pod on switch ip-10-0-1-98.ec2.internal I0403 00:44:26.258296 4001 kube.go:256] Updating pod openshift-console/console-66974c5f69-pvgpq I0403 00:44:26.270891 4001 pod.go:62] [openshift-console/console-66974c5f69-pvgpq] pod update took 12.617155ms I0403 00:44:26.270918 4001 base_network_controller_pods.go:937] [default/openshift-console/console-66974c5f69-pvgpq] addLogicalPort annotation time took 12.652481ms I0403 00:44:26.272429 4001 pods.go:271] [openshift-console/console-66974c5f69-pvgpq] addLogicalPort took 14.275145ms, libovsdb time 1.095359ms I0403 00:44:26.667245 4001 cni.go:403] [openshift-console/console-66974c5f69-pvgpq 608a35db308716dec71a89d3a2ac4397a92cc76efd9a7bb22822cae040cc8b02 network default NAD default] ADD starting CNI request [openshift-console/console-66974c5f69-pvgpq 608a35db308716dec71a89d3a2ac4397a92cc76efd9a7bb22822cae040cc8b02 network default NAD default] I0403 00:44:26.677513 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-66974c5f69-pvgpq, hostIfaceName: 608a35db308716d, network: default, NAD default, SandboxID: "608a35db308716dec71a89d3a2ac4397a92cc76efd9a7bb22822cae040cc8b02", PCI device ID: , UID: "74b5bb1e-3d83-4afd-b350-427b1442bf6f", MAC: 0a:58:0a:83:00:1c, IPs: [10.131.0.28/23] I0403 00:44:26.742331 4001 cni.go:424] [openshift-console/console-66974c5f69-pvgpq 608a35db308716dec71a89d3a2ac4397a92cc76efd9a7bb22822cae040cc8b02 network default NAD default] ADD finished CNI request [openshift-console/console-66974c5f69-pvgpq 608a35db308716dec71a89d3a2ac4397a92cc76efd9a7bb22822cae040cc8b02 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:cf:fb:e3:e2:ef\",\"name\":\"608a35db308716d\"},{\"mac\":\"0a:58:0a:83:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f3d0595a-4050-4e39-b841-b44685ff636b\"}],\"ips\":[{\"address\":\"10.131.0.28/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:44:27.065576 4001 obj_retry.go:458] Detected object openshift-console/console-794fbf75b9-mddt4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:44:27.065605 4001 obj_retry.go:458] Detected object openshift-console/console-794fbf75b9-mddt4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:44:27.071805 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-794fbf75b9-mddt4 I0403 00:44:27.071840 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-794fbf75b9-mddt4 W0403 00:44:31.084589 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0403 00:44:31.084592 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0403 00:44:31.084643 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0403 00:44:31.684735 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0403 00:44:32.101698 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0403 00:44:32.130504 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0403 00:44:32.130534 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0403 00:44:32.130562 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0403 00:44:36.015351 4001 obj_retry.go:464] Detected object openshift-console/console-794fbf75b9-m5x5b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:44:36.015889 4001 obj_retry.go:464] Detected object openshift-console/console-794fbf75b9-m5x5b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:44:36.698026 4001 obj_retry.go:458] Detected object openshift-console/console-794fbf75b9-m5x5b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:44:36.698057 4001 obj_retry.go:458] Detected object openshift-console/console-794fbf75b9-m5x5b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:44:36.706258 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-794fbf75b9-m5x5b I0403 00:44:36.706282 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-794fbf75b9-m5x5b W0403 00:44:37.123253 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0403 00:44:46.224317 4001 cni.go:403] [openshift-ingress/router-default-5cf556f5bc-qkqjp 09bf1629b194d8fef38bab7a2583201f2e677cebdbe5a085fe67f8955110da34 network default NAD default] DEL starting CNI request [openshift-ingress/router-default-5cf556f5bc-qkqjp 09bf1629b194d8fef38bab7a2583201f2e677cebdbe5a085fe67f8955110da34 network default NAD default] I0403 00:44:46.309362 4001 cni.go:424] [openshift-ingress/router-default-5cf556f5bc-qkqjp 09bf1629b194d8fef38bab7a2583201f2e677cebdbe5a085fe67f8955110da34 network default NAD default] DEL finished CNI request [openshift-ingress/router-default-5cf556f5bc-qkqjp 09bf1629b194d8fef38bab7a2583201f2e677cebdbe5a085fe67f8955110da34 network default NAD default], result "{}", err I0403 00:44:46.390029 4001 obj_retry.go:464] Detected object openshift-ingress/router-default-5cf556f5bc-qkqjp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:44:46.390071 4001 pods.go:176] Deleting pod: openshift-ingress/router-default-5cf556f5bc-qkqjp I0403 00:44:46.391581 4001 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-5cf556f5bc-qkqjp, ips: 10.131.0.18 I0403 00:44:46.391612 4001 obj_retry.go:464] Detected object openshift-ingress/router-default-5cf556f5bc-qkqjp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0403 00:44:46.851759 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:44:46.851781 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:44:46.851792 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:44:46.851797 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0403 00:44:47.164573 4001 obj_retry.go:458] Detected object openshift-ingress/router-default-5cf556f5bc-qkqjp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:44:47.164595 4001 obj_retry.go:458] Detected object openshift-ingress/router-default-5cf556f5bc-qkqjp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:44:47.170791 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-5cf556f5bc-qkqjp I0403 00:44:47.170829 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-5cf556f5bc-qkqjp W0403 00:44:47.236461 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:44:47.236491 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:44:53.108668 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0403 00:45:01.609009 4001 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:45:01.609604 4001 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:45:01.876830 4001 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:45:01.876855 4001 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:45:01.885552 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0403 00:45:01.885577 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0403 00:45:03.036302 4001 obj_retry.go:464] Detected object openshift-backplane-managed-scripts/osd-delete-backplane-script-resources-29586282-zxhfn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:45:03.036809 4001 obj_retry.go:464] Detected object openshift-backplane-managed-scripts/osd-delete-backplane-script-resources-29586282-zxhfn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:45:03.896063 4001 obj_retry.go:458] Detected object openshift-backplane-managed-scripts/osd-delete-backplane-script-resources-29586282-zxhfn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:45:03.896105 4001 obj_retry.go:458] Detected object openshift-backplane-managed-scripts/osd-delete-backplane-script-resources-29586282-zxhfn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:45:05.210109 4001 cni.go:403] [openshift-multus/network-metrics-daemon-psb2l 831e8756a9a25a3f4a37865a94342e2a70b67fe02840c473858d0296d6241be8 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-psb2l 831e8756a9a25a3f4a37865a94342e2a70b67fe02840c473858d0296d6241be8 network default NAD default] I0403 00:45:05.219709 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-psb2l, hostIfaceName: 831e8756a9a25a3, network: default, NAD default, SandboxID: "831e8756a9a25a3f4a37865a94342e2a70b67fe02840c473858d0296d6241be8", PCI device ID: , UID: "d2407d5d-65a4-4b06-9ddb-989ec792e6c0", MAC: 0a:58:0a:83:00:04, IPs: [10.131.0.4/23] I0403 00:45:05.270454 4001 cni.go:424] [openshift-multus/network-metrics-daemon-psb2l 831e8756a9a25a3f4a37865a94342e2a70b67fe02840c473858d0296d6241be8 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-psb2l 831e8756a9a25a3f4a37865a94342e2a70b67fe02840c473858d0296d6241be8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:29:d4:49:0e:fb\",\"name\":\"831e8756a9a25a3\"},{\"mac\":\"0a:58:0a:83:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6c165f43-da56-425f-9158-46bb9b3c044b\"}],\"ips\":[{\"address\":\"10.131.0.4/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:45:05.877938 4001 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:45:05.878593 4001 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:45:06.203536 4001 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:45:06.203628 4001 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:45:06.210808 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0403 00:45:06.210881 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0403 00:45:23.501529 4001 cni.go:403] [openshift-monitoring/alertmanager-main-0 6ea0498045cd26c85478b6362f0b02c1c89eb22cc23fb19cafb04c27543bb396 network default NAD default] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 6ea0498045cd26c85478b6362f0b02c1c89eb22cc23fb19cafb04c27543bb396 network default NAD default] I0403 00:45:23.578927 4001 cni.go:424] [openshift-monitoring/alertmanager-main-0 6ea0498045cd26c85478b6362f0b02c1c89eb22cc23fb19cafb04c27543bb396 network default NAD default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 6ea0498045cd26c85478b6362f0b02c1c89eb22cc23fb19cafb04c27543bb396 network default NAD default], result "{}", err I0403 00:45:23.639188 4001 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:45:23.639225 4001 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0403 00:45:23.640649 4001 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.131.0.23 I0403 00:45:23.640685 4001 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:45:24.339337 4001 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:45:24.339392 4001 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:45:24.345474 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0403 00:45:24.345509 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0403 00:45:24.385401 4001 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:45:24.385572 4001 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0403 00:45:24.411004 4001 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 25.473801ms I0403 00:45:24.411025 4001 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 25.499268ms I0403 00:45:24.412262 4001 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 26.88097ms, libovsdb time 880.013µs I0403 00:45:24.747183 4001 cni.go:403] [openshift-monitoring/alertmanager-main-0 eed30936df823b2d7cede3ac4740bc3db3719c08bf3dcf8b0b7de977c5ebad4a network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 eed30936df823b2d7cede3ac4740bc3db3719c08bf3dcf8b0b7de977c5ebad4a network default NAD default] I0403 00:45:24.756598 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: eed30936df823b2, network: default, NAD default, SandboxID: "eed30936df823b2d7cede3ac4740bc3db3719c08bf3dcf8b0b7de977c5ebad4a", PCI device ID: , UID: "519fce1f-5632-47b6-bed5-90e5332db190", MAC: 0a:58:0a:83:00:1d, IPs: [10.131.0.29/23] I0403 00:45:24.806538 4001 cni.go:424] [openshift-monitoring/alertmanager-main-0 eed30936df823b2d7cede3ac4740bc3db3719c08bf3dcf8b0b7de977c5ebad4a network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 eed30936df823b2d7cede3ac4740bc3db3719c08bf3dcf8b0b7de977c5ebad4a network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:24:2a:53:e0:2f\",\"name\":\"eed30936df823b2\"},{\"mac\":\"0a:58:0a:83:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/91e08c65-d088-4b63-90c3-accae69c828b\"}],\"ips\":[{\"address\":\"10.131.0.29/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:45:30.424902 4001 cni.go:403] [openshift-console/console-66974c5f69-pvgpq 608a35db308716dec71a89d3a2ac4397a92cc76efd9a7bb22822cae040cc8b02 network default NAD default] DEL starting CNI request [openshift-console/console-66974c5f69-pvgpq 608a35db308716dec71a89d3a2ac4397a92cc76efd9a7bb22822cae040cc8b02 network default NAD default] I0403 00:45:30.509618 4001 cni.go:424] [openshift-console/console-66974c5f69-pvgpq 608a35db308716dec71a89d3a2ac4397a92cc76efd9a7bb22822cae040cc8b02 network default NAD default] DEL finished CNI request [openshift-console/console-66974c5f69-pvgpq 608a35db308716dec71a89d3a2ac4397a92cc76efd9a7bb22822cae040cc8b02 network default NAD default], result "{}", err I0403 00:45:30.555628 4001 obj_retry.go:464] Detected object openshift-console/console-66974c5f69-pvgpq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:45:30.555666 4001 pods.go:176] Deleting pod: openshift-console/console-66974c5f69-pvgpq I0403 00:45:30.556857 4001 pods.go:236] Attempting to release IPs for pod: openshift-console/console-66974c5f69-pvgpq, ips: 10.131.0.28 I0403 00:45:30.556889 4001 obj_retry.go:464] Detected object openshift-console/console-66974c5f69-pvgpq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:45:30.565201 4001 base_network_controller_pods.go:487] [default/openshift-console/console-59c69c9866-h8prf] creating logical port openshift-console_console-59c69c9866-h8prf for pod on switch ip-10-0-1-98.ec2.internal I0403 00:45:30.565335 4001 kube.go:256] Updating pod openshift-console/console-59c69c9866-h8prf I0403 00:45:30.579501 4001 pod.go:62] [openshift-console/console-59c69c9866-h8prf] pod update took 14.189397ms I0403 00:45:30.579524 4001 base_network_controller_pods.go:937] [default/openshift-console/console-59c69c9866-h8prf] addLogicalPort annotation time took 14.21591ms I0403 00:45:30.580712 4001 pods.go:271] [openshift-console/console-59c69c9866-h8prf] addLogicalPort took 15.527229ms, libovsdb time 848.761µs I0403 00:45:30.926640 4001 cni.go:403] [openshift-console/console-59c69c9866-h8prf 94560cc8ca247d4bde5ed31b72969f15429f4a72d379f2ce8f9ce64a4ad00b2b network default NAD default] ADD starting CNI request [openshift-console/console-59c69c9866-h8prf 94560cc8ca247d4bde5ed31b72969f15429f4a72d379f2ce8f9ce64a4ad00b2b network default NAD default] I0403 00:45:30.936816 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-59c69c9866-h8prf, hostIfaceName: 94560cc8ca247d4, network: default, NAD default, SandboxID: "94560cc8ca247d4bde5ed31b72969f15429f4a72d379f2ce8f9ce64a4ad00b2b", PCI device ID: , UID: "3043b01c-04bb-4e7f-9fe7-1ae8497e69bf", MAC: 0a:58:0a:83:00:1e, IPs: [10.131.0.30/23] I0403 00:45:30.987673 4001 cni.go:424] [openshift-console/console-59c69c9866-h8prf 94560cc8ca247d4bde5ed31b72969f15429f4a72d379f2ce8f9ce64a4ad00b2b network default NAD default] ADD finished CNI request [openshift-console/console-59c69c9866-h8prf 94560cc8ca247d4bde5ed31b72969f15429f4a72d379f2ce8f9ce64a4ad00b2b network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:f1:3f:5e:f6:bb\",\"name\":\"94560cc8ca247d4\"},{\"mac\":\"0a:58:0a:83:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fe627bb4-b28b-4528-8017-87405df2ff81\"}],\"ips\":[{\"address\":\"10.131.0.30/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:45:31.394445 4001 obj_retry.go:458] Detected object openshift-console/console-66974c5f69-pvgpq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:45:31.394468 4001 obj_retry.go:458] Detected object openshift-console/console-66974c5f69-pvgpq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:45:31.403235 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-66974c5f69-pvgpq I0403 00:45:31.403265 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-66974c5f69-pvgpq I0403 00:45:41.219469 4001 obj_retry.go:464] Detected object openshift-console/console-66974c5f69-l5snp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:45:41.220018 4001 obj_retry.go:464] Detected object openshift-console/console-66974c5f69-l5snp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0403 00:45:41.452007 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0403 00:45:42.042726 4001 obj_retry.go:458] Detected object openshift-console/console-66974c5f69-l5snp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:45:42.042752 4001 obj_retry.go:458] Detected object openshift-console/console-66974c5f69-l5snp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:45:42.045729 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-66974c5f69-l5snp I0403 00:45:42.045755 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-66974c5f69-l5snp W0403 00:45:54.797153 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:45:54.797178 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0403 00:45:54.797186 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0403 00:46:03.071578 4001 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-96.ec2.internal I0403 00:46:03.071612 4001 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-96.ec2.internal: 41.807µs I0403 00:46:03.071585 4001 egressfirewall.go:628] Syncing node "ip-10-0-0-96.ec2.internal" for egress firewall I0403 00:46:08.376672 4001 cni.go:403] [openshift-monitoring/prometheus-k8s-0 678c4ccef61e6caaffe7b0b7119c77a24d94a0d731f7dae9f84634083bfbe480 network default NAD default] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 678c4ccef61e6caaffe7b0b7119c77a24d94a0d731f7dae9f84634083bfbe480 network default NAD default] I0403 00:46:08.451236 4001 cni.go:424] [openshift-monitoring/prometheus-k8s-0 678c4ccef61e6caaffe7b0b7119c77a24d94a0d731f7dae9f84634083bfbe480 network default NAD default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 678c4ccef61e6caaffe7b0b7119c77a24d94a0d731f7dae9f84634083bfbe480 network default NAD default], result "{}", err I0403 00:46:08.510031 4001 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:46:08.510067 4001 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0403 00:46:08.511302 4001 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.131.0.26 I0403 00:46:08.511329 4001 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:46:08.858267 4001 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:46:08.858294 4001 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:46:08.864259 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0403 00:46:08.864289 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0403 00:46:08.909141 4001 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:46:08.909262 4001 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0403 00:46:08.935488 4001 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 26.267695ms I0403 00:46:08.935514 4001 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 26.298038ms I0403 00:46:08.936802 4001 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 27.681979ms, libovsdb time 926.926µs I0403 00:46:09.275473 4001 cni.go:403] [openshift-monitoring/prometheus-k8s-0 67d4b1968b59496a0bc0d3a1fe4afbc98ed337c436ceba213cfad70a1d915f33 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 67d4b1968b59496a0bc0d3a1fe4afbc98ed337c436ceba213cfad70a1d915f33 network default NAD default] I0403 00:46:09.286257 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 67d4b1968b59496, network: default, NAD default, SandboxID: "67d4b1968b59496a0bc0d3a1fe4afbc98ed337c436ceba213cfad70a1d915f33", PCI device ID: , UID: "158452c5-0af2-4990-88c2-b0cbb07c4ef1", MAC: 0a:58:0a:83:00:1f, IPs: [10.131.0.31/23] I0403 00:46:09.336036 4001 cni.go:424] [openshift-monitoring/prometheus-k8s-0 67d4b1968b59496a0bc0d3a1fe4afbc98ed337c436ceba213cfad70a1d915f33 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 67d4b1968b59496a0bc0d3a1fe4afbc98ed337c436ceba213cfad70a1d915f33 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:12:ae:89:31:80\",\"name\":\"67d4b1968b59496\"},{\"mac\":\"0a:58:0a:83:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/485acb2d-4ebd-40d1-9432-d0e19c202fe5\"}],\"ips\":[{\"address\":\"10.131.0.31/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:46:45.944472 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 W0403 00:47:09.920974 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0403 00:47:09.920997 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0403 00:47:09.936036 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 W0403 00:47:13.298654 4001 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0403 00:47:16.922688 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0403 00:47:17.447895 4001 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586280-tb8pp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:47:17.448493 4001 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586280-tb8pp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:47:18.338932 4001 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586280-tb8pp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:47:18.338968 4001 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586280-tb8pp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:47:24.335510 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0403 00:47:24.335580 4001 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-4zv8f IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-4zv8f: no pod IPs found I0403 00:47:27.735001 4001 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0403 00:47:52.934745 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0403 00:48:01.917726 4001 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0403 00:48:02.938960 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0403 00:48:08.956760 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0403 00:48:12.827844 4001 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=135 I0403 00:48:29.825749 4001 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0403 00:49:35.825688 4001 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=78 I0403 00:49:35.828115 4001 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=17 I0403 00:50:04.164963 4001 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586290-9svgp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:04.165594 4001 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586290-9svgp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:05.056421 4001 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586290-9svgp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:05.056452 4001 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586290-9svgp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:08.196205 4001 cni.go:403] [openshift-console/console-59c69c9866-h8prf 94560cc8ca247d4bde5ed31b72969f15429f4a72d379f2ce8f9ce64a4ad00b2b network default NAD default] DEL starting CNI request [openshift-console/console-59c69c9866-h8prf 94560cc8ca247d4bde5ed31b72969f15429f4a72d379f2ce8f9ce64a4ad00b2b network default NAD default] I0403 00:50:08.278965 4001 cni.go:424] [openshift-console/console-59c69c9866-h8prf 94560cc8ca247d4bde5ed31b72969f15429f4a72d379f2ce8f9ce64a4ad00b2b network default NAD default] DEL finished CNI request [openshift-console/console-59c69c9866-h8prf 94560cc8ca247d4bde5ed31b72969f15429f4a72d379f2ce8f9ce64a4ad00b2b network default NAD default], result "{}", err I0403 00:50:08.324443 4001 obj_retry.go:464] Detected object openshift-console/console-59c69c9866-h8prf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:08.324506 4001 pods.go:176] Deleting pod: openshift-console/console-59c69c9866-h8prf I0403 00:50:08.325785 4001 pods.go:236] Attempting to release IPs for pod: openshift-console/console-59c69c9866-h8prf, ips: 10.131.0.30 I0403 00:50:08.325811 4001 obj_retry.go:464] Detected object openshift-console/console-59c69c9866-h8prf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:08.334481 4001 base_network_controller_pods.go:487] [default/openshift-console/console-5bbcddd644-lzmzp] creating logical port openshift-console_console-5bbcddd644-lzmzp for pod on switch ip-10-0-1-98.ec2.internal I0403 00:50:08.334581 4001 kube.go:256] Updating pod openshift-console/console-5bbcddd644-lzmzp I0403 00:50:08.347836 4001 pod.go:62] [openshift-console/console-5bbcddd644-lzmzp] pod update took 13.280003ms I0403 00:50:08.347860 4001 base_network_controller_pods.go:937] [default/openshift-console/console-5bbcddd644-lzmzp] addLogicalPort annotation time took 13.311247ms I0403 00:50:08.349166 4001 pods.go:271] [openshift-console/console-5bbcddd644-lzmzp] addLogicalPort took 14.702884ms, libovsdb time 830.925µs I0403 00:50:08.602227 4001 obj_retry.go:458] Detected object openshift-console/console-59c69c9866-h8prf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:08.602249 4001 obj_retry.go:458] Detected object openshift-console/console-59c69c9866-h8prf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:08.604842 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-59c69c9866-h8prf I0403 00:50:08.604874 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-59c69c9866-h8prf I0403 00:50:08.696040 4001 cni.go:403] [openshift-console/console-5bbcddd644-lzmzp 275afd1a04977a17c7c9616b55994e94028c50a2c157f907156e9da57b4c8eb5 network default NAD default] ADD starting CNI request [openshift-console/console-5bbcddd644-lzmzp 275afd1a04977a17c7c9616b55994e94028c50a2c157f907156e9da57b4c8eb5 network default NAD default] I0403 00:50:08.705794 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5bbcddd644-lzmzp, hostIfaceName: 275afd1a04977a1, network: default, NAD default, SandboxID: "275afd1a04977a17c7c9616b55994e94028c50a2c157f907156e9da57b4c8eb5", PCI device ID: , UID: "d792200e-27ae-4e90-8e55-840253897497", MAC: 0a:58:0a:83:00:20, IPs: [10.131.0.32/23] I0403 00:50:08.755874 4001 cni.go:424] [openshift-console/console-5bbcddd644-lzmzp 275afd1a04977a17c7c9616b55994e94028c50a2c157f907156e9da57b4c8eb5 network default NAD default] ADD finished CNI request [openshift-console/console-5bbcddd644-lzmzp 275afd1a04977a17c7c9616b55994e94028c50a2c157f907156e9da57b4c8eb5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:43:dc:59:59:93\",\"name\":\"275afd1a04977a1\"},{\"mac\":\"0a:58:0a:83:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/91e7afc8-2179-459f-b652-601c1e5e9da9\"}],\"ips\":[{\"address\":\"10.131.0.32/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:50:19.327432 4001 obj_retry.go:464] Detected object openshift-console/console-59c69c9866-wb9wz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:19.327987 4001 obj_retry.go:464] Detected object openshift-console/console-59c69c9866-wb9wz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0403 00:50:19.666962 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0403 00:50:20.006978 4001 obj_retry.go:458] Detected object openshift-console/console-59c69c9866-wb9wz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:20.007005 4001 obj_retry.go:458] Detected object openshift-console/console-59c69c9866-wb9wz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:20.010494 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-59c69c9866-wb9wz I0403 00:50:20.010528 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-59c69c9866-wb9wz I0403 00:50:28.955232 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0403 00:50:39.939844 4001 namespace.go:98] [tsf] adding namespace I0403 00:50:39.940790 4001 namespace.go:102] [tsf] adding namespace took 924.511µs I0403 00:50:39.969007 4001 namespace.go:142] [tsf] updating namespace I0403 00:50:39.974917 4001 namespace.go:142] [tsf] updating namespace I0403 00:50:43.985648 4001 namespace.go:98] [rhbk-operator] adding namespace I0403 00:50:43.986620 4001 namespace.go:102] [rhbk-operator] adding namespace took 920.504µs I0403 00:50:43.990223 4001 namespace.go:98] [tsf-tpa] adding namespace I0403 00:50:43.990856 4001 namespace.go:102] [tsf-tpa] adding namespace took 615.553µs I0403 00:50:43.997985 4001 namespace.go:98] [tsf-keycloak] adding namespace I0403 00:50:43.998029 4001 namespace.go:98] [konflux-operator] adding namespace I0403 00:50:43.998629 4001 namespace.go:102] [tsf-keycloak] adding namespace took 621.094µs I0403 00:50:43.999194 4001 namespace.go:102] [konflux-operator] adding namespace took 1.150822ms I0403 00:50:44.003070 4001 namespace.go:98] [cert-manager-operator] adding namespace I0403 00:50:44.003182 4001 namespace.go:98] [tsf-tas] adding namespace I0403 00:50:44.003205 4001 namespace.go:98] [konflux-ui] adding namespace I0403 00:50:44.003374 4001 namespace.go:98] [rhtpa-operator] adding namespace I0403 00:50:44.003831 4001 namespace.go:102] [cert-manager-operator] adding namespace took 677.673µs I0403 00:50:44.004381 4001 namespace.go:102] [tsf-tas] adding namespace took 1.180199ms I0403 00:50:44.004862 4001 namespace.go:102] [konflux-ui] adding namespace took 1.645986ms I0403 00:50:44.005340 4001 namespace.go:102] [rhtpa-operator] adding namespace took 1.952076ms I0403 00:50:44.019933 4001 namespace.go:142] [rhbk-operator] updating namespace I0403 00:50:44.038252 4001 namespace.go:142] [rhbk-operator] updating namespace I0403 00:50:44.038394 4001 namespace.go:142] [konflux-operator] updating namespace I0403 00:50:44.038497 4001 namespace.go:142] [tsf-tpa] updating namespace I0403 00:50:44.038569 4001 namespace.go:142] [cert-manager-operator] updating namespace I0403 00:50:44.041159 4001 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:50:44.047844 4001 namespace.go:142] [tsf-tas] updating namespace I0403 00:50:44.051025 4001 namespace.go:142] [rhtpa-operator] updating namespace I0403 00:50:44.051128 4001 namespace.go:142] [konflux-ui] updating namespace I0403 00:50:44.062369 4001 namespace.go:142] [rhbk-operator] updating namespace I0403 00:50:44.081897 4001 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:50:44.099218 4001 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:50:44.103855 4001 namespace.go:142] [konflux-operator] updating namespace I0403 00:50:44.118645 4001 namespace.go:142] [konflux-operator] updating namespace I0403 00:50:44.129705 4001 namespace.go:142] [cert-manager-operator] updating namespace I0403 00:50:44.141655 4001 namespace.go:142] [cert-manager-operator] updating namespace I0403 00:50:44.145430 4001 namespace.go:142] [tsf-tas] updating namespace I0403 00:50:44.160533 4001 namespace.go:142] [tsf-tas] updating namespace I0403 00:50:44.172688 4001 namespace.go:142] [konflux-ui] updating namespace I0403 00:50:44.189869 4001 namespace.go:142] [konflux-ui] updating namespace I0403 00:50:44.197770 4001 namespace.go:142] [rhtpa-operator] updating namespace I0403 00:50:44.209629 4001 namespace.go:142] [rhtpa-operator] updating namespace I0403 00:50:44.221879 4001 namespace.go:142] [tsf-tpa] updating namespace I0403 00:50:44.238174 4001 namespace.go:142] [tsf-tpa] updating namespace I0403 00:50:45.229189 4001 namespace.go:142] [tsf-tpa] updating namespace I0403 00:50:45.245388 4001 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:50:46.309700 4001 base_network_controller_pods.go:487] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql] creating logical port openshift-marketplace_3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql for pod on switch ip-10-0-1-98.ec2.internal I0403 00:50:46.309841 4001 kube.go:256] Updating pod openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql I0403 00:50:46.322125 4001 pod.go:62] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql] pod update took 12.316463ms I0403 00:50:46.322156 4001 base_network_controller_pods.go:937] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql] addLogicalPort annotation time took 12.352958ms I0403 00:50:46.323551 4001 pods.go:271] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql] addLogicalPort took 13.869771ms, libovsdb time 990.229µs I0403 00:50:46.417182 4001 base_network_controller_pods.go:487] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl] creating logical port openshift-marketplace_925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl for pod on switch ip-10-0-1-98.ec2.internal I0403 00:50:46.417304 4001 kube.go:256] Updating pod openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl I0403 00:50:46.446048 4001 pod.go:62] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl] pod update took 28.777189ms I0403 00:50:46.446072 4001 base_network_controller_pods.go:937] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl] addLogicalPort annotation time took 28.806706ms I0403 00:50:46.447300 4001 pods.go:271] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl] addLogicalPort took 30.133362ms, libovsdb time 844.116µs I0403 00:50:46.670689 4001 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql c8a8228b3b6f103eae91e12514ab516bee4d355835d0d37c22919db77bf3692d network default NAD default] ADD starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql c8a8228b3b6f103eae91e12514ab516bee4d355835d0d37c22919db77bf3692d network default NAD default] I0403 00:50:46.679992 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql, hostIfaceName: c8a8228b3b6f103, network: default, NAD default, SandboxID: "c8a8228b3b6f103eae91e12514ab516bee4d355835d0d37c22919db77bf3692d", PCI device ID: , UID: "850c91cd-77ba-4214-97b7-aadf167b227a", MAC: 0a:58:0a:83:00:21, IPs: [10.131.0.33/23] I0403 00:50:46.730747 4001 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql c8a8228b3b6f103eae91e12514ab516bee4d355835d0d37c22919db77bf3692d network default NAD default] ADD finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql c8a8228b3b6f103eae91e12514ab516bee4d355835d0d37c22919db77bf3692d network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:45:71:92:e7:d2\",\"name\":\"c8a8228b3b6f103\"},{\"mac\":\"0a:58:0a:83:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2b188ddd-3ee7-4adc-8d22-ff76b1079420\"}],\"ips\":[{\"address\":\"10.131.0.33/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:50:46.777941 4001 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl 2257dfebaa3a27a7d69ffd1a38df53265c261760d708c03391b02594483a8977 network default NAD default] ADD starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl 2257dfebaa3a27a7d69ffd1a38df53265c261760d708c03391b02594483a8977 network default NAD default] I0403 00:50:46.787414 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl, hostIfaceName: 2257dfebaa3a27a, network: default, NAD default, SandboxID: "2257dfebaa3a27a7d69ffd1a38df53265c261760d708c03391b02594483a8977", PCI device ID: , UID: "4559688f-3212-468b-8adf-1aaecce2f616", MAC: 0a:58:0a:83:00:22, IPs: [10.131.0.34/23] I0403 00:50:46.839894 4001 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl 2257dfebaa3a27a7d69ffd1a38df53265c261760d708c03391b02594483a8977 network default NAD default] ADD finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl 2257dfebaa3a27a7d69ffd1a38df53265c261760d708c03391b02594483a8977 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:f5:d7:9a:59:06\",\"name\":\"2257dfebaa3a27a\"},{\"mac\":\"0a:58:0a:83:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f7c923da-6b36-4ff7-ae9f-addb44496642\"}],\"ips\":[{\"address\":\"10.131.0.34/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:50:49.681678 4001 base_network_controller_pods.go:487] [default/openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb] creating logical port openshift-marketplace_f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb for pod on switch ip-10-0-1-98.ec2.internal I0403 00:50:49.681778 4001 kube.go:256] Updating pod openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb I0403 00:50:49.693434 4001 pod.go:62] [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb] pod update took 11.685588ms I0403 00:50:49.693459 4001 base_network_controller_pods.go:937] [default/openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb] addLogicalPort annotation time took 11.715074ms I0403 00:50:49.694852 4001 pods.go:271] [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb] addLogicalPort took 13.19614ms, libovsdb time 973.171µs I0403 00:50:50.052711 4001 cni.go:403] [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb 4cb642bbafe6401c81f97b206a9b914a030503cd1df6beb5ba960047295368d1 network default NAD default] ADD starting CNI request [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb 4cb642bbafe6401c81f97b206a9b914a030503cd1df6beb5ba960047295368d1 network default NAD default] I0403 00:50:50.063219 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb, hostIfaceName: 4cb642bbafe6401, network: default, NAD default, SandboxID: "4cb642bbafe6401c81f97b206a9b914a030503cd1df6beb5ba960047295368d1", PCI device ID: , UID: "160a2d2f-fadc-42b8-a815-88bed94b7ac0", MAC: 0a:58:0a:83:00:23, IPs: [10.131.0.35/23] I0403 00:50:50.112294 4001 cni.go:424] [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb 4cb642bbafe6401c81f97b206a9b914a030503cd1df6beb5ba960047295368d1 network default NAD default] ADD finished CNI request [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb 4cb642bbafe6401c81f97b206a9b914a030503cd1df6beb5ba960047295368d1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:8b:4c:a6:9e:48\",\"name\":\"4cb642bbafe6401\"},{\"mac\":\"0a:58:0a:83:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8c3cde79-b551-4a01-a614-b5fc3a459c84\"}],\"ips\":[{\"address\":\"10.131.0.35/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:50:50.829170 4001 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql c8a8228b3b6f103eae91e12514ab516bee4d355835d0d37c22919db77bf3692d network default NAD default] DEL starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql c8a8228b3b6f103eae91e12514ab516bee4d355835d0d37c22919db77bf3692d network default NAD default] I0403 00:50:50.831713 4001 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=724 I0403 00:50:50.899023 4001 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql c8a8228b3b6f103eae91e12514ab516bee4d355835d0d37c22919db77bf3692d network default NAD default] DEL finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql c8a8228b3b6f103eae91e12514ab516bee4d355835d0d37c22919db77bf3692d network default NAD default], result "{}", err I0403 00:50:50.957416 4001 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:50.957456 4001 pods.go:176] Deleting pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql I0403 00:50:50.958981 4001 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql, ips: 10.131.0.33 I0403 00:50:50.959017 4001 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:50.959045 4001 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:50:50.959053 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0403 00:50:50.959295 4001 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:50:50.959315 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0403 00:50:51.335421 4001 base_network_controller_pods.go:487] [default/openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq] creating logical port openshift-marketplace_53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq for pod on switch ip-10-0-1-98.ec2.internal I0403 00:50:51.335547 4001 kube.go:256] Updating pod openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq I0403 00:50:51.347902 4001 pod.go:62] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq] pod update took 12.382245ms I0403 00:50:51.347934 4001 base_network_controller_pods.go:937] [default/openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq] addLogicalPort annotation time took 12.419518ms I0403 00:50:51.349500 4001 pods.go:271] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq] addLogicalPort took 14.096511ms, libovsdb time 996.869µs I0403 00:50:51.696015 4001 cni.go:403] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq 9be769e0dae2199fc431bfdffb1d51f94e9d0a57f39fbcf42a36256cef95bcdf network default NAD default] ADD starting CNI request [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq 9be769e0dae2199fc431bfdffb1d51f94e9d0a57f39fbcf42a36256cef95bcdf network default NAD default] I0403 00:50:51.705753 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq, hostIfaceName: 9be769e0dae2199, network: default, NAD default, SandboxID: "9be769e0dae2199fc431bfdffb1d51f94e9d0a57f39fbcf42a36256cef95bcdf", PCI device ID: , UID: "b17a0686-add8-4dd9-bc1a-9a46f5b52129", MAC: 0a:58:0a:83:00:24, IPs: [10.131.0.36/23] I0403 00:50:51.755225 4001 cni.go:424] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq 9be769e0dae2199fc431bfdffb1d51f94e9d0a57f39fbcf42a36256cef95bcdf network default NAD default] ADD finished CNI request [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq 9be769e0dae2199fc431bfdffb1d51f94e9d0a57f39fbcf42a36256cef95bcdf network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:57:f6:df:bd:30\",\"name\":\"9be769e0dae2199\"},{\"mac\":\"0a:58:0a:83:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1e0f58c4-1803-40b1-8e6f-cba4ffd82ad3\"}],\"ips\":[{\"address\":\"10.131.0.36/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:50:51.827183 4001 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=73 I0403 00:50:51.980376 4001 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:51.980403 4001 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:51.980415 4001 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:51.980431 4001 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nmlql of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:52.848976 4001 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl 2257dfebaa3a27a7d69ffd1a38df53265c261760d708c03391b02594483a8977 network default NAD default] DEL starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl 2257dfebaa3a27a7d69ffd1a38df53265c261760d708c03391b02594483a8977 network default NAD default] I0403 00:50:52.927151 4001 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl 2257dfebaa3a27a7d69ffd1a38df53265c261760d708c03391b02594483a8977 network default NAD default] DEL finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl 2257dfebaa3a27a7d69ffd1a38df53265c261760d708c03391b02594483a8977 network default NAD default], result "{}", err I0403 00:50:52.976897 4001 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:52.976931 4001 pods.go:176] Deleting pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl I0403 00:50:52.978402 4001 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl, ips: 10.131.0.34 I0403 00:50:52.978436 4001 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:52.978459 4001 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:50:52.978467 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0403 00:50:52.978660 4001 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:50:52.978677 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0403 00:50:53.856619 4001 cni.go:403] [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb 4cb642bbafe6401c81f97b206a9b914a030503cd1df6beb5ba960047295368d1 network default NAD default] DEL starting CNI request [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb 4cb642bbafe6401c81f97b206a9b914a030503cd1df6beb5ba960047295368d1 network default NAD default] I0403 00:50:53.930095 4001 cni.go:424] [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb 4cb642bbafe6401c81f97b206a9b914a030503cd1df6beb5ba960047295368d1 network default NAD default] DEL finished CNI request [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb 4cb642bbafe6401c81f97b206a9b914a030503cd1df6beb5ba960047295368d1 network default NAD default], result "{}", err I0403 00:50:53.944713 4001 base_network_controller_pods.go:487] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4] creating logical port openshift-marketplace_30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:50:53.944842 4001 kube.go:256] Updating pod openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 I0403 00:50:53.958486 4001 pod.go:62] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4] pod update took 13.666604ms I0403 00:50:53.958512 4001 base_network_controller_pods.go:937] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4] addLogicalPort annotation time took 13.701422ms I0403 00:50:53.959740 4001 pods.go:271] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4] addLogicalPort took 15.042807ms, libovsdb time 873.948µs I0403 00:50:53.997708 4001 obj_retry.go:464] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:53.997741 4001 pods.go:176] Deleting pod: openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb I0403 00:50:53.999201 4001 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb, ips: 10.131.0.35 I0403 00:50:53.999226 4001 obj_retry.go:464] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:53.999247 4001 obj_retry.go:464] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:50:53.999255 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0403 00:50:53.999435 4001 obj_retry.go:464] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:50:53.999451 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0403 00:50:54.005206 4001 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:54.005233 4001 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:54.005246 4001 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:54.005258 4001 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5nxxwl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:54.060567 4001 base_network_controller_pods.go:487] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t] creating logical port openshift-marketplace_18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t for pod on switch ip-10-0-1-98.ec2.internal I0403 00:50:54.060695 4001 kube.go:256] Updating pod openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t I0403 00:50:54.074064 4001 pod.go:62] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t] pod update took 13.39767ms I0403 00:50:54.074098 4001 base_network_controller_pods.go:937] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t] addLogicalPort annotation time took 13.424066ms I0403 00:50:54.075350 4001 pods.go:271] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t] addLogicalPort took 14.802131ms, libovsdb time 915.554µs I0403 00:50:54.303489 4001 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 84741065e3ccd196e333e6ace6e6ba88fb1f1b662cadf56aa6785231465978eb network default NAD default] ADD starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 84741065e3ccd196e333e6ace6e6ba88fb1f1b662cadf56aa6785231465978eb network default NAD default] I0403 00:50:54.312914 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4, hostIfaceName: 84741065e3ccd19, network: default, NAD default, SandboxID: "84741065e3ccd196e333e6ace6e6ba88fb1f1b662cadf56aa6785231465978eb", PCI device ID: , UID: "05981029-b2b2-47f9-8fcc-474c8e03b488", MAC: 0a:58:0a:83:00:25, IPs: [10.131.0.37/23] I0403 00:50:54.362155 4001 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 84741065e3ccd196e333e6ace6e6ba88fb1f1b662cadf56aa6785231465978eb network default NAD default] ADD finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 84741065e3ccd196e333e6ace6e6ba88fb1f1b662cadf56aa6785231465978eb network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:ab:45:6d:6b:fa\",\"name\":\"84741065e3ccd19\"},{\"mac\":\"0a:58:0a:83:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e51e4aa6-6a8f-403f-bc10-5af3d4c0bff0\"}],\"ips\":[{\"address\":\"10.131.0.37/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:50:54.422044 4001 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t 955e5ad3ef3e174d6c08c03a19a5a077e9744694d955dc47af12e9ed0970cc4a network default NAD default] ADD starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t 955e5ad3ef3e174d6c08c03a19a5a077e9744694d955dc47af12e9ed0970cc4a network default NAD default] I0403 00:50:54.432351 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t, hostIfaceName: 955e5ad3ef3e174, network: default, NAD default, SandboxID: "955e5ad3ef3e174d6c08c03a19a5a077e9744694d955dc47af12e9ed0970cc4a", PCI device ID: , UID: "750033c9-73fd-47fa-90d3-6376bf558f86", MAC: 0a:58:0a:83:00:26, IPs: [10.131.0.38/23] I0403 00:50:54.481462 4001 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t 955e5ad3ef3e174d6c08c03a19a5a077e9744694d955dc47af12e9ed0970cc4a network default NAD default] ADD finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t 955e5ad3ef3e174d6c08c03a19a5a077e9744694d955dc47af12e9ed0970cc4a network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:82:f6:1a:77:1f\",\"name\":\"955e5ad3ef3e174\"},{\"mac\":\"0a:58:0a:83:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b8e2ed52-55bb-40db-9293-0f3a55289337\"}],\"ips\":[{\"address\":\"10.131.0.38/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:50:55.019301 4001 obj_retry.go:458] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:55.019328 4001 obj_retry.go:458] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:55.019342 4001 obj_retry.go:458] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:55.019353 4001 obj_retry.go:458] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebvxjqb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:55.882375 4001 cni.go:403] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq 9be769e0dae2199fc431bfdffb1d51f94e9d0a57f39fbcf42a36256cef95bcdf network default NAD default] DEL starting CNI request [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq 9be769e0dae2199fc431bfdffb1d51f94e9d0a57f39fbcf42a36256cef95bcdf network default NAD default] I0403 00:50:55.962601 4001 cni.go:424] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq 9be769e0dae2199fc431bfdffb1d51f94e9d0a57f39fbcf42a36256cef95bcdf network default NAD default] DEL finished CNI request [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq 9be769e0dae2199fc431bfdffb1d51f94e9d0a57f39fbcf42a36256cef95bcdf network default NAD default], result "{}", err I0403 00:50:56.013865 4001 obj_retry.go:464] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:56.013907 4001 pods.go:176] Deleting pod: openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq I0403 00:50:56.015411 4001 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq, ips: 10.131.0.36 I0403 00:50:56.015442 4001 obj_retry.go:464] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:56.015461 4001 obj_retry.go:464] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:50:56.015469 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0403 00:50:56.015721 4001 obj_retry.go:464] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:50:56.015737 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0403 00:50:57.047490 4001 obj_retry.go:458] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:57.047521 4001 obj_retry.go:458] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:57.047545 4001 obj_retry.go:458] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:57.047557 4001 obj_retry.go:458] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece4ztkq of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:57.894708 4001 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t 955e5ad3ef3e174d6c08c03a19a5a077e9744694d955dc47af12e9ed0970cc4a network default NAD default] DEL starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t 955e5ad3ef3e174d6c08c03a19a5a077e9744694d955dc47af12e9ed0970cc4a network default NAD default] I0403 00:50:57.895805 4001 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 84741065e3ccd196e333e6ace6e6ba88fb1f1b662cadf56aa6785231465978eb network default NAD default] DEL starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 84741065e3ccd196e333e6ace6e6ba88fb1f1b662cadf56aa6785231465978eb network default NAD default] I0403 00:50:57.977345 4001 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t 955e5ad3ef3e174d6c08c03a19a5a077e9744694d955dc47af12e9ed0970cc4a network default NAD default] DEL finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t 955e5ad3ef3e174d6c08c03a19a5a077e9744694d955dc47af12e9ed0970cc4a network default NAD default], result "{}", err I0403 00:50:57.985969 4001 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 84741065e3ccd196e333e6ace6e6ba88fb1f1b662cadf56aa6785231465978eb network default NAD default] DEL finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 84741065e3ccd196e333e6ace6e6ba88fb1f1b662cadf56aa6785231465978eb network default NAD default], result "{}", err I0403 00:50:58.045984 4001 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:58.046017 4001 pods.go:176] Deleting pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t I0403 00:50:58.047457 4001 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t, ips: 10.131.0.38 I0403 00:50:58.047494 4001 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:58.047521 4001 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:50:58.047534 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0403 00:50:58.047731 4001 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:50:58.047749 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0403 00:50:58.071385 4001 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:58.071418 4001 pods.go:176] Deleting pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 I0403 00:50:58.072793 4001 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4, ips: 10.131.0.37 I0403 00:50:58.072825 4001 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:50:58.072852 4001 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:50:58.072862 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0403 00:50:58.073039 4001 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0403 00:50:58.073053 4001 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0403 00:50:59.168501 4001 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:59.168527 4001 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:59.168538 4001 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:59.168564 4001 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7qqf7t of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:59.403185 4001 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:59.403208 4001 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:59.403231 4001 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:50:59.403241 4001 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf6tdj4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:51:05.400242 4001 namespace.go:98] [cert-manager] adding namespace I0403 00:51:05.401093 4001 namespace.go:102] [cert-manager] adding namespace took 815.232µs I0403 00:51:05.418046 4001 namespace.go:142] [cert-manager] updating namespace I0403 00:51:05.427667 4001 namespace.go:142] [cert-manager] updating namespace I0403 00:51:10.917327 4001 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 W0403 00:51:12.251012 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0403 00:51:12.489820 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0403 00:51:12.496619 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0403 00:51:13.430602 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0403 00:51:13.438393 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0403 00:51:13.447872 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0403 00:51:13.453268 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0403 00:51:15.268231 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0403 00:51:15.268252 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0403 00:51:18.605676 4001 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-ktd5g] creating logical port konflux-operator_konflux-operator-controller-manager-5bc5d8c98f-ktd5g for pod on switch ip-10-0-1-98.ec2.internal I0403 00:51:18.605803 4001 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-ktd5g I0403 00:51:18.629562 4001 pod.go:62] [konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-ktd5g] pod update took 23.783041ms I0403 00:51:18.629589 4001 base_network_controller_pods.go:937] [default/konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-ktd5g] addLogicalPort annotation time took 23.818475ms I0403 00:51:18.630732 4001 pods.go:271] [konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-ktd5g] addLogicalPort took 25.078936ms, libovsdb time 827.819µs I0403 00:51:18.900905 4001 namespace.go:142] [openshift-operators] updating namespace I0403 00:51:18.910312 4001 namespace.go:142] [openshift-operators] updating namespace I0403 00:51:18.968145 4001 cni.go:403] [konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-ktd5g b20dc12401c100effc0a298d63a48b09eb6cf5b805108528cd7be1f98f2f1fc8 network default NAD default] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-ktd5g b20dc12401c100effc0a298d63a48b09eb6cf5b805108528cd7be1f98f2f1fc8 network default NAD default] I0403 00:51:18.978846 4001 helper_linux.go:450] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-5bc5d8c98f-ktd5g, hostIfaceName: b20dc12401c100e, network: default, NAD default, SandboxID: "b20dc12401c100effc0a298d63a48b09eb6cf5b805108528cd7be1f98f2f1fc8", PCI device ID: , UID: "5e3940fd-eaaf-4aed-874c-bc6e14839f9c", MAC: 0a:58:0a:83:00:27, IPs: [10.131.0.39/23] I0403 00:51:19.028782 4001 cni.go:424] [konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-ktd5g b20dc12401c100effc0a298d63a48b09eb6cf5b805108528cd7be1f98f2f1fc8 network default NAD default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-ktd5g b20dc12401c100effc0a298d63a48b09eb6cf5b805108528cd7be1f98f2f1fc8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:36:71:08:56:70\",\"name\":\"b20dc12401c100e\"},{\"mac\":\"0a:58:0a:83:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b57dac64-fa09-47ac-ac9c-2d5f1292d8dd\"}],\"ips\":[{\"address\":\"10.131.0.39/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err W0403 00:51:22.131919 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0403 00:51:22.951332 4001 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0403 00:51:24.752147 4001 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-k7h8r] creating logical port openshift-operators_rhtas-operator-controller-manager-6987b7f98c-k7h8r for pod on switch ip-10-0-1-98.ec2.internal I0403 00:51:24.752289 4001 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-k7h8r I0403 00:51:24.806294 4001 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-k7h8r] pod update took 54.030201ms I0403 00:51:24.806321 4001 base_network_controller_pods.go:937] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-k7h8r] addLogicalPort annotation time took 54.064273ms I0403 00:51:24.807568 4001 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-k7h8r] addLogicalPort took 55.437987ms, libovsdb time 900.428µs I0403 00:51:25.118326 4001 cni.go:403] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-k7h8r 1c66fce0dcfaa2878400cc18375d8f491590259a17e001e49876f072d7c8fc64 network default NAD default] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-k7h8r 1c66fce0dcfaa2878400cc18375d8f491590259a17e001e49876f072d7c8fc64 network default NAD default] I0403 00:51:25.128206 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-6987b7f98c-k7h8r, hostIfaceName: 1c66fce0dcfaa28, network: default, NAD default, SandboxID: "1c66fce0dcfaa2878400cc18375d8f491590259a17e001e49876f072d7c8fc64", PCI device ID: , UID: "1b932582-6be3-426a-841b-feb6678135e3", MAC: 0a:58:0a:83:00:28, IPs: [10.131.0.40/23] I0403 00:51:25.180018 4001 cni.go:424] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-k7h8r 1c66fce0dcfaa2878400cc18375d8f491590259a17e001e49876f072d7c8fc64 network default NAD default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-k7h8r 1c66fce0dcfaa2878400cc18375d8f491590259a17e001e49876f072d7c8fc64 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:3f:26:6b:28:bc\",\"name\":\"1c66fce0dcfaa28\"},{\"mac\":\"0a:58:0a:83:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/964d1090-1db2-4505-87cb-8013ae20344d\"}],\"ips\":[{\"address\":\"10.131.0.40/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:51:28.160996 4001 namespace.go:98] [trusted-artifact-signer] adding namespace I0403 00:51:28.161942 4001 namespace.go:102] [trusted-artifact-signer] adding namespace took 922.767µs I0403 00:51:28.166987 4001 namespace.go:142] [trusted-artifact-signer] updating namespace W0403 00:51:28.219875 4001 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0403 00:51:28.304321 4001 namespace.go:142] [trusted-artifact-signer] updating namespace I0403 00:51:28.416246 4001 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-bvdqs] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-bvdqs for pod on switch ip-10-0-1-98.ec2.internal I0403 00:51:28.416335 4001 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-bvdqs I0403 00:51:28.436911 4001 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-bvdqs] pod update took 20.592736ms I0403 00:51:28.436944 4001 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-bvdqs] addLogicalPort annotation time took 20.63155ms I0403 00:51:28.438117 4001 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-bvdqs] addLogicalPort took 21.885863ms, libovsdb time 789.931µs I0403 00:51:28.790859 4001 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-bvdqs 5f27e5ec6c79f6e743b20fbdf0b9e18a086c2ee6c6897dbd79036380ece08942 network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-bvdqs 5f27e5ec6c79f6e743b20fbdf0b9e18a086c2ee6c6897dbd79036380ece08942 network default NAD default] I0403 00:51:28.799912 4001 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-bvdqs, hostIfaceName: 5f27e5ec6c79f6e, network: default, NAD default, SandboxID: "5f27e5ec6c79f6e743b20fbdf0b9e18a086c2ee6c6897dbd79036380ece08942", PCI device ID: , UID: "0af57bf5-f0c6-416a-84e6-ccf0ef53a364", MAC: 0a:58:0a:83:00:29, IPs: [10.131.0.41/23] I0403 00:51:28.854973 4001 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-bvdqs 5f27e5ec6c79f6e743b20fbdf0b9e18a086c2ee6c6897dbd79036380ece08942 network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-bvdqs 5f27e5ec6c79f6e743b20fbdf0b9e18a086c2ee6c6897dbd79036380ece08942 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:34:6f:d6:aa:49\",\"name\":\"5f27e5ec6c79f6e\"},{\"mac\":\"0a:58:0a:83:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/de425756-e221-4add-95f5-4152d63f4998\"}],\"ips\":[{\"address\":\"10.131.0.41/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:51:31.040484 4001 namespace.go:142] [openshift-operators] updating namespace I0403 00:51:31.382399 4001 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6684678d86-wjt46] creating logical port openshift-operators_openshift-pipelines-operator-6684678d86-wjt46 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:51:31.382511 4001 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6684678d86-wjt46 I0403 00:51:31.444829 4001 pod.go:62] [openshift-operators/openshift-pipelines-operator-6684678d86-wjt46] pod update took 62.341682ms I0403 00:51:31.444855 4001 base_network_controller_pods.go:937] [default/openshift-operators/openshift-pipelines-operator-6684678d86-wjt46] addLogicalPort annotation time took 62.374858ms I0403 00:51:31.446077 4001 pods.go:271] [openshift-operators/openshift-pipelines-operator-6684678d86-wjt46] addLogicalPort took 63.696778ms, libovsdb time 845.602µs I0403 00:51:31.732183 4001 cni.go:403] [openshift-operators/openshift-pipelines-operator-6684678d86-wjt46 b7c1c96e63a5d8ea0cd989e603a1cd29c22cd54a366b54865e30a92de34ff9e5 network default NAD default] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-wjt46 b7c1c96e63a5d8ea0cd989e603a1cd29c22cd54a366b54865e30a92de34ff9e5 network default NAD default] I0403 00:51:31.741962 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6684678d86-wjt46, hostIfaceName: b7c1c96e63a5d8e, network: default, NAD default, SandboxID: "b7c1c96e63a5d8ea0cd989e603a1cd29c22cd54a366b54865e30a92de34ff9e5", PCI device ID: , UID: "bcae84d5-3974-445a-bdf8-8658d6478c5f", MAC: 0a:58:0a:83:00:2a, IPs: [10.131.0.42/23] I0403 00:51:31.790810 4001 cni.go:424] [openshift-operators/openshift-pipelines-operator-6684678d86-wjt46 b7c1c96e63a5d8ea0cd989e603a1cd29c22cd54a366b54865e30a92de34ff9e5 network default NAD default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-wjt46 b7c1c96e63a5d8ea0cd989e603a1cd29c22cd54a366b54865e30a92de34ff9e5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:ec:06:2c:2d:f2\",\"name\":\"b7c1c96e63a5d8e\"},{\"mac\":\"0a:58:0a:83:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c544b942-eee7-4d8d-adbd-f591aaeae696\"}],\"ips\":[{\"address\":\"10.131.0.42/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err W0403 00:51:33.036874 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0403 00:51:39.111018 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0403 00:51:48.514069 4001 namespace.go:98] [openshift-pipelines] adding namespace I0403 00:51:48.515075 4001 namespace.go:102] [openshift-pipelines] adding namespace took 960.122µs I0403 00:51:48.533075 4001 namespace.go:142] [openshift-pipelines] updating namespace I0403 00:51:49.229803 4001 namespace.go:142] [cert-manager] updating namespace I0403 00:51:49.427178 4001 namespace.go:142] [cert-manager-operator] updating namespace I0403 00:51:49.830305 4001 namespace.go:142] [konflux-operator] updating namespace I0403 00:51:50.031755 4001 namespace.go:142] [konflux-ui] updating namespace I0403 00:51:50.232787 4001 namespace.go:142] [open-cluster-management-2peb9ob26umtfq43icvb8p97pncv0c79] updating namespace I0403 00:51:50.633793 4001 namespace.go:142] [rhbk-operator] updating namespace I0403 00:51:50.832608 4001 namespace.go:142] [rhtpa-operator] updating namespace I0403 00:51:51.035155 4001 namespace.go:142] [trusted-artifact-signer] updating namespace I0403 00:51:51.234810 4001 namespace.go:142] [tsf] updating namespace I0403 00:51:51.438044 4001 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:51:51.638025 4001 namespace.go:142] [tsf-tas] updating namespace I0403 00:51:51.839342 4001 namespace.go:142] [tsf-tpa] updating namespace I0403 00:51:51.967106 4001 namespace.go:142] [cert-manager] updating namespace I0403 00:51:51.975380 4001 namespace.go:142] [cert-manager-operator] updating namespace I0403 00:51:51.984468 4001 namespace.go:142] [default] updating namespace I0403 00:51:51.993577 4001 namespace.go:142] [konflux-operator] updating namespace I0403 00:51:52.002229 4001 namespace.go:142] [konflux-ui] updating namespace I0403 00:51:52.008995 4001 namespace.go:142] [open-cluster-management-2peb9ob26umtfq43icvb8p97pncv0c79] updating namespace I0403 00:51:52.017284 4001 namespace.go:142] [openshift] updating namespace I0403 00:51:52.024920 4001 namespace.go:142] [rhbk-operator] updating namespace I0403 00:51:52.034453 4001 namespace.go:142] [rhtpa-operator] updating namespace I0403 00:51:52.040949 4001 namespace.go:142] [trusted-artifact-signer] updating namespace I0403 00:51:52.059816 4001 namespace.go:142] [tsf] updating namespace I0403 00:51:52.080611 4001 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:51:52.098692 4001 namespace.go:142] [tsf-tas] updating namespace I0403 00:51:52.120656 4001 namespace.go:142] [tsf-tpa] updating namespace I0403 00:51:52.221351 4001 namespace.go:142] [cert-manager] updating namespace I0403 00:51:52.320147 4001 namespace.go:142] [cert-manager-operator] updating namespace I0403 00:51:52.422173 4001 namespace.go:142] [default] updating namespace I0403 00:51:52.522395 4001 namespace.go:142] [konflux-operator] updating namespace I0403 00:51:52.623283 4001 namespace.go:142] [konflux-ui] updating namespace I0403 00:51:52.726175 4001 namespace.go:142] [open-cluster-management-2peb9ob26umtfq43icvb8p97pncv0c79] updating namespace I0403 00:51:52.819015 4001 namespace.go:142] [openshift] updating namespace I0403 00:51:52.920944 4001 namespace.go:142] [rhbk-operator] updating namespace I0403 00:51:53.021380 4001 namespace.go:142] [rhtpa-operator] updating namespace I0403 00:51:53.120363 4001 namespace.go:142] [trusted-artifact-signer] updating namespace I0403 00:51:53.225676 4001 namespace.go:142] [tsf] updating namespace I0403 00:51:53.321729 4001 namespace.go:142] [tsf-keycloak] updating namespace I0403 00:51:53.423456 4001 namespace.go:142] [tsf-tas] updating namespace I0403 00:51:53.520892 4001 namespace.go:142] [tsf-tpa] updating namespace I0403 00:51:53.785293 4001 namespace.go:142] [openshift-pipelines] updating namespace I0403 00:51:59.645366 4001 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:51:59.645889 4001 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:51:59.669399 4001 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:51:59.669418 4001 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:51:59.676560 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0403 00:51:59.676587 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions W0403 00:52:05.556766 4001 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0403 00:52:05.556788 4001 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0403 00:52:05.556794 4001 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0403 00:52:06.562408 4001 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0403 00:52:06.562429 4001 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0403 00:52:06.562434 4001 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0403 00:52:06.562450 4001 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0403 00:52:08.605740 4001 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0403 00:52:08.605763 4001 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0403 00:52:08.605768 4001 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0403 00:52:10.826689 4001 namespace.go:142] [openshift-pipelines] updating namespace W0403 00:52:16.606557 4001 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0403 00:52:16.626203 4001 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0403 00:52:25.891124 4001 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-kg6md] creating logical port openshift-pipelines_tekton-triggers-controller-68564d75ff-kg6md for pod on switch ip-10-0-1-98.ec2.internal I0403 00:52:25.891245 4001 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-68564d75ff-kg6md I0403 00:52:25.913203 4001 pod.go:62] [openshift-pipelines/tekton-triggers-controller-68564d75ff-kg6md] pod update took 21.976983ms I0403 00:52:25.913233 4001 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-kg6md] addLogicalPort annotation time took 22.012612ms I0403 00:52:25.914864 4001 pods.go:271] [openshift-pipelines/tekton-triggers-controller-68564d75ff-kg6md] addLogicalPort took 23.779311ms, libovsdb time 1.120872ms I0403 00:52:25.965702 4001 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-8gxmh] creating logical port openshift-pipelines_tekton-triggers-webhook-55d94ff6c6-8gxmh for pod on switch ip-10-0-1-98.ec2.internal I0403 00:52:25.965800 4001 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-8gxmh I0403 00:52:25.980220 4001 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-8gxmh] pod update took 14.440263ms I0403 00:52:25.980252 4001 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-8gxmh] addLogicalPort annotation time took 14.475128ms I0403 00:52:25.981670 4001 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-8gxmh] addLogicalPort took 15.989987ms, libovsdb time 939.939µs I0403 00:52:26.256532 4001 cni.go:403] [openshift-pipelines/tekton-triggers-controller-68564d75ff-kg6md 5fafc8e9e4978d17a5ac297b031b8d47808700626954db4689e7b2c67d5ebe56 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-kg6md 5fafc8e9e4978d17a5ac297b031b8d47808700626954db4689e7b2c67d5ebe56 network default NAD default] I0403 00:52:26.265738 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-68564d75ff-kg6md, hostIfaceName: 5fafc8e9e4978d1, network: default, NAD default, SandboxID: "5fafc8e9e4978d17a5ac297b031b8d47808700626954db4689e7b2c67d5ebe56", PCI device ID: , UID: "12357a8b-4ec7-4fa1-88ad-d91510f20722", MAC: 0a:58:0a:83:00:2b, IPs: [10.131.0.43/23] I0403 00:52:26.321371 4001 cni.go:424] [openshift-pipelines/tekton-triggers-controller-68564d75ff-kg6md 5fafc8e9e4978d17a5ac297b031b8d47808700626954db4689e7b2c67d5ebe56 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-kg6md 5fafc8e9e4978d17a5ac297b031b8d47808700626954db4689e7b2c67d5ebe56 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:0f:54:35:c1:4e\",\"name\":\"5fafc8e9e4978d1\"},{\"mac\":\"0a:58:0a:83:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d6084429-2a5f-4bd9-bfd9-ce39118e4a9f\"}],\"ips\":[{\"address\":\"10.131.0.43/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:52:26.329375 4001 cni.go:403] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-8gxmh 504350c37568b86f35ecc5450ad88c2463eb5b18b5558973462de844966a3754 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-8gxmh 504350c37568b86f35ecc5450ad88c2463eb5b18b5558973462de844966a3754 network default NAD default] I0403 00:52:26.339440 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-55d94ff6c6-8gxmh, hostIfaceName: 504350c37568b86, network: default, NAD default, SandboxID: "504350c37568b86f35ecc5450ad88c2463eb5b18b5558973462de844966a3754", PCI device ID: , UID: "8b3b6f8c-d01b-4a15-abc6-80ed3e198eba", MAC: 0a:58:0a:83:00:2c, IPs: [10.131.0.44/23] I0403 00:52:26.391300 4001 cni.go:424] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-8gxmh 504350c37568b86f35ecc5450ad88c2463eb5b18b5558973462de844966a3754 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-8gxmh 504350c37568b86f35ecc5450ad88c2463eb5b18b5558973462de844966a3754 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:a0:98:95:2a:3a\",\"name\":\"504350c37568b86\"},{\"mac\":\"0a:58:0a:83:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/caeae248-f8d3-40f5-a364-fb11fdadc9e3\"}],\"ips\":[{\"address\":\"10.131.0.44/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err W0403 00:52:29.620673 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0403 00:52:39.864115 4001 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:52:39.864662 4001 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:52:39.903344 4001 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:52:39.903367 4001 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:52:39.910305 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0403 00:52:39.910337 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure W0403 00:52:50.737399 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0403 00:52:50.739593 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:52:50.739608 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0403 00:52:51.664897 4001 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0403 00:52:54.735218 4001 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:52:54.735337 4001 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0403 00:52:54.746749 4001 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 11.455804ms I0403 00:52:54.746772 4001 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 11.486309ms I0403 00:52:54.748080 4001 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 12.881591ms, libovsdb time 847.508µs I0403 00:52:57.947808 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0403 00:52:59.598825 4001 cni.go:403] [openshift-pipelines/tekton-results-postgres-0 ba6693f8de98dd3dcb4bc2b931a527b208bac0f07e68d1403fd0e8db29ca78b3 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 ba6693f8de98dd3dcb4bc2b931a527b208bac0f07e68d1403fd0e8db29ca78b3 network default NAD default] I0403 00:52:59.608567 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: ba6693f8de98dd3, network: default, NAD default, SandboxID: "ba6693f8de98dd3dcb4bc2b931a527b208bac0f07e68d1403fd0e8db29ca78b3", PCI device ID: , UID: "bca54e64-b4b9-4eab-8572-03480931fa8e", MAC: 0a:58:0a:83:00:2d, IPs: [10.131.0.45/23] I0403 00:52:59.660035 4001 cni.go:424] [openshift-pipelines/tekton-results-postgres-0 ba6693f8de98dd3dcb4bc2b931a527b208bac0f07e68d1403fd0e8db29ca78b3 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 ba6693f8de98dd3dcb4bc2b931a527b208bac0f07e68d1403fd0e8db29ca78b3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:8e:36:df:a9:3a\",\"name\":\"ba6693f8de98dd3\"},{\"mac\":\"0a:58:0a:83:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1c881caa-d634-4adc-aabd-07cc1c95a5f1\"}],\"ips\":[{\"address\":\"10.131.0.45/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:53:09.680058 4001 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0403 00:53:11.002461 4001 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-7dzjw] creating logical port openshift-pipelines_tekton-results-api-7b4b7bb78c-7dzjw for pod on switch ip-10-0-1-98.ec2.internal I0403 00:53:11.002622 4001 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-7b4b7bb78c-7dzjw I0403 00:53:11.017441 4001 pod.go:62] [openshift-pipelines/tekton-results-api-7b4b7bb78c-7dzjw] pod update took 14.852615ms I0403 00:53:11.017467 4001 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-7dzjw] addLogicalPort annotation time took 14.88418ms I0403 00:53:11.018784 4001 pods.go:271] [openshift-pipelines/tekton-results-api-7b4b7bb78c-7dzjw] addLogicalPort took 16.344729ms, libovsdb time 864.281µs I0403 00:53:11.276261 4001 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-v9b7s] creating logical port tsf-keycloak_tsf-iam-v9b7s for pod on switch ip-10-0-1-98.ec2.internal I0403 00:53:11.276388 4001 kube.go:256] Updating pod tsf-keycloak/tsf-iam-v9b7s I0403 00:53:11.290374 4001 pod.go:62] [tsf-keycloak/tsf-iam-v9b7s] pod update took 14.010612ms I0403 00:53:11.290403 4001 base_network_controller_pods.go:937] [default/tsf-keycloak/tsf-iam-v9b7s] addLogicalPort annotation time took 14.042822ms I0403 00:53:11.291769 4001 pods.go:271] [tsf-keycloak/tsf-iam-v9b7s] addLogicalPort took 15.527902ms, libovsdb time 895.962µs I0403 00:53:11.637233 4001 cni.go:403] [tsf-keycloak/tsf-iam-v9b7s a9b47d587eaf4c48a62667e7bbcd98a8a8195746556fd79cec2280c88d6be978 network default NAD default] ADD starting CNI request [tsf-keycloak/tsf-iam-v9b7s a9b47d587eaf4c48a62667e7bbcd98a8a8195746556fd79cec2280c88d6be978 network default NAD default] I0403 00:53:11.646476 4001 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-v9b7s, hostIfaceName: a9b47d587eaf4c4, network: default, NAD default, SandboxID: "a9b47d587eaf4c48a62667e7bbcd98a8a8195746556fd79cec2280c88d6be978", PCI device ID: , UID: "f8f51196-75e3-4a18-9d0f-4720b924d40d", MAC: 0a:58:0a:83:00:2f, IPs: [10.131.0.47/23] I0403 00:53:11.699883 4001 cni.go:424] [tsf-keycloak/tsf-iam-v9b7s a9b47d587eaf4c48a62667e7bbcd98a8a8195746556fd79cec2280c88d6be978 network default NAD default] ADD finished CNI request [tsf-keycloak/tsf-iam-v9b7s a9b47d587eaf4c48a62667e7bbcd98a8a8195746556fd79cec2280c88d6be978 network default NAD default], result "{\"interfaces\":[{\"mac\":\"76:29:14:f3:27:60\",\"name\":\"a9b47d587eaf4c4\"},{\"mac\":\"0a:58:0a:83:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/280963b9-8212-4288-ba34-3dd270eef800\"}],\"ips\":[{\"address\":\"10.131.0.47/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:53:11.963776 4001 cni.go:403] [openshift-pipelines/tekton-results-api-7b4b7bb78c-7dzjw 5e2506acccafb138d60dc48dfd67808e97e1f3baf6da0c3e0ba6958090b0b9b1 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-7dzjw 5e2506acccafb138d60dc48dfd67808e97e1f3baf6da0c3e0ba6958090b0b9b1 network default NAD default] I0403 00:53:11.973986 4001 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-7b4b7bb78c-7dzjw, hostIfaceName: 5e2506acccafb13, network: default, NAD default, SandboxID: "5e2506acccafb138d60dc48dfd67808e97e1f3baf6da0c3e0ba6958090b0b9b1", PCI device ID: , UID: "2630c02a-aff8-4f08-a701-b427a68df2c3", MAC: 0a:58:0a:83:00:2e, IPs: [10.131.0.46/23] I0403 00:53:12.024905 4001 cni.go:424] [openshift-pipelines/tekton-results-api-7b4b7bb78c-7dzjw 5e2506acccafb138d60dc48dfd67808e97e1f3baf6da0c3e0ba6958090b0b9b1 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-7dzjw 5e2506acccafb138d60dc48dfd67808e97e1f3baf6da0c3e0ba6958090b0b9b1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:8c:7d:0c:53:98\",\"name\":\"5e2506acccafb13\"},{\"mac\":\"0a:58:0a:83:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/55fe340a-9112-4a67-8959-0616429e8459\"}],\"ips\":[{\"address\":\"10.131.0.46/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:53:12.120557 4001 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:53:12.121106 4001 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:53:12.873307 4001 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:53:12.873331 4001 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:53:12.877932 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0403 00:53:12.877964 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0403 00:53:13.886961 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:53:13.886983 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:53:13.891139 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0403 00:53:18.690682 4001 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0403 00:53:18.690705 4001 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0403 00:53:18.690711 4001 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0403 00:53:31.972468 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0403 00:53:31.972490 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0403 00:53:31.972495 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0403 00:53:35.148235 4001 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:53:35.148874 4001 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:53:35.169989 4001 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:53:35.170016 4001 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:53:35.176054 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0403 00:53:35.176103 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0403 00:53:41.856265 4001 cni.go:403] [tsf-keycloak/tsf-iam-v9b7s a9b47d587eaf4c48a62667e7bbcd98a8a8195746556fd79cec2280c88d6be978 network default NAD default] DEL starting CNI request [tsf-keycloak/tsf-iam-v9b7s a9b47d587eaf4c48a62667e7bbcd98a8a8195746556fd79cec2280c88d6be978 network default NAD default] I0403 00:53:41.940763 4001 cni.go:424] [tsf-keycloak/tsf-iam-v9b7s a9b47d587eaf4c48a62667e7bbcd98a8a8195746556fd79cec2280c88d6be978 network default NAD default] DEL finished CNI request [tsf-keycloak/tsf-iam-v9b7s a9b47d587eaf4c48a62667e7bbcd98a8a8195746556fd79cec2280c88d6be978 network default NAD default], result "{}", err I0403 00:53:42.046243 4001 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-v9b7s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:53:42.046282 4001 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-v9b7s I0403 00:53:42.047647 4001 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-v9b7s, ips: 10.131.0.47 I0403 00:53:42.047681 4001 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-v9b7s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:53:43.073334 4001 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-v9b7s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:53:43.073357 4001 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-v9b7s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:53:43.447792 4001 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:53:43.448389 4001 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:53:44.060308 4001 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:53:44.060344 4001 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:53:44.066491 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0403 00:53:44.066516 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0403 00:53:44.099941 4001 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-1-98.ec2.internal I0403 00:53:44.100036 4001 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0403 00:53:44.127112 4001 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 27.096371ms I0403 00:53:44.127139 4001 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 27.128974ms I0403 00:53:44.128338 4001 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 28.414753ms, libovsdb time 810.578µs I0403 00:53:44.462889 4001 cni.go:403] [tsf-keycloak/keycloak-0 d28cb61ccbb89263e7e8eb24c331e8954e377726f744b59dd57df8957a69682b network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 d28cb61ccbb89263e7e8eb24c331e8954e377726f744b59dd57df8957a69682b network default NAD default] I0403 00:53:44.472635 4001 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: d28cb61ccbb8926, network: default, NAD default, SandboxID: "d28cb61ccbb89263e7e8eb24c331e8954e377726f744b59dd57df8957a69682b", PCI device ID: , UID: "4c561b98-b726-411f-bd70-7a2690ebe4b9", MAC: 0a:58:0a:83:00:30, IPs: [10.131.0.48/23] I0403 00:53:44.524571 4001 cni.go:424] [tsf-keycloak/keycloak-0 d28cb61ccbb89263e7e8eb24c331e8954e377726f744b59dd57df8957a69682b network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 d28cb61ccbb89263e7e8eb24c331e8954e377726f744b59dd57df8957a69682b network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:5f:05:3a:09:ea\",\"name\":\"d28cb61ccbb8926\"},{\"mac\":\"0a:58:0a:83:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f3a5f5e8-ec42-4ef1-81ef-76b9eded5fc8\"}],\"ips\":[{\"address\":\"10.131.0.48/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err W0403 00:53:45.085804 4001 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0403 00:53:45.085824 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0403 00:53:45.878832 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:53:45.878850 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:53:45.878928 4001 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0403 00:53:47.961556 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 W0403 00:53:50.086024 4001 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0403 00:53:59.185661 4001 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0403 00:53:59.784406 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:53:59.784429 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0403 00:53:59.786997 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0403 00:54:01.194205 4001 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0403 00:54:02.320868 4001 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-w9nfj-fvhdm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:54:02.321454 4001 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-w9nfj-fvhdm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:54:03.240100 4001 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-w9nfj-fvhdm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:54:03.240124 4001 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-w9nfj-fvhdm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0403 00:54:20.247887 4001 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0403 00:54:23.267271 4001 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0403 00:54:23.267298 4001 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller I0403 00:54:27.614125 4001 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:54:27.614679 4001 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:54:27.633563 4001 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:54:27.633581 4001 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:54:27.637781 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0403 00:54:27.637808 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0403 00:54:36.473113 4001 obj_retry.go:464] Detected object tsf/patch-tekton-config-ft5z9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:54:36.473661 4001 obj_retry.go:464] Detected object tsf/patch-tekton-config-ft5z9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:54:37.492805 4001 obj_retry.go:458] Detected object tsf/patch-tekton-config-ft5z9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:54:37.492834 4001 obj_retry.go:458] Detected object tsf/patch-tekton-config-ft5z9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:54:41.505522 4001 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-hqlnx-zzhfz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:54:41.506142 4001 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-hqlnx-zzhfz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:54:42.381057 4001 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-hqlnx-zzhfz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:54:42.381100 4001 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-hqlnx-zzhfz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:54:44.490851 4001 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-5mpsc-sfhx4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:54:44.491468 4001 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-5mpsc-sfhx4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:54:45.409956 4001 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-5mpsc-sfhx4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:54:45.409982 4001 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-5mpsc-sfhx4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0403 00:54:49.414878 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0403 00:54:49.609584 4001 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-8xzl8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:54:49.610049 4001 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-8xzl8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:54:50.473238 4001 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-8xzl8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:54:50.473271 4001 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-8xzl8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:54:50.523867 4001 obj_retry.go:458] Detected object tsf/patch-tekton-config-ft5z9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:54:50.523895 4001 obj_retry.go:458] Detected object tsf/patch-tekton-config-ft5z9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:54:50.526243 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-ft5z9 I0403 00:54:50.526267 4001 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-ft5z9 W0403 00:54:53.490710 4001 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0403 00:54:54.498518 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0403 00:54:59.471675 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0403 00:55:15.907921 4001 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-8mhkm-tzkdb] creating logical port tsf-tas_tuf-repository-init-8mhkm-tzkdb for pod on switch ip-10-0-1-98.ec2.internal I0403 00:55:15.908015 4001 kube.go:256] Updating pod tsf-tas/tuf-repository-init-8mhkm-tzkdb I0403 00:55:15.919874 4001 pod.go:62] [tsf-tas/tuf-repository-init-8mhkm-tzkdb] pod update took 11.881262ms I0403 00:55:15.919902 4001 base_network_controller_pods.go:937] [default/tsf-tas/tuf-repository-init-8mhkm-tzkdb] addLogicalPort annotation time took 11.913524ms I0403 00:55:15.921428 4001 pods.go:271] [tsf-tas/tuf-repository-init-8mhkm-tzkdb] addLogicalPort took 13.531339ms, libovsdb time 987.944µs I0403 00:55:22.870938 4001 cni.go:403] [tsf-tas/tuf-repository-init-8mhkm-tzkdb 550ab6afabe9a23835acd979578b64df6cb1b2180cb087f42fb90b848dcec4b0 network default NAD default] ADD starting CNI request [tsf-tas/tuf-repository-init-8mhkm-tzkdb 550ab6afabe9a23835acd979578b64df6cb1b2180cb087f42fb90b848dcec4b0 network default NAD default] I0403 00:55:22.880415 4001 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-8mhkm-tzkdb, hostIfaceName: 550ab6afabe9a23, network: default, NAD default, SandboxID: "550ab6afabe9a23835acd979578b64df6cb1b2180cb087f42fb90b848dcec4b0", PCI device ID: , UID: "39b37e68-d1da-4326-8889-c26b4d07d9b6", MAC: 0a:58:0a:83:00:31, IPs: [10.131.0.49/23] I0403 00:55:22.932327 4001 cni.go:424] [tsf-tas/tuf-repository-init-8mhkm-tzkdb 550ab6afabe9a23835acd979578b64df6cb1b2180cb087f42fb90b848dcec4b0 network default NAD default] ADD finished CNI request [tsf-tas/tuf-repository-init-8mhkm-tzkdb 550ab6afabe9a23835acd979578b64df6cb1b2180cb087f42fb90b848dcec4b0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:6b:8e:83:7e:4c\",\"name\":\"550ab6afabe9a23\"},{\"mac\":\"0a:58:0a:83:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/05e251c3-72cf-4407-927e-b3215b531519\"}],\"ips\":[{\"address\":\"10.131.0.49/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err I0403 00:55:27.473850 4001 cni.go:403] [tsf-tas/tuf-repository-init-8mhkm-tzkdb 550ab6afabe9a23835acd979578b64df6cb1b2180cb087f42fb90b848dcec4b0 network default NAD default] DEL starting CNI request [tsf-tas/tuf-repository-init-8mhkm-tzkdb 550ab6afabe9a23835acd979578b64df6cb1b2180cb087f42fb90b848dcec4b0 network default NAD default] I0403 00:55:27.554004 4001 cni.go:424] [tsf-tas/tuf-repository-init-8mhkm-tzkdb 550ab6afabe9a23835acd979578b64df6cb1b2180cb087f42fb90b848dcec4b0 network default NAD default] DEL finished CNI request [tsf-tas/tuf-repository-init-8mhkm-tzkdb 550ab6afabe9a23835acd979578b64df6cb1b2180cb087f42fb90b848dcec4b0 network default NAD default], result "{}", err I0403 00:55:27.603872 4001 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-8mhkm-tzkdb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 00:55:27.603909 4001 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-8mhkm-tzkdb I0403 00:55:27.605442 4001 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-8mhkm-tzkdb, ips: 10.131.0.49 I0403 00:55:27.605472 4001 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-8mhkm-tzkdb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 00:55:28.491676 4001 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-8mhkm-tzkdb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:55:28.491700 4001 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-8mhkm-tzkdb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 00:55:32.999750 4001 base_network_controller_pods.go:487] [default/tsf-tas/tuf-bc5768c88-gc4gz] creating logical port tsf-tas_tuf-bc5768c88-gc4gz for pod on switch ip-10-0-1-98.ec2.internal I0403 00:55:32.999858 4001 kube.go:256] Updating pod tsf-tas/tuf-bc5768c88-gc4gz I0403 00:55:33.014548 4001 pod.go:62] [tsf-tas/tuf-bc5768c88-gc4gz] pod update took 14.716017ms I0403 00:55:33.014578 4001 base_network_controller_pods.go:937] [default/tsf-tas/tuf-bc5768c88-gc4gz] addLogicalPort annotation time took 14.751452ms I0403 00:55:33.015911 4001 pods.go:271] [tsf-tas/tuf-bc5768c88-gc4gz] addLogicalPort took 16.182746ms, libovsdb time 857.967µs I0403 00:55:37.941684 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0403 00:55:42.988254 4001 cni.go:403] [tsf-tas/tuf-bc5768c88-gc4gz c297e633f33650d9f7f446d067a78348fd9dfeb5b235ba3b2f4693e96a4903e5 network default NAD default] ADD starting CNI request [tsf-tas/tuf-bc5768c88-gc4gz c297e633f33650d9f7f446d067a78348fd9dfeb5b235ba3b2f4693e96a4903e5 network default NAD default] I0403 00:55:42.997903 4001 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-bc5768c88-gc4gz, hostIfaceName: c297e633f33650d, network: default, NAD default, SandboxID: "c297e633f33650d9f7f446d067a78348fd9dfeb5b235ba3b2f4693e96a4903e5", PCI device ID: , UID: "5f5867ce-0af3-4551-8620-d0bbeb2511e9", MAC: 0a:58:0a:83:00:32, IPs: [10.131.0.50/23] I0403 00:55:43.049660 4001 cni.go:424] [tsf-tas/tuf-bc5768c88-gc4gz c297e633f33650d9f7f446d067a78348fd9dfeb5b235ba3b2f4693e96a4903e5 network default NAD default] ADD finished CNI request [tsf-tas/tuf-bc5768c88-gc4gz c297e633f33650d9f7f446d067a78348fd9dfeb5b235ba3b2f4693e96a4903e5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:46:7c:bf:f9:c3\",\"name\":\"c297e633f33650d\"},{\"mac\":\"0a:58:0a:83:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/39c14f31-a441-4319-85f8-9257a9b00e08\"}],\"ips\":[{\"address\":\"10.131.0.50/23\",\"gateway\":\"10.131.0.1\",\"interface\":1}]}", err W0403 00:55:44.559773 4001 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0403 00:55:55.568213 4001 util.go:959] Failed to build node endpoints for node ip-10-0-1-98.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf I0403 00:56:07.828212 4001 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0403 00:56:21.828657 4001 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=63 I0403 00:56:21.830296 4001 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=98 I0403 00:56:41.950836 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=11 I0403 00:56:47.925061 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=11 I0403 00:56:50.920419 4001 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0403 00:57:02.954467 4001 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0403 00:57:34.937265 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0403 00:57:45.831064 4001 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=140 I0403 00:58:59.957618 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0403 00:59:14.964058 4001 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0403 00:59:57.829926 4001 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=71 I0403 01:00:04.163251 4001 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586300-mqb86 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 01:00:04.163755 4001 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586300-mqb86 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0403 01:00:04.919700 4001 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0403 01:00:05.039658 4001 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586300-mqb86 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 01:00:05.039690 4001 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29586300-mqb86 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0403 01:00:40.834186 4001 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=476 I0403 01:00:55.547005 4001 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0403 01:00:55.547650 4001 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it