+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-195.ec2.internal ++ [[ -n ip-10-0-2-195.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-195.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0327 22:58:29.110932695 - disable conntrack on geneve port' I0327 22:58:29.110932695 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0327 22:58:29.128059153 - starting ovnkube-node + echo 'I0327 22:58:29.128059153 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-195.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-195.ec2.internal --init-node ip-10-0-2-195.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-195.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0327 22:58:29.170785 3901 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0327 22:58:29.170832 3901 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-29bd08d30b.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0327 22:58:29.172351 3901 kube.go:419] Waiting for certificate I0327 22:58:29.172394 3901 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0327 22:58:29.172431 3901 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0327 22:58:29.173587 3901 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 22:58:29.202286 3901 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 22:58:29.202302 3901 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 22:58:29.205414 3901 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 22:58:29.209615 3901 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-h7kfp" I0327 22:58:29.225045 3901 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-h7kfp" I0327 22:58:29.225106 3901 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 22:58:30.173140 3901 kube.go:426] Certificate found I0327 22:58:30.173740 3901 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 22:58:30.174715 3901 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0327 22:58:30.229618 3901 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-28 22:53:29 +0000 UTC" deadline="2026-03-28 18:31:34.625083509 +0000 UTC" I0327 22:58:30.229651 3901 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h33m4.395442257s" I0327 22:58:30.245792 3901 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 22:58:30.265117 3901 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0327 22:58:30.265277 3901 factory.go:531] Starting watch factory I0327 22:58:30.265362 3901 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.265378 3901 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.265381 3901 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.265382 3901 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.265399 3901 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.265397 3901 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.265405 3901 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.265415 3901 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.265851 3901 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.265894 3901 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.266169 3901 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.266191 3901 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.267267 3901 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0327 22:58:30.272749 3901 controller_manager.go:368] Starting the ovnkube controller I0327 22:58:30.272758 3901 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-195.ec2.internal I0327 22:58:30.272797 3901 controller_manager.go:393] NBDB zone sync took: 33.123µs I0327 22:58:30.272803 3901 factory.go:531] Starting watch factory I0327 22:58:30.292904 3901 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.294906 3901 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.295255 3901 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.295614 3901 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0327 22:58:30.295640 3901 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.296288 3901 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.297550 3901 ovn_northd.go:121] Registering OVN northd metrics I0327 22:58:30.301504 3901 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0327 22:58:30.301529 3901 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0327 22:58:30.305782 3901 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:30.315313 3901 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0327 22:58:30.371432 3901 factory.go:1890] *v1.Node informer cache synced successfully I0327 22:58:30.371446 3901 factory.go:1890] *v1.Service informer cache synced successfully I0327 22:58:30.371450 3901 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 22:58:30.371453 3901 factory.go:1890] *v1.Pod informer cache synced successfully I0327 22:58:30.371456 3901 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 22:58:30.371461 3901 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 22:58:30.371542 3901 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 22:58:30.371561 3901 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 22:58:30.371553 3901 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 22:58:30.371584 3901 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 22:58:30.374898 3901 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 22:58:30.374902 3901 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 22:58:30.376788 3901 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 22:58:30.376800 3901 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 22:58:30.376856 3901 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.376869 3901 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.378707 3901 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 22:58:30.378720 3901 factory.go:1890] *v1.Pod informer cache synced successfully I0327 22:58:30.378725 3901 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 22:58:30.378729 3901 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 22:58:30.378734 3901 factory.go:1890] *v1.Node informer cache synced successfully I0327 22:58:30.378737 3901 factory.go:1890] *v1.Service informer cache synced successfully I0327 22:58:30.379613 3901 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.382267 3901 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 22:58:30.382348 3901 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.382370 3901 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.384783 3901 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 22:58:30.384791 3901 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 22:58:30.385292 3901 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.388028 3901 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 22:58:30.388078 3901 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.388088 3901 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.390172 3901 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 22:58:30.390890 3901 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.393394 3901 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 22:58:30.393464 3901 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.393496 3901 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.395687 3901 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 22:58:30.396261 3901 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.398683 3901 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 22:58:30.398733 3901 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.398744 3901 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.400897 3901 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 22:58:30.401416 3901 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.403812 3901 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 22:58:30.403948 3901 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 22:58:30.403973 3901 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 22:58:30.406305 3901 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 22:58:30.406757 3901 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 22:58:30.409285 3901 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 22:58:30.409354 3901 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.409370 3901 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.409793 3901 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.409811 3901 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.411680 3901 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 22:58:30.412138 3901 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.412541 3901 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:30.414559 3901 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 22:58:30.414569 3901 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 22:58:30.414575 3901 factory.go:636] Watch Factory start up complete, took: 149.30294ms I0327 22:58:30.415014 3901 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0327 22:58:30.417379 3901 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 22:58:30.423249 3901 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 22:58:30.423261 3901 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 22:58:30.423268 3901 factory.go:636] Watch Factory start up complete, took: 150.463723ms I0327 22:58:30.423275 3901 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-2-195.ec2.internal" zone I0327 22:58:30.454977 3901 default_node_network_controller.go:709] Initializing the default node network controller I0327 22:58:30.504756 3901 udn_isolation.go:101] Starting UDN host isolation manager I0327 22:58:30.510968 3901 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0327 22:58:30.552126 3901 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0327 22:58:30.552208 3901 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0327 22:58:30.552223 3901 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0327 22:58:30.584224 3901 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0327 22:58:30.584288 3901 default_node_network_controller.go:808] Node ip-10-0-2-195.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0327 22:58:30.590198 3901 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.195"] k8s.ovn.org/zone-name:ip-10-0-2-195.ec2.internal] on node ip-10-0-2-195.ec2.internal I0327 22:58:30.610620 3901 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0327 22:58:30.616160 3901 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0327 22:58:30.616317 3901 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0327 22:58:30.616433 3901 gateway_init.go:263] Preparing Gateway I0327 22:58:30.616446 3901 gateway_shared_intf.go:1658] Creating new gateway I0327 22:58:30.673458 3901 controller_manager.go:427] Waiting for node in zone sync took: 250.165618ms I0327 22:58:30.678301 3901 controller_manager.go:608] Switching to transit router for layer2 networks I0327 22:58:30.678315 3901 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-195.ec2.internal I0327 22:58:30.687935 3901 iptables.go:108] Creating table: filter chain: FORWARD I0327 22:58:30.692702 3901 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 22:58:30.694146 3901 iptables.go:108] Creating table: filter chain: OUTPUT I0327 22:58:30.698169 3901 controller_manager.go:288] SCTP support detected in OVN I0327 22:58:30.699302 3901 services_controller.go:69] Creating services controller for network=default I0327 22:58:30.699390 3901 udn_enabled_svc.go:80] Waiting for service informer to sync I0327 22:58:30.699417 3901 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0327 22:58:30.699452 3901 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0327 22:58:30.699489 3901 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0327 22:58:30.699503 3901 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 22:58:30.699672 3901 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0327 22:58:30.699726 3901 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0327 22:58:30.699762 3901 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 24.807µs I0327 22:58:30.699765 3901 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 56.681µs I0327 22:58:30.699775 3901 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 48.541µs I0327 22:58:30.699782 3901 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0327 22:58:30.699794 3901 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0327 22:58:30.699804 3901 default_network_controller.go:321] Starting the default network controller I0327 22:58:30.699885 3901 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 15.123µs I0327 22:58:30.700014 3901 udn_enabled_svc.go:95] Performing full resync I0327 22:58:30.700347 3901 udn_enabled_svc.go:99] Waiting for handler to sync I0327 22:58:30.700360 3901 udn_enabled_svc.go:104] Starting worker I0327 22:58:30.700754 3901 gateway_shared_intf.go:1787] Gateway Creation Complete I0327 22:58:30.700769 3901 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-195.ec2.internal","mac-address":"0e:0c:18:90:64:35","ip-addresses":["10.0.2.195/24"],"ip-address":"10.0.2.195/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:7c69d865-5453-49f9-a3f8-c091d01f35d8 k8s.ovn.org/node-encap-ips:["10.0.2.195"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.195/24"} k8s.ovn.org/zone-name:ip-10-0-2-195.ec2.internal] on node ip-10-0-2-195.ec2.internal I0327 22:58:30.708630 3901 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0327 22:58:30.708690 3901 repair.go:33] Syncing exgw routes took 45.861µs I0327 22:58:30.708708 3901 default_network_controller.go:407] Starting all the Watchers... I0327 22:58:30.708841 3901 namespace.go:98] [openshift-insights] adding namespace I0327 22:58:30.708844 3901 namespace.go:98] [openshift-kube-scheduler] adding namespace I0327 22:58:30.708851 3901 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0327 22:58:30.708849 3901 namespace.go:98] [openshift-config] adding namespace I0327 22:58:30.708858 3901 namespace.go:98] [openshift-node] adding namespace I0327 22:58:30.708859 3901 namespace.go:98] [openshift-marketplace] adding namespace I0327 22:58:30.708868 3901 namespace.go:98] [openshift-customer-monitoring] adding namespace I0327 22:58:30.708872 3901 namespace.go:98] [openshift-config-managed] adding namespace I0327 22:58:30.708880 3901 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0327 22:58:30.708882 3901 namespace.go:98] [openshift-console-operator] adding namespace I0327 22:58:30.708882 3901 namespace.go:98] [openshift-package-operator] adding namespace I0327 22:58:30.708884 3901 namespace.go:98] [openshift-apiserver-operator] adding namespace I0327 22:58:30.708900 3901 namespace.go:98] [openshift-service-ca-operator] adding namespace I0327 22:58:30.708910 3901 namespace.go:98] [openshift-kube-apiserver] adding namespace I0327 22:58:30.708902 3901 namespace.go:98] [openshift-route-controller-manager] adding namespace I0327 22:58:30.709499 3901 namespace.go:102] [openshift-insights] adding namespace took 646.305µs I0327 22:58:30.709515 3901 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0327 22:58:30.710039 3901 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 519.427µs I0327 22:58:30.710052 3901 namespace.go:98] [openshift-backplane-cee] adding namespace I0327 22:58:30.710648 3901 namespace.go:102] [openshift-backplane-cee] adding namespace took 591.069µs I0327 22:58:30.710659 3901 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0327 22:58:30.711219 3901 namespace.go:102] [openshift-kube-scheduler] adding namespace took 2.361939ms I0327 22:58:30.711236 3901 namespace.go:98] [openshift-monitoring] adding namespace I0327 22:58:30.711784 3901 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 2.923738ms I0327 22:58:30.711799 3901 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0327 22:58:30.712345 3901 namespace.go:102] [openshift-config] adding namespace took 3.483033ms I0327 22:58:30.712353 3901 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0327 22:58:30.712878 3901 namespace.go:102] [openshift-node] adding namespace took 4.010524ms I0327 22:58:30.712893 3901 namespace.go:98] [openshift-must-gather-operator] adding namespace I0327 22:58:30.713311 3901 namespace.go:102] [openshift-marketplace] adding namespace took 4.442626ms I0327 22:58:30.713322 3901 namespace.go:98] [openshift-backplane-mobb] adding namespace I0327 22:58:30.713914 3901 namespace.go:102] [openshift-customer-monitoring] adding namespace took 5.03833ms I0327 22:58:30.713936 3901 namespace.go:98] [openshift-logging] adding namespace I0327 22:58:30.714433 3901 namespace.go:102] [openshift-config-managed] adding namespace took 5.545353ms I0327 22:58:30.714447 3901 namespace.go:98] [openshift-ingress] adding namespace I0327 22:58:30.714962 3901 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.077061ms I0327 22:58:30.714981 3901 namespace.go:98] [openshift-backplane-csm] adding namespace I0327 22:58:30.715414 3901 namespace.go:102] [openshift-console-operator] adding namespace took 6.528129ms I0327 22:58:30.715426 3901 namespace.go:98] [openshift-machine-config-operator] adding namespace I0327 22:58:30.715899 3901 namespace.go:102] [openshift-package-operator] adding namespace took 7.008214ms I0327 22:58:30.715920 3901 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0327 22:58:30.716495 3901 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.600637ms I0327 22:58:30.716511 3901 namespace.go:98] [kube-public] adding namespace I0327 22:58:30.717108 3901 namespace.go:102] [openshift-service-ca-operator] adding namespace took 8.191178ms I0327 22:58:30.717126 3901 namespace.go:98] [openshift-backplane] adding namespace I0327 22:58:30.717716 3901 namespace.go:102] [openshift-kube-apiserver] adding namespace took 8.796651ms I0327 22:58:30.717732 3901 namespace.go:98] [openshift-machine-api] adding namespace I0327 22:58:30.718346 3901 namespace.go:102] [openshift-route-controller-manager] adding namespace took 9.421865ms I0327 22:58:30.718359 3901 namespace.go:98] [openshift-infra] adding namespace I0327 22:58:30.718665 3901 default_node_network_controller.go:518] Node connection status = connected I0327 22:58:30.718893 3901 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 8.22875ms I0327 22:58:30.718905 3901 namespace.go:98] [openshift-ingress-canary] adding namespace I0327 22:58:30.719384 3901 namespace.go:102] [openshift-monitoring] adding namespace took 8.142918ms I0327 22:58:30.719398 3901 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0327 22:58:30.719847 3901 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 8.040291ms I0327 22:58:30.719865 3901 namespace.go:98] [openshift-authentication-operator] adding namespace I0327 22:58:30.720423 3901 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 8.063821ms I0327 22:58:30.720439 3901 namespace.go:98] [openshift-backplane-srep] adding namespace I0327 22:58:30.721025 3901 namespace.go:102] [openshift-must-gather-operator] adding namespace took 8.124705ms I0327 22:58:30.721044 3901 namespace.go:98] [openshift-network-console] adding namespace I0327 22:58:30.721597 3901 namespace.go:102] [openshift-backplane-mobb] adding namespace took 8.268752ms I0327 22:58:30.721614 3901 namespace.go:98] [openshift-config-operator] adding namespace I0327 22:58:30.722117 3901 namespace.go:102] [openshift-logging] adding namespace took 8.173915ms I0327 22:58:30.722134 3901 namespace.go:98] [openshift-host-network] adding namespace I0327 22:58:30.722742 3901 namespace.go:102] [openshift-ingress] adding namespace took 8.288446ms I0327 22:58:30.722764 3901 namespace.go:98] [openshift-apiserver] adding namespace I0327 22:58:30.723295 3901 namespace.go:102] [openshift-backplane-csm] adding namespace took 8.306458ms I0327 22:58:30.723313 3901 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0327 22:58:30.723916 3901 namespace.go:102] [openshift-machine-config-operator] adding namespace took 8.481752ms I0327 22:58:30.723939 3901 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0327 22:58:30.724493 3901 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 8.566378ms I0327 22:58:30.724508 3901 namespace.go:98] [openshift-authentication] adding namespace I0327 22:58:30.725011 3901 namespace.go:102] [kube-public] adding namespace took 8.494721ms I0327 22:58:30.725025 3901 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0327 22:58:30.725464 3901 namespace.go:102] [openshift-backplane] adding namespace took 8.329329ms I0327 22:58:30.725496 3901 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0327 22:58:30.725972 3901 namespace.go:102] [openshift-machine-api] adding namespace took 8.233579ms I0327 22:58:30.725985 3901 namespace.go:98] [default] adding namespace I0327 22:58:30.726429 3901 namespace.go:102] [openshift-infra] adding namespace took 8.06505ms I0327 22:58:30.726443 3901 namespace.go:98] [openshift-network-diagnostics] adding namespace I0327 22:58:30.726883 3901 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.973308ms I0327 22:58:30.727323 3901 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.921518ms I0327 22:58:30.727334 3901 namespace.go:98] [openshift-cluster-version] adding namespace I0327 22:58:30.727972 3901 namespace.go:102] [openshift-authentication-operator] adding namespace took 8.097939ms I0327 22:58:30.727994 3901 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0327 22:58:30.728880 3901 namespace.go:102] [openshift-backplane-srep] adding namespace took 8.434722ms I0327 22:58:30.728897 3901 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0327 22:58:30.729330 3901 namespace.go:102] [openshift-network-console] adding namespace took 8.278944ms I0327 22:58:30.729344 3901 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0327 22:58:30.729826 3901 namespace.go:102] [openshift-config-operator] adding namespace took 8.206085ms I0327 22:58:30.729841 3901 namespace.go:98] [open-cluster-management-2pabjk1o0k4lirtjul81bm4bi3nfk4gj] adding namespace I0327 22:58:30.730381 3901 namespace.go:102] [openshift-host-network] adding namespace took 8.242448ms I0327 22:58:30.730393 3901 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0327 22:58:30.730884 3901 namespace.go:102] [openshift-apiserver] adding namespace took 8.114492ms I0327 22:58:30.730893 3901 namespace.go:98] [openshift-console-user-settings] adding namespace I0327 22:58:30.731370 3901 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 8.050407ms I0327 22:58:30.731388 3901 namespace.go:98] [openshift-network-node-identity] adding namespace I0327 22:58:30.731827 3901 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.881824ms I0327 22:58:30.731843 3901 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0327 22:58:30.732297 3901 namespace.go:102] [openshift-authentication] adding namespace took 7.783949ms I0327 22:58:30.732307 3901 namespace.go:98] [openshift-network-operator] adding namespace I0327 22:58:30.732758 3901 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.725266ms I0327 22:58:30.732779 3901 namespace.go:98] [openshift-multus] adding namespace I0327 22:58:30.733227 3901 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.725814ms I0327 22:58:30.733241 3901 namespace.go:98] [dedicated-admin] adding namespace W0327 22:58:30.733756 3901 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-kwvhq: pod openshift-network-diagnostics/network-check-target-kwvhq: no pod IPs found W0327 22:58:30.733802 3901 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-qgvsg: pod openshift-network-diagnostics/network-check-target-qgvsg: no pod IPs found I0327 22:58:30.733804 3901 namespace.go:102] [default] adding namespace took 7.814286ms I0327 22:58:30.733815 3901 namespace.go:98] [openshift-ingress-operator] adding namespace I0327 22:58:30.734350 3901 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.90053ms I0327 22:58:30.734365 3901 namespace.go:98] [openshift-operators-redhat] adding namespace I0327 22:58:30.734819 3901 namespace.go:102] [openshift-cluster-version] adding namespace took 7.479549ms I0327 22:58:30.734834 3901 namespace.go:98] [openshift] adding namespace I0327 22:58:30.735288 3901 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.28797ms I0327 22:58:30.735300 3901 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0327 22:58:30.735745 3901 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.843084ms I0327 22:58:30.735760 3901 namespace.go:98] [kube-system] adding namespace I0327 22:58:30.736244 3901 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.893505ms I0327 22:58:30.736257 3901 namespace.go:98] [openshift-image-registry] adding namespace I0327 22:58:30.736725 3901 namespace.go:102] [open-cluster-management-2pabjk1o0k4lirtjul81bm4bi3nfk4gj] adding namespace took 6.879385ms I0327 22:58:30.736741 3901 namespace.go:98] [openshift-console] adding namespace I0327 22:58:30.737180 3901 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.782542ms I0327 22:58:30.737192 3901 namespace.go:98] [openshift-backplane-tam] adding namespace I0327 22:58:30.737613 3901 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.715794ms I0327 22:58:30.737628 3901 namespace.go:98] [openshift-backplane-cse] adding namespace I0327 22:58:30.738100 3901 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.706048ms I0327 22:58:30.738113 3901 namespace.go:98] [kube-node-lease] adding namespace I0327 22:58:30.738616 3901 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.768381ms I0327 22:58:30.738627 3901 namespace.go:98] [openshift-kube-controller-manager] adding namespace W0327 22:58:30.739042 3901 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-n7l7j: pod openshift-multus/network-metrics-daemon-n7l7j: no pod IPs found I0327 22:58:30.739052 3901 namespace.go:102] [openshift-network-operator] adding namespace took 6.738464ms I0327 22:58:30.739058 3901 namespace.go:98] [open-cluster-management-agent-addon] adding namespace W0327 22:58:30.739059 3901 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-996pn: pod openshift-multus/network-metrics-daemon-996pn: no pod IPs found I0327 22:58:30.739575 3901 namespace.go:102] [openshift-multus] adding namespace took 6.790049ms I0327 22:58:30.739590 3901 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0327 22:58:30.740009 3901 namespace.go:102] [dedicated-admin] adding namespace took 6.761724ms I0327 22:58:30.740025 3901 namespace.go:98] [openshift-dns-operator] adding namespace I0327 22:58:30.740416 3901 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.595486ms I0327 22:58:30.740429 3901 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0327 22:58:30.740877 3901 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.505495ms I0327 22:58:30.740891 3901 namespace.go:98] [openshift-operators] adding namespace I0327 22:58:30.741308 3901 namespace.go:102] [openshift] adding namespace took 6.468051ms I0327 22:58:30.741323 3901 namespace.go:98] [openshift-cluster-samples-operator] adding namespace W0327 22:58:30.741718 3901 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-5kf2n: pod kube-system/global-pull-secret-syncer-5kf2n: no pod IPs found I0327 22:58:30.741766 3901 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.461415ms I0327 22:58:30.742238 3901 namespace.go:102] [kube-system] adding namespace took 6.471678ms I0327 22:58:30.742249 3901 namespace.go:98] [openshift-etcd] adding namespace I0327 22:58:30.742669 3901 namespace.go:102] [openshift-image-registry] adding namespace took 6.408814ms I0327 22:58:30.742679 3901 namespace.go:98] [openshift-dns] adding namespace I0327 22:58:30.743117 3901 namespace.go:102] [openshift-console] adding namespace took 6.36941ms I0327 22:58:30.743554 3901 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.355761ms I0327 22:58:30.744011 3901 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.377336ms I0327 22:58:30.744419 3901 namespace.go:102] [kube-node-lease] adding namespace took 6.300841ms I0327 22:58:30.744432 3901 namespace.go:98] [openshift-controller-manager] adding namespace I0327 22:58:30.744878 3901 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.244611ms I0327 22:58:30.745302 3901 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.2389ms I0327 22:58:30.745742 3901 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.146952ms I0327 22:58:30.746185 3901 namespace.go:102] [openshift-dns-operator] adding namespace took 6.154076ms I0327 22:58:30.746716 3901 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.281159ms I0327 22:58:30.747333 3901 namespace.go:102] [openshift-operators] adding namespace took 6.434994ms I0327 22:58:30.747899 3901 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.569083ms I0327 22:58:30.748422 3901 namespace.go:102] [openshift-etcd] adding namespace took 6.16447ms I0327 22:58:30.748882 3901 namespace.go:102] [openshift-dns] adding namespace took 6.196761ms I0327 22:58:30.749396 3901 namespace.go:102] [openshift-controller-manager] adding namespace took 4.956667ms I0327 22:58:30.749824 3901 zone_ic_handler.go:186] Time taken to create transit switch: 295.896µs I0327 22:58:30.749925 3901 master.go:532] Adding or Updating local node "ip-10-0-2-195.ec2.internal" for network "default" I0327 22:58:30.749929 3901 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-48.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-48.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-48.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-48.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-48.ec2.internal I0327 22:58:30.750367 3901 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-88.ec2.internal for the network default I0327 22:58:30.751317 3901 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-88.ec2.internal" on network "default" took: 960.657µs W0327 22:58:30.758570 3901 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-5kf2n: pod kube-system/global-pull-secret-syncer-5kf2n: no pod IPs found W0327 22:58:30.758590 3901 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-996pn: pod openshift-multus/network-metrics-daemon-996pn: no pod IPs found W0327 22:58:30.758602 3901 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-kwvhq: pod openshift-network-diagnostics/network-check-target-kwvhq: no pod IPs found I0327 22:58:30.759509 3901 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-2-195.ec2.internal for the network default I0327 22:58:30.760235 3901 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0327 22:58:30.760261 3901 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0327 22:58:30.760277 3901 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0327 22:58:30.760279 3901 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-48.ec2.internal in network "default" I0327 22:58:30.760308 3901 node_tracker.go:235] Node ip-10-0-0-48.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-48.ec2.internal" I0327 22:58:30.760321 3901 node_tracker.go:169] Node ip-10-0-0-48.ec2.internal switch + router changed, syncing services in network "default" I0327 22:58:30.760328 3901 services_controller.go:574] Full service sync requested for network=default I0327 22:58:30.760335 3901 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-88.ec2.internal in network "default" I0327 22:58:30.760365 3901 node_tracker.go:169] Node ip-10-0-1-88.ec2.internal switch + router changed, syncing services in network "default" I0327 22:58:30.760371 3901 services_controller.go:574] Full service sync requested for network=default I0327 22:58:30.760378 3901 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-195.ec2.internal in network "default" I0327 22:58:30.760398 3901 node_tracker.go:169] Node ip-10-0-2-195.ec2.internal switch + router changed, syncing services in network "default" I0327 22:58:30.760405 3901 services_controller.go:574] Full service sync requested for network=default I0327 22:58:30.861348 3901 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0327 22:58:30.861368 3901 services_controller.go:208] Setting up event handlers for services for network=default I0327 22:58:30.861420 3901 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0327 22:58:30.861480 3901 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0327 22:58:30.861495 3901 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0327 22:58:30.962418 3901 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0327 22:58:30.962439 3901 repair.go:60] Starting repairing loop for services I0327 22:58:30.962543 3901 repair.go:131] Deleted 0 stale service LBs I0327 22:58:30.962567 3901 repair.go:137] Deleted 0 stale Chassis Template Vars I0327 22:58:30.962583 3901 repair.go:62] Finished repairing loop for services: 146.682µs I0327 22:58:30.962608 3901 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0327 22:58:30.962618 3901 services_controller.go:254] Starting workers for network=default I0327 22:58:30.962960 3901 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-5kf2n] creating logical port kube-system_global-pull-secret-syncer-5kf2n for pod on switch ip-10-0-2-195.ec2.internal I0327 22:58:30.962967 3901 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-996pn] creating logical port openshift-multus_network-metrics-daemon-996pn for pod on switch ip-10-0-2-195.ec2.internal I0327 22:58:30.962981 3901 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-kwvhq] creating logical port openshift-network-diagnostics_network-check-target-kwvhq for pod on switch ip-10-0-2-195.ec2.internal I0327 22:58:30.963092 3901 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-5kf2n I0327 22:58:30.963105 3901 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-kwvhq I0327 22:58:30.963130 3901 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-qgvsg, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-qgvsg: suppressed error logged: pod openshift-network-diagnostics/network-check-target-qgvsg: no pod IPs found I0327 22:58:30.963129 3901 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-n7l7j, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-n7l7j: suppressed error logged: pod openshift-multus/network-metrics-daemon-n7l7j: no pod IPs found I0327 22:58:30.963273 3901 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-996pn I0327 22:58:30.982122 3901 pod.go:62] [kube-system/global-pull-secret-syncer-5kf2n] pod update took 19.088397ms I0327 22:58:30.982138 3901 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-5kf2n] addLogicalPort annotation time took 19.107095ms I0327 22:58:30.982984 3901 pods.go:271] [kube-system/global-pull-secret-syncer-5kf2n] addLogicalPort took 20.036514ms, libovsdb time 643.106µs I0327 22:58:30.985662 3901 pod.go:62] [openshift-network-diagnostics/network-check-target-kwvhq] pod update took 22.616146ms I0327 22:58:30.985678 3901 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-kwvhq] addLogicalPort annotation time took 22.635942ms I0327 22:58:30.986041 3901 pod.go:62] [openshift-multus/network-metrics-daemon-996pn] pod update took 22.996332ms I0327 22:58:30.986058 3901 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-996pn] addLogicalPort annotation time took 23.0158ms I0327 22:58:30.986497 3901 pods.go:271] [openshift-network-diagnostics/network-check-target-kwvhq] addLogicalPort took 23.525583ms, libovsdb time 619.42µs I0327 22:58:30.986818 3901 pods.go:271] [openshift-multus/network-metrics-daemon-996pn] addLogicalPort took 23.872348ms, libovsdb time 558.833µs I0327 22:58:30.987219 3901 admin_network_policy_controller.go:227] Starting controller default-network-controller I0327 22:58:30.987247 3901 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0327 22:58:30.987265 3901 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0327 22:58:30.987275 3901 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0327 22:58:30.987461 3901 repair.go:29] Repairing admin network policies took 179.552µs I0327 22:58:30.987592 3901 repair.go:92] Repairing baseline admin network policies took 107.187µs I0327 22:58:30.987919 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0327 22:58:30.987942 3901 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0327 22:58:30.989461 3901 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0327 22:58:30.989488 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0327 22:58:30.989497 3901 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0327 22:58:30.990941 3901 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0327 22:58:30.990960 3901 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0327 22:58:30.990968 3901 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0327 22:58:30.993023 3901 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0327 22:58:30.993044 3901 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0327 22:58:30.993070 3901 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0327 22:58:30.993629 3901 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0327 22:58:30.993649 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0327 22:58:30.993656 3901 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0327 22:58:30.993973 3901 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0327 22:58:30.993995 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0327 22:58:30.994008 3901 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0327 22:58:30.994533 3901 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0327 22:58:30.994553 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0327 22:58:30.994564 3901 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0327 22:58:30.996106 3901 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0327 22:58:30.996123 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0327 22:58:30.996129 3901 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0327 22:58:30.996407 3901 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0327 22:58:30.996426 3901 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0327 22:58:30.996438 3901 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0327 22:58:30.997749 3901 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0327 22:58:30.997767 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0327 22:58:30.997782 3901 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0327 22:58:30.998227 3901 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0327 22:58:30.998244 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0327 22:58:30.998697 3901 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0327 22:58:30.998711 3901 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 22:58:30.999402 3901 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0327 22:58:30.999420 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0327 22:58:30.999428 3901 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0327 22:58:31.000827 3901 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0327 22:58:31.000844 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0327 22:58:31.000868 3901 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 22:58:31.001600 3901 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0327 22:58:31.001620 3901 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0327 22:58:31.001629 3901 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0327 22:58:31.002557 3901 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0327 22:58:31.002581 3901 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0327 22:58:31.002603 3901 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0327 22:58:31.003688 3901 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0327 22:58:31.003708 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0327 22:58:31.003720 3901 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0327 22:58:31.004420 3901 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0327 22:58:31.004437 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0327 22:58:31.004444 3901 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0327 22:58:31.005012 3901 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0327 22:58:31.005034 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0327 22:58:31.005044 3901 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0327 22:58:31.005528 3901 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0327 22:58:31.005543 3901 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0327 22:58:31.005555 3901 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0327 22:58:31.007839 3901 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0327 22:58:31.007857 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0327 22:58:31.007864 3901 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0327 22:58:31.008317 3901 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0327 22:58:31.008334 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0327 22:58:31.008343 3901 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0327 22:58:31.008773 3901 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0327 22:58:31.008793 3901 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0327 22:58:31.008804 3901 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0327 22:58:31.009238 3901 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0327 22:58:31.009253 3901 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0327 22:58:31.009262 3901 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0327 22:58:31.010398 3901 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0327 22:58:31.010415 3901 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0327 22:58:31.010427 3901 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0327 22:58:31.011535 3901 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0327 22:58:31.011553 3901 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0327 22:58:31.011560 3901 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0327 22:58:31.012021 3901 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0327 22:58:31.012038 3901 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0327 22:58:31.012044 3901 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0327 22:58:31.012301 3901 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0327 22:58:31.012324 3901 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0327 22:58:31.012334 3901 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0327 22:58:31.013137 3901 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0327 22:58:31.013157 3901 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0327 22:58:31.013166 3901 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0327 22:58:31.014117 3901 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0327 22:58:31.014135 3901 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0327 22:58:31.014146 3901 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0327 22:58:31.015647 3901 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0327 22:58:31.021088 3901 egressfirewall.go:319] Starting EgressFirewall controller I0327 22:58:31.021118 3901 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0327 22:58:31.021140 3901 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0327 22:58:31.021156 3901 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0327 22:58:31.021173 3901 controller.go:133] Adding controller egress-firewall-controller event handlers I0327 22:58:31.021214 3901 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0327 22:58:31.021229 3901 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0327 22:58:31.021485 3901 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0327 22:58:31.021503 3901 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0327 22:58:31.021515 3901 egressqos.go:193] Setting up event handlers for EgressQoS I0327 22:58:31.021590 3901 egressfirewall.go:628] Syncing node "ip-10-0-0-48.ec2.internal" for egress firewall I0327 22:58:31.021604 3901 egressfirewall.go:628] Syncing node "ip-10-0-1-88.ec2.internal" for egress firewall I0327 22:58:31.021610 3901 egressfirewall.go:628] Syncing node "ip-10-0-2-195.ec2.internal" for egress firewall I0327 22:58:31.021624 3901 egressqos.go:245] Starting EgressQoS Controller I0327 22:58:31.021639 3901 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0327 22:58:31.021650 3901 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0327 22:58:31.021660 3901 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0327 22:58:31.021669 3901 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0327 22:58:31.021678 3901 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0327 22:58:31.021685 3901 shared_informer.go:356] "Caches are synced" controller="egressqos" I0327 22:58:31.021694 3901 egressqos.go:259] Repairing EgressQoSes I0327 22:58:31.021701 3901 egressqos.go:399] Starting repairing loop for egressqos I0327 22:58:31.021779 3901 egressqos.go:401] Finished repairing loop for egressqos: 72.527µs I0327 22:58:31.021801 3901 egressservice_zone.go:132] Setting up event handlers for Egress Services I0327 22:58:31.021829 3901 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-195.ec2.internal I0327 22:58:31.021871 3901 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-195.ec2.internal : 43.359µs I0327 22:58:31.021962 3901 egressservice_zone.go:210] Starting Egress Services Controller I0327 22:58:31.021975 3901 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 22:58:31.021986 3901 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 22:58:31.021991 3901 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 22:58:31.021995 3901 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 22:58:31.022000 3901 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 22:58:31.022004 3901 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 22:58:31.022009 3901 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0327 22:58:31.022012 3901 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0327 22:58:31.022016 3901 egressservice_zone.go:228] Repairing Egress Services I0327 22:58:31.022289 3901 master_controller.go:88] Starting Admin Policy Based Route Controller I0327 22:58:31.022298 3901 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 22:58:31.022317 3901 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-88.ec2.internal I0327 22:58:31.022329 3901 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-88.ec2.internal: 14.58µs I0327 22:58:31.022337 3901 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-195.ec2.internal I0327 22:58:31.022345 3901 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-195.ec2.internal: 6.013µs I0327 22:58:31.022352 3901 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-48.ec2.internal I0327 22:58:31.022355 3901 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-48.ec2.internal: 4.768µs I0327 22:58:31.022423 3901 default_network_controller.go:568] Completing all the Watchers took 313.703178ms I0327 22:58:31.022437 3901 default_network_controller.go:572] Starting unidling controllers I0327 22:58:31.022449 3901 unidle.go:46] Registering OVN SB ControllerEvent handler I0327 22:58:31.022456 3901 unidle.go:63] Populating Initial ContollerEvent events I0327 22:58:31.022481 3901 unidle.go:79] Setting up event handlers for services I0327 22:58:31.022551 3901 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0327 22:58:31.219521 3901 default_node_network_controller.go:518] Node connection status = connected I0327 22:58:31.719832 3901 default_node_network_controller.go:518] Node connection status = connected I0327 22:58:32.220203 3901 default_node_network_controller.go:518] Node connection status = connected I0327 22:58:32.553796 3901 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0327 22:58:32.719898 3901 default_node_network_controller.go:518] Node connection status = connected I0327 22:58:32.736296 3901 bridgeconfig.go:537] Gateway is ready I0327 22:58:32.736320 3901 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0327 22:58:32.747702 3901 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.195. Updating OVN encap IP. I0327 22:58:32.752501 3901 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.195 - it is already configured I0327 22:58:32.752518 3901 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.195/24:{} 172.20.0.1/32:{}]. Updating annotations. I0327 22:58:32.752962 3901 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.195/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-195.ec2.internal","mac-address":"0e:0c:18:90:64:35","ip-addresses":["10.0.2.195/24"],"ip-address":"10.0.2.195/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:7c69d865-5453-49f9-a3f8-c091d01f35d8 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.195/24"}] on node ip-10-0-2-195.ec2.internal I0327 22:58:32.768072 3901 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.2.195 I0327 22:58:32.768149 3901 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-195.ec2.internal in network "default" I0327 22:58:32.768077 3901 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-195.ec2.internal I0327 22:58:32.768260 3901 node_tracker.go:169] Node ip-10-0-2-195.ec2.internal switch + router changed, syncing services in network "default" I0327 22:58:32.768266 3901 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-195.ec2.internal: 190.159µs I0327 22:58:32.768230 3901 master.go:532] Adding or Updating local node "ip-10-0-2-195.ec2.internal" for network "default" I0327 22:58:32.768288 3901 services_controller.go:574] Full service sync requested for network=default I0327 22:58:32.768154 3901 egressfirewall.go:628] Syncing node "ip-10-0-2-195.ec2.internal" for egress firewall I0327 22:58:32.768239 3901 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-195.ec2.internal I0327 22:58:32.773670 3901 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-195.ec2.internal. Updating no re-route policies I0327 22:58:32.773750 3901 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-195.ec2.internal I0327 22:58:32.773758 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-195.ec2.internal I0327 22:58:32.773767 3901 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-195.ec2.internal I0327 22:58:32.773767 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-8cxnm I0327 22:58:32.773788 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-195.ec2.internal I0327 22:58:32.773791 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-8cxnm I0327 22:58:32.773795 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-k96st I0327 22:58:32.773801 3901 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-195.ec2.internal after 0 failed attempt(s) I0327 22:58:32.773802 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-7n5h6 I0327 22:58:32.773809 3901 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-8cxnm after 0 failed attempt(s) I0327 22:58:32.773811 3901 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-195.ec2.internal after 0 failed attempt(s) I0327 22:58:32.773791 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-n7l7j I0327 22:58:32.773818 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-7n5h6 I0327 22:58:32.773778 3901 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-4472c I0327 22:58:32.773838 3901 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-4472c I0327 22:58:32.773858 3901 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-7n5h6 after 0 failed attempt(s) I0327 22:58:32.773867 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-zvm99 I0327 22:58:32.773801 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-996pn I0327 22:58:32.773877 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-zvm99 I0327 22:58:32.773880 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-996pn I0327 22:58:32.773807 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-k96st I0327 22:58:32.773896 3901 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-zvm99 after 0 failed attempt(s) I0327 22:58:32.773858 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5lwjh I0327 22:58:32.773901 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-9zppl I0327 22:58:32.773914 3901 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-996pn] creating logical port openshift-multus_network-metrics-daemon-996pn for pod on switch ip-10-0-2-195.ec2.internal I0327 22:58:32.773919 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-9zppl I0327 22:58:32.773926 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-69vpr I0327 22:58:32.773941 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-69vpr I0327 22:58:32.773906 3901 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-k96st after 0 failed attempt(s) I0327 22:58:32.773862 3901 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-4472c after 0 failed attempt(s) I0327 22:58:32.773909 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5lwjh I0327 22:58:32.773966 3901 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-69vpr after 0 failed attempt(s) I0327 22:58:32.773974 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-qgvsg I0327 22:58:32.773942 3901 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-9zppl after 0 failed attempt(s) I0327 22:58:32.773970 3901 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5lwjh after 0 failed attempt(s) I0327 22:58:32.773986 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-qgvsg I0327 22:58:32.773823 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-n7l7j I0327 22:58:32.774032 3901 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-qgvsg, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-qgvsg: suppressed error logged: pod openshift-network-diagnostics/network-check-target-qgvsg: no pod IPs found I0327 22:58:32.774041 3901 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-n7l7j, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-n7l7j: suppressed error logged: pod openshift-multus/network-metrics-daemon-n7l7j: no pod IPs found I0327 22:58:32.774051 3901 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-n9kkh I0327 22:58:32.774060 3901 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-n9kkh I0327 22:58:32.774076 3901 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-n9kkh after 0 failed attempt(s) I0327 22:58:32.774080 3901 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-5kf2n I0327 22:58:32.774091 3901 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-5kf2n I0327 22:58:32.774094 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-7hx4d I0327 22:58:32.774101 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-7hx4d I0327 22:58:32.774107 3901 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-kwvhq I0327 22:58:32.774117 3901 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-7hx4d after 0 failed attempt(s) I0327 22:58:32.774118 3901 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-kwvhq I0327 22:58:32.774125 3901 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-5kf2n] creating logical port kube-system_global-pull-secret-syncer-5kf2n for pod on switch ip-10-0-2-195.ec2.internal I0327 22:58:32.774188 3901 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-kwvhq] creating logical port openshift-network-diagnostics_network-check-target-kwvhq for pod on switch ip-10-0-2-195.ec2.internal I0327 22:58:32.774856 3901 pods.go:271] [kube-system/global-pull-secret-syncer-5kf2n] addLogicalPort took 738.788µs, libovsdb time 265.238µs I0327 22:58:32.774873 3901 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-5kf2n after 0 failed attempt(s) I0327 22:58:32.774960 3901 pods.go:271] [openshift-multus/network-metrics-daemon-996pn] addLogicalPort took 1.05678ms, libovsdb time 377.587µs I0327 22:58:32.774971 3901 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-996pn after 0 failed attempt(s) I0327 22:58:32.775083 3901 pods.go:271] [openshift-network-diagnostics/network-check-target-kwvhq] addLogicalPort took 938.336µs, libovsdb time 347.651µs I0327 22:58:32.775107 3901 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-kwvhq after 0 failed attempt(s) I0327 22:58:32.782490 3901 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0327 22:58:32.793006 3901 iptables.go:108] Creating table: mangle chain: OUTPUT I0327 22:58:32.796449 3901 iptables.go:108] Creating table: nat chain: OUTPUT I0327 22:58:32.820541 3901 iptables.go:108] Creating table: nat chain: PREROUTING I0327 22:58:32.867774 3901 iptables.go:108] Creating table: filter chain: FORWARD I0327 22:58:32.885926 3901 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0327 22:58:32.886065 3901 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0327 22:58:32.886114 3901 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0327 22:58:32.886205 3901 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0327 22:58:32.886253 3901 gateway_init.go:325] Gateway and management port readiness took 2.170242522s I0327 22:58:32.886282 3901 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0327 22:58:32.886317 3901 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0327 22:58:32.886328 3901 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0327 22:58:32.886336 3901 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 22:58:32.886391 3901 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0327 22:58:32.886486 3901 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 62.412µs I0327 22:58:32.886707 3901 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0327 22:58:32.886731 3901 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 14.478µs I0327 22:58:32.886739 3901 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 33.148µs I0327 22:58:32.886744 3901 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0327 22:58:32.886754 3901 nad_controller.go:173] [node-nad-controller NAD controller]: started I0327 22:58:32.886761 3901 default_node_network_controller.go:898] Starting the default node network controller I0327 22:58:32.886767 3901 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0327 22:58:32.886846 3901 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 12.402µs I0327 22:58:32.886918 3901 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0327 22:58:32.886930 3901 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0327 22:58:32.886960 3901 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a8b120 0xc0002c7d50 0xc0002c7e30 0xc0002c7ea0 0xc0002c7f10 0xc0002d0070 0xc0002d00e0 0xc000aa5680 0xc0002d01c0 0xc0002d0230 0xc0007262d0 0xc00601afc0 0xc0002d02a0 map[0x2f67040:0xc003482b40 0x2f67580:0xc003482cc0 0x2f68000:0xc003482a80 0x2f682a0:0xc003482ae0 0x2f687e0:0xc003482ba0 0x2f68a80:0xc003482c00 0x2f68d20:0xc003482c60 0x2f69500:0xc003482d20 0x2f697a0:0xc0066000c0 0x2f93d40:0xc000e64f00 0x2f940e0:0xc000e64f60 0x2f94bc0:0xc000bcf080 0x2f96520:0xc001b74000 0x2f97000:0xc000e65140 0x2f97e80:0xc000a11bc0] 0xc000aa6bd0 0} I0327 22:58:32.887020 3901 gateway.go:151] Starting gateway service sync I0327 22:58:32.887438 3901 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0327 22:58:32.889621 3901 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0327 22:58:32.901231 3901 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0327 22:58:32.902365 3901 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0327 22:58:32.903433 3901 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0327 22:58:32.937525 3901 gateway.go:168] Gateway service sync done. Time taken: 50.492648ms I0327 22:58:32.938118 3901 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30195 I0327 22:58:32.938928 3901 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a8b120 0xc0002c7d50 0xc0002c7e30 0xc0002c7ea0 0xc0002c7f10 0xc0002d0070 0xc0002d00e0 0xc000aa5680 0xc0002d01c0 0xc0002d0230 0xc0007262d0 0xc00601afc0 0xc0002d02a0 map[0x2f67040:0xc003482b40 0x2f67580:0xc003482cc0 0x2f68000:0xc003482a80 0x2f682a0:0xc003482ae0 0x2f687e0:0xc003482ba0 0x2f68a80:0xc003482c00 0x2f68d20:0xc003482c60 0x2f69500:0xc003482d20 0x2f697a0:0xc0066000c0 0x2f93d40:0xc000e64f00 0x2f940e0:0xc000e64f60 0x2f94bc0:0xc000bcf080 0x2f96520:0xc001b74000 0x2f97000:0xc000e65140 0x2f97e80:0xc000a11bc0] 0xc000aa6bd0 0} I0327 22:58:32.939626 3901 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a8b120 0xc0002c7d50 0xc0002c7e30 0xc0002c7ea0 0xc0002c7f10 0xc0002d0070 0xc0002d00e0 0xc000aa5680 0xc0002d01c0 0xc0002d0230 0xc0007262d0 0xc00601afc0 0xc0002d02a0 map[0x2f67040:0xc003482b40 0x2f67580:0xc003482cc0 0x2f68000:0xc003482a80 0x2f682a0:0xc003482ae0 0x2f687e0:0xc003482ba0 0x2f68a80:0xc003482c00 0x2f68d20:0xc003482c60 0x2f69500:0xc003482d20 0x2f697a0:0xc0066000c0 0x2f93d40:0xc000e64f00 0x2f940e0:0xc000e64f60 0x2f94bc0:0xc000bcf080 0x2f96520:0xc001b74000 0x2f97000:0xc000e65140 0x2f97e80:0xc000a11bc0] 0xc000aa6bd0 0} I0327 22:58:32.939682 3901 gateway.go:519] Reconciling gateway with updates I0327 22:58:32.945153 3901 gateway.go:326] Spawning Conntrack Rule Check Thread I0327 22:58:32.945184 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 22:58:32.945201 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 22:58:32.945206 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 22:58:32.945211 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 22:58:32.945214 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 22:58:32.945221 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 22:58:32.945233 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 22:58:32.945254 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 22:58:32.945259 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 22:58:32.945269 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 22:58:32.945271 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 22:58:32.945276 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 22:58:32.945279 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 22:58:32.945275 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 22:58:32.945296 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 22:58:32.945291 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 22:58:32.945302 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 22:58:32.945308 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 22:58:32.945309 3901 default_node_network_controller.go:943] Gateway and management port readiness took 58.367147ms I0327 22:58:32.945322 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 22:58:32.945327 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 22:58:32.945269 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 22:58:32.945334 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 22:58:32.945342 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 22:58:32.945340 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 22:58:32.945283 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 22:58:32.945297 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 22:58:32.945352 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 22:58:32.945307 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 22:58:32.945362 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 22:58:32.945363 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 22:58:32.945367 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 22:58:32.945224 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 22:58:32.945374 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 22:58:32.945375 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 22:58:32.945385 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 22:58:32.945392 3901 port_claim.go:71] Svc openshift-ingress/router-default: port 31577 is already open, no action needed I0327 22:58:32.945394 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 22:58:32.945397 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 22:58:32.945400 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 22:58:32.945324 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 22:58:32.945281 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 22:58:32.945410 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 22:58:32.945219 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 22:58:32.945296 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 22:58:32.945419 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 22:58:32.945422 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 22:58:32.945422 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 22:58:32.945427 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 22:58:32.945408 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 22:58:32.945387 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 22:58:32.945446 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 22:58:32.945449 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 22:58:32.945452 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 22:58:32.945355 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 22:58:32.945499 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 22:58:32.945325 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 22:58:32.945512 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 22:58:32.945518 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 22:58:32.945518 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 22:58:32.945525 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 22:58:32.945534 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 22:58:32.945341 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 22:58:32.945569 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 22:58:32.945373 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 22:58:32.945611 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 22:58:32.945501 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 22:58:32.945446 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 22:58:32.945694 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 22:58:32.945226 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 22:58:32.945753 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 22:58:32.945400 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 22:58:32.945820 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 22:58:32.945432 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 22:58:32.945350 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 22:58:32.945879 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 22:58:32.945357 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 22:58:32.945921 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 22:58:32.945926 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 22:58:32.945369 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 22:58:32.945933 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 22:58:32.945402 3901 port_claim.go:71] Svc openshift-ingress/router-default: port 31084 is already open, no action needed I0327 22:58:32.945410 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 22:58:32.945977 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 22:58:32.945983 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 22:58:32.945411 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 22:58:32.945311 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 22:58:32.945344 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 22:58:32.945440 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 22:58:32.945203 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 22:58:32.946109 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 22:58:32.945441 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 22:58:32.945342 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 22:58:32.945454 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 22:58:32.946174 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 22:58:32.945456 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 22:58:32.946208 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 22:58:32.945456 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 22:58:32.946230 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 22:58:32.945462 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 22:58:32.946255 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 22:58:32.945457 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 22:58:32.946262 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 22:58:32.946268 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 22:58:32.945481 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 22:58:32.946274 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 22:58:32.946277 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 22:58:32.945485 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 22:58:32.946283 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 22:58:32.945484 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 22:58:32.946321 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 22:58:32.946829 3901 node_ip_handler_linux.go:160] Node IP manager is running W0327 22:58:32.984604 3901 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 22:58:32.984730 3901 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 22:58:32.984838 3901 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0327 22:58:32.993828 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 22:58:32.993856 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 22:58:32.993871 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 22:58:32.993878 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 22:58:33.034710 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 22:58:33.034734 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 22:58:33.034743 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 22:58:33.034749 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 22:58:33.034769 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 22:58:33.034776 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 22:58:33.034782 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 22:58:33.034789 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 22:58:33.034794 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 22:58:33.078283 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 22:58:33.078291 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 22:58:33.078312 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 22:58:33.078321 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 22:58:33.078345 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 22:58:33.078357 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 22:58:33.078369 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 22:58:33.078375 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 22:58:33.078383 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 22:58:33.078391 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 22:58:33.078397 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 22:58:33.880677 3901 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-88.ec2.internal I0327 22:58:33.880709 3901 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-88.ec2.internal: 40.591µs I0327 22:58:34.397086 3901 default_node_network_controller.go:1364] Starting node controller node sync W0327 22:58:34.397359 3901 egressip_healthcheck.go:76] Health checking using insecure connection I0327 22:58:34.397419 3901 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0327 22:58:34.408796 3901 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.689918ms I0327 22:58:34.408960 3901 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-88.ec2.internal", IP: 10.0.1.88 to PMTUD blocking rules I0327 22:58:34.408969 3901 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-88.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 22:58:34.414242 3901 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0327 22:58:34.414519 3901 egressservice_node.go:93] Setting up event handlers for Egress Services I0327 22:58:34.414618 3901 egressservice_node.go:183] Starting Egress Services Controller I0327 22:58:34.414632 3901 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 22:58:34.414643 3901 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 22:58:34.414649 3901 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 22:58:34.414654 3901 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 22:58:34.414659 3901 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 22:58:34.414664 3901 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 22:58:34.414668 3901 egressservice_node.go:197] Repairing Egress Services I0327 22:58:34.424049 3901 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0327 22:58:34.424061 3901 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 22:58:34.424234 3901 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0327 22:58:34.424246 3901 link_network_manager.go:117] Link manager is running I0327 22:58:34.424254 3901 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0327 22:58:34.424330 3901 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0327 22:58:34.424595 3901 vrf_manager.go:128] VRF manager is running I0327 22:58:34.424675 3901 node_controller_manager.go:437] Removing flows to drop GARP I0327 22:58:34.424683 3901 gateway.go:519] Reconciling gateway with updates I0327 22:58:34.445327 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 22:58:34.445331 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 22:58:34.445343 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 22:58:34.445340 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 22:58:34.445350 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 22:58:34.445353 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 22:58:34.445355 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 22:58:34.445358 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 22:58:34.445356 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 22:58:34.445365 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 22:58:34.445371 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 22:58:34.445370 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 22:58:34.445372 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 22:58:34.445382 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 22:58:34.445390 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 22:58:34.445359 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 22:58:34.445397 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 22:58:34.445396 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 22:58:34.445403 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 22:58:34.445409 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 22:58:34.445343 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 22:58:34.445417 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 22:58:34.445417 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 22:58:34.445419 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 22:58:34.445418 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 22:58:34.445430 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 22:58:34.445430 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 22:58:34.445435 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 22:58:34.445334 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 22:58:34.445443 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 22:58:34.445447 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 22:58:34.445450 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 22:58:34.445453 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 22:58:34.445454 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 22:58:34.445457 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 22:58:34.445463 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 22:58:34.445487 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 22:58:34.445493 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 22:58:34.445494 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 22:58:34.445504 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 22:58:34.445498 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 22:58:34.445345 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 22:58:34.445513 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 22:58:34.445517 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 22:58:34.445518 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 22:58:34.445462 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 22:58:34.445525 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 22:58:34.445530 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 22:58:34.445526 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 22:58:34.445535 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 22:58:34.445538 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 22:58:34.445447 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 22:58:34.445541 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 22:58:34.445541 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 22:58:34.445544 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 22:58:34.445422 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 22:58:34.445551 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 22:58:34.445556 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 22:58:34.445559 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 22:58:34.445562 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 22:58:34.445562 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 22:58:34.445600 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 22:58:34.445608 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 22:58:34.445610 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 22:58:34.445535 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 22:58:34.445616 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 22:58:34.445448 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 22:58:34.445627 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 22:58:34.445630 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 22:58:34.445627 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 22:58:34.445631 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 22:58:34.445634 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 22:58:34.445640 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 22:58:34.445642 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 22:58:34.445537 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 22:58:34.445644 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 22:58:34.445648 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 22:58:34.445651 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 22:58:34.445655 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 22:58:34.445598 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 22:58:34.445494 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 22:58:34.445450 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 22:58:34.445506 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 22:58:34.445519 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 22:58:34.445506 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 22:58:34.445716 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 22:58:34.445721 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 22:58:34.445512 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 22:58:34.445756 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 22:58:34.445764 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 22:58:34.445519 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 22:58:34.445520 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 22:58:34.445781 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 22:58:34.445528 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 22:58:34.445526 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 22:58:34.445830 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 22:58:34.445834 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 22:58:34.445528 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 22:58:34.445841 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 22:58:34.445533 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 22:58:34.445865 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 22:58:34.445881 3901 port_claim.go:71] Svc openshift-ingress/router-default: port 31577 is already open, no action needed I0327 22:58:34.445889 3901 port_claim.go:71] Svc openshift-ingress/router-default: port 31084 is already open, no action needed I0327 22:58:34.445531 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 22:58:34.445920 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 22:58:34.445546 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 22:58:34.445943 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 22:58:34.445549 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 22:58:34.445549 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 22:58:34.445986 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 22:58:34.445551 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 22:58:34.445554 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 22:58:34.445632 3901 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 22:58:34.446039 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 22:58:34.445634 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 22:58:34.445512 3901 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 22:58:34.483558 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 22:58:34.483583 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 22:58:34.483594 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 22:58:34.483602 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 22:58:34.483610 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 22:58:34.483623 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 22:58:34.483636 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 22:58:34.483646 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 22:58:34.483678 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 22:58:34.483692 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 22:58:34.483700 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 22:58:34.522776 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 22:58:34.522782 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 22:58:34.522810 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 22:58:34.522817 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 22:58:34.522825 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 22:58:34.522830 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 22:58:34.522837 3901 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 22:58:35.996786 3901 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-n9kkh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 22:58:35.996829 3901 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-n9kkh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 22:58:36.214998 3901 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-n9kkh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:58:36.215023 3901 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-n9kkh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:58:36.848382 3901 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-195.ec2.internal I0327 22:58:36.848408 3901 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-195.ec2.internal: 33.497µs I0327 22:58:36.874845 3901 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-656cf7696d-mgnhg] creating logical port openshift-image-registry_image-registry-656cf7696d-mgnhg for pod on switch ip-10-0-2-195.ec2.internal I0327 22:58:36.874959 3901 kube.go:256] Updating pod openshift-image-registry/image-registry-656cf7696d-mgnhg I0327 22:58:36.892558 3901 pod.go:62] [openshift-image-registry/image-registry-656cf7696d-mgnhg] pod update took 17.623346ms I0327 22:58:36.892576 3901 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-656cf7696d-mgnhg] addLogicalPort annotation time took 17.647621ms I0327 22:58:36.894129 3901 pods.go:271] [openshift-image-registry/image-registry-656cf7696d-mgnhg] addLogicalPort took 19.29511ms, libovsdb time 1.293742ms I0327 22:58:36.898617 3901 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-mcftb] creating logical port openshift-dns_dns-default-mcftb for pod on switch ip-10-0-2-195.ec2.internal I0327 22:58:36.898680 3901 kube.go:256] Updating pod openshift-dns/dns-default-mcftb I0327 22:58:36.914398 3901 pod.go:62] [openshift-dns/dns-default-mcftb] pod update took 15.734599ms I0327 22:58:36.914418 3901 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-mcftb] addLogicalPort annotation time took 15.757881ms I0327 22:58:36.914505 3901 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-zw8mk] creating logical port openshift-ingress-canary_ingress-canary-zw8mk for pod on switch ip-10-0-2-195.ec2.internal I0327 22:58:36.914615 3901 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-zw8mk I0327 22:58:36.915593 3901 pods.go:271] [openshift-dns/dns-default-mcftb] addLogicalPort took 16.983788ms, libovsdb time 839.945µs I0327 22:58:36.929249 3901 pod.go:62] [openshift-ingress-canary/ingress-canary-zw8mk] pod update took 14.651106ms I0327 22:58:36.929270 3901 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-zw8mk] addLogicalPort annotation time took 14.688556ms I0327 22:58:36.930331 3901 pods.go:271] [openshift-ingress-canary/ingress-canary-zw8mk] addLogicalPort took 15.87138ms, libovsdb time 732.101µs I0327 22:58:36.951369 3901 default_network_controller.go:1008] Node "ip-10-0-0-48.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 22:58:37.343058 3901 cni.go:403] [openshift-ingress-canary/ingress-canary-zw8mk 15aea5afb5a441e58486bbcf396f24133c3f1295220acb94c31d47729d5b1e39 network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-zw8mk 15aea5afb5a441e58486bbcf396f24133c3f1295220acb94c31d47729d5b1e39 network default NAD default] I0327 22:58:37.373364 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-zw8mk, hostIfaceName: 15aea5afb5a441e, network: default, NAD default, SandboxID: "15aea5afb5a441e58486bbcf396f24133c3f1295220acb94c31d47729d5b1e39", PCI device ID: , UID: "13a887ad-dd25-4d43-87cb-3607e3800104", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0327 22:58:37.440966 3901 cni.go:424] [openshift-ingress-canary/ingress-canary-zw8mk 15aea5afb5a441e58486bbcf396f24133c3f1295220acb94c31d47729d5b1e39 network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-zw8mk 15aea5afb5a441e58486bbcf396f24133c3f1295220acb94c31d47729d5b1e39 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:90:6d:5d:5a:6c\",\"name\":\"15aea5afb5a441e\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/06338f76-ce84-485d-8141-421fe2839ddf\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 22:58:39.297009 3901 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0327 22:58:39.316996 3901 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 22:58:43.105196 3901 cni.go:403] [kube-system/global-pull-secret-syncer-5kf2n 8d1491bb4bed3e2492308c5c4d9655e2a0b80db2736af6c71c941aa7c134b38b network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-5kf2n 8d1491bb4bed3e2492308c5c4d9655e2a0b80db2736af6c71c941aa7c134b38b network default NAD default] I0327 22:58:43.113507 3901 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-5kf2n, hostIfaceName: 8d1491bb4bed3e2, network: default, NAD default, SandboxID: "8d1491bb4bed3e2492308c5c4d9655e2a0b80db2736af6c71c941aa7c134b38b", PCI device ID: , UID: "9e8f996a-6555-4f38-a6c9-9d91aad96eba", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0327 22:58:43.153022 3901 cni.go:424] [kube-system/global-pull-secret-syncer-5kf2n 8d1491bb4bed3e2492308c5c4d9655e2a0b80db2736af6c71c941aa7c134b38b network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-5kf2n 8d1491bb4bed3e2492308c5c4d9655e2a0b80db2736af6c71c941aa7c134b38b network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:82:c2:03:4a:72\",\"name\":\"8d1491bb4bed3e2\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/faa2bd76-fd84-4205-b236-f3be4db7a71c\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 22:58:46.661852 3901 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9] creating logical port openshift-deployment-validation-operator_ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 for pod on switch ip-10-0-2-195.ec2.internal I0327 22:58:46.661990 3901 kube.go:256] Updating pod openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 I0327 22:58:46.674721 3901 pod.go:62] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9] pod update took 12.802973ms I0327 22:58:46.674737 3901 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9] addLogicalPort annotation time took 12.824336ms I0327 22:58:46.675994 3901 pods.go:271] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9] addLogicalPort took 14.158321ms, libovsdb time 966.374µs I0327 22:58:47.657921 3901 default_network_controller.go:1008] Node "ip-10-0-0-48.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 22:58:48.152251 3901 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 9cb2ad7da49b4204eb50becff456af6a11462d13173e9f89b746395c21c77ec2 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 9cb2ad7da49b4204eb50becff456af6a11462d13173e9f89b746395c21c77ec2 network default NAD default] I0327 22:58:48.160765 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9, hostIfaceName: 9cb2ad7da49b420, network: default, NAD default, SandboxID: "9cb2ad7da49b4204eb50becff456af6a11462d13173e9f89b746395c21c77ec2", PCI device ID: , UID: "9fd1a76e-4b31-43cb-b6c5-0e8ce77f26cb", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0327 22:58:48.199541 3901 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 9cb2ad7da49b4204eb50becff456af6a11462d13173e9f89b746395c21c77ec2 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 9cb2ad7da49b4204eb50becff456af6a11462d13173e9f89b746395c21c77ec2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:4c:d6:a9:51:47\",\"name\":\"9cb2ad7da49b420\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b5a55988-f1fe-443a-99b8-f8ba1ec05d6f\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 22:58:48.764831 3901 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-48.ec2.internal I0327 22:58:48.764853 3901 egressfirewall.go:628] Syncing node "ip-10-0-0-48.ec2.internal" for egress firewall I0327 22:58:48.764857 3901 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-48.ec2.internal: 32.061µs I0327 22:58:48.764892 3901 default_network_controller.go:1008] Node "ip-10-0-0-48.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 22:58:54.972142 3901 default_network_controller.go:1008] Node "ip-10-0-0-48.ec2.internal" in remote zone "ip-10-0-0-48.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 22:58:54.972153 3901 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-48.ec2.internal in network "default" I0327 22:58:54.972192 3901 node_tracker.go:235] Node ip-10-0-0-48.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-48.ec2.internal" I0327 22:58:54.972206 3901 node_tracker.go:169] Node ip-10-0-0-48.ec2.internal switch + router changed, syncing services in network "default" I0327 22:58:54.972218 3901 services_controller.go:574] Full service sync requested for network=default W0327 22:58:54.972456 3901 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 22:58:55.061536 3901 default_network_controller.go:1008] Node "ip-10-0-0-48.ec2.internal" in remote zone "ip-10-0-0-48.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 22:58:55.086904 3901 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-48.ec2.internal in network "default" I0327 22:58:55.086926 3901 default_network_controller.go:1008] Node "ip-10-0-0-48.ec2.internal" in remote zone "ip-10-0-0-48.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 22:58:55.086951 3901 node_tracker.go:169] Node ip-10-0-0-48.ec2.internal switch + router changed, syncing services in network "default" I0327 22:58:55.086961 3901 services_controller.go:574] Full service sync requested for network=default W0327 22:58:55.087248 3901 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 22:58:55.087582 3901 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-48.ec2.internal for the network default I0327 22:58:55.088754 3901 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-48.ec2.internal" on network "default" took: 1.185475ms I0327 22:58:56.637959 3901 egressfirewall.go:628] Syncing node "ip-10-0-0-48.ec2.internal" for egress firewall I0327 22:58:56.637979 3901 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-48.ec2.internal in network "default" I0327 22:58:56.638029 3901 node_tracker.go:169] Node ip-10-0-0-48.ec2.internal switch + router changed, syncing services in network "default" I0327 22:58:56.638043 3901 services_controller.go:574] Full service sync requested for network=default I0327 22:58:56.637966 3901 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-48.ec2.internal I0327 22:58:56.638062 3901 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-48.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 22:58:56.638068 3901 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-48.ec2.internal: 107.797µs I0327 22:58:56.638074 3901 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-48.ec2.internal", IP: 10.0.0.48 to PMTUD blocking rules W0327 22:58:56.638263 3901 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 22:58:56.644062 3901 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-48.ec2.internal. Updating no re-route policies I0327 22:58:56.653709 3901 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-48.ec2.internal", IP: 10.0.0.48 to PMTUD blocking rules I0327 22:58:56.653723 3901 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-48.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 22:59:00.615376 3901 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-48.ec2.internal I0327 22:59:00.615409 3901 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-48.ec2.internal: 38.249µs I0327 22:59:05.282393 3901 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 9cb2ad7da49b4204eb50becff456af6a11462d13173e9f89b746395c21c77ec2 network default NAD default] DEL starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 9cb2ad7da49b4204eb50becff456af6a11462d13173e9f89b746395c21c77ec2 network default NAD default] I0327 22:59:05.349265 3901 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 9cb2ad7da49b4204eb50becff456af6a11462d13173e9f89b746395c21c77ec2 network default NAD default] DEL finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 9cb2ad7da49b4204eb50becff456af6a11462d13173e9f89b746395c21c77ec2 network default NAD default], result "{}", err I0327 22:59:05.408555 3901 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 22:59:05.408591 3901 pods.go:176] Deleting pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 I0327 22:59:05.409859 3901 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9, ips: 10.129.0.9 I0327 22:59:05.409887 3901 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 22:59:05.409907 3901 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 22:59:05.409917 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0327 22:59:05.410113 3901 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 22:59:05.410128 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0327 22:59:06.433556 3901 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:06.433578 3901 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:06.433593 3901 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:06.433609 3901 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:06.835076 3901 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 22:59:06.846432 3901 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 22:59:09.229695 3901 cni.go:403] [openshift-network-diagnostics/network-check-target-kwvhq e3249b4ee727eb7dfd0a78708ef5ba065503076004afd0b3d838be78fc01b04a network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-kwvhq e3249b4ee727eb7dfd0a78708ef5ba065503076004afd0b3d838be78fc01b04a network default NAD default] I0327 22:59:09.239440 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-kwvhq, hostIfaceName: e3249b4ee727eb7, network: default, NAD default, SandboxID: "e3249b4ee727eb7dfd0a78708ef5ba065503076004afd0b3d838be78fc01b04a", PCI device ID: , UID: "721e02e4-93d4-486c-b296-e3ff45cd1c57", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0327 22:59:09.278442 3901 cni.go:424] [openshift-network-diagnostics/network-check-target-kwvhq e3249b4ee727eb7dfd0a78708ef5ba065503076004afd0b3d838be78fc01b04a network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-kwvhq e3249b4ee727eb7dfd0a78708ef5ba065503076004afd0b3d838be78fc01b04a network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:6a:c9:80:bd:0e\",\"name\":\"e3249b4ee727eb7\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7801d111-e486-4164-ba26-3c8f757b9a81\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 22:59:21.871097 3901 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-85bf9b9667-mm9hl] creating logical port openshift-ingress_router-default-85bf9b9667-mm9hl for pod on switch ip-10-0-2-195.ec2.internal I0327 22:59:21.871188 3901 kube.go:256] Updating pod openshift-ingress/router-default-85bf9b9667-mm9hl I0327 22:59:21.881715 3901 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-6bc69697db-p9rvv] creating logical port openshift-ingress_router-default-6bc69697db-p9rvv for pod on switch ip-10-0-2-195.ec2.internal I0327 22:59:21.881803 3901 kube.go:256] Updating pod openshift-ingress/router-default-6bc69697db-p9rvv I0327 22:59:21.906402 3901 pod.go:62] [openshift-ingress/router-default-85bf9b9667-mm9hl] pod update took 35.244631ms I0327 22:59:21.906421 3901 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-85bf9b9667-mm9hl] addLogicalPort annotation time took 35.267828ms I0327 22:59:21.907465 3901 pods.go:271] [openshift-ingress/router-default-85bf9b9667-mm9hl] addLogicalPort took 36.385819ms, libovsdb time 776.889µs I0327 22:59:21.909084 3901 pod.go:62] [openshift-ingress/router-default-6bc69697db-p9rvv] pod update took 27.317345ms I0327 22:59:21.909101 3901 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-6bc69697db-p9rvv] addLogicalPort annotation time took 27.33906ms I0327 22:59:21.909982 3901 pods.go:271] [openshift-ingress/router-default-6bc69697db-p9rvv] addLogicalPort took 28.283289ms, libovsdb time 668.493µs I0327 22:59:22.079208 3901 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-65bqx] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-65bqx for pod on switch ip-10-0-2-195.ec2.internal I0327 22:59:22.079290 3901 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-65bqx I0327 22:59:22.096030 3901 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-65bqx] pod update took 16.758893ms I0327 22:59:22.096049 3901 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-65bqx] addLogicalPort annotation time took 16.784174ms I0327 22:59:22.097052 3901 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-65bqx] addLogicalPort took 17.860842ms, libovsdb time 771.808µs W0327 22:59:28.026877 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 22:59:28.392620 3901 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0327 22:59:28.393434 3901 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 793.232µs I0327 22:59:28.425573 3901 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 22:59:28.516315 3901 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-bl5jt] creating logical port openshift-kube-storage-version-migrator_migrator-854b5fb5c9-bl5jt for pod on switch ip-10-0-2-195.ec2.internal I0327 22:59:28.516395 3901 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-854b5fb5c9-bl5jt I0327 22:59:28.534747 3901 pod.go:62] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-bl5jt] pod update took 18.372209ms I0327 22:59:28.534764 3901 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-bl5jt] addLogicalPort annotation time took 18.393686ms I0327 22:59:28.535876 3901 pods.go:271] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-bl5jt] addLogicalPort took 19.578977ms, libovsdb time 818.246µs I0327 22:59:28.919277 3901 cni.go:403] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-bl5jt 02f2e26025e07300c6f0d5747667236da8c334d09cec93f9eee3a588d54f483b network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-bl5jt 02f2e26025e07300c6f0d5747667236da8c334d09cec93f9eee3a588d54f483b network default NAD default] I0327 22:59:28.928408 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-854b5fb5c9-bl5jt, hostIfaceName: 02f2e26025e0730, network: default, NAD default, SandboxID: "02f2e26025e07300c6f0d5747667236da8c334d09cec93f9eee3a588d54f483b", PCI device ID: , UID: "cca0cdd9-5199-4ffe-ad33-9d69fd164251", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0327 22:59:28.966889 3901 cni.go:424] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-bl5jt 02f2e26025e07300c6f0d5747667236da8c334d09cec93f9eee3a588d54f483b network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-bl5jt 02f2e26025e07300c6f0d5747667236da8c334d09cec93f9eee3a588d54f483b network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:57:b3:0a:e0:1c\",\"name\":\"02f2e26025e0730\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0a0ce90-1b71-424b-b0bf-47bbe4e94e35\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 22:59:29.464051 3901 namespace.go:98] [openshift-service-ca] adding namespace I0327 22:59:29.464883 3901 namespace.go:102] [openshift-service-ca] adding namespace took 806.932µs I0327 22:59:29.479891 3901 namespace.go:142] [openshift-service-ca] updating namespace W0327 22:59:37.597377 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 22:59:40.874561 3901 cni.go:403] [openshift-image-registry/image-registry-656cf7696d-mgnhg 25c06dfe12518602f59fd64301df4ce5f3ee57c4c4aea659b02d4603a2072dda network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-656cf7696d-mgnhg 25c06dfe12518602f59fd64301df4ce5f3ee57c4c4aea659b02d4603a2072dda network default NAD default] I0327 22:59:40.883166 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-656cf7696d-mgnhg, hostIfaceName: 25c06dfe1251860, network: default, NAD default, SandboxID: "25c06dfe12518602f59fd64301df4ce5f3ee57c4c4aea659b02d4603a2072dda", PCI device ID: , UID: "143326f1-1776-45d6-8dc7-c40cf33ca109", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0327 22:59:40.922655 3901 cni.go:424] [openshift-image-registry/image-registry-656cf7696d-mgnhg 25c06dfe12518602f59fd64301df4ce5f3ee57c4c4aea659b02d4603a2072dda network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-656cf7696d-mgnhg 25c06dfe12518602f59fd64301df4ce5f3ee57c4c4aea659b02d4603a2072dda network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:26:bb:0b:06:ee\",\"name\":\"25c06dfe1251860\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d12f2d79-f9fc-4967-aa2d-f8dc52f96717\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 22:59:41.195673 3901 cni.go:403] [openshift-dns/dns-default-mcftb 81bc17a548438fea381f6545019d224918a53fe07d6bca49081ae9891508d473 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-mcftb 81bc17a548438fea381f6545019d224918a53fe07d6bca49081ae9891508d473 network default NAD default] I0327 22:59:41.204909 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-mcftb, hostIfaceName: 81bc17a548438fe, network: default, NAD default, SandboxID: "81bc17a548438fea381f6545019d224918a53fe07d6bca49081ae9891508d473", PCI device ID: , UID: "d8efc82a-c961-4c5d-8e07-4e056cf5d734", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0327 22:59:41.250640 3901 cni.go:424] [openshift-dns/dns-default-mcftb 81bc17a548438fea381f6545019d224918a53fe07d6bca49081ae9891508d473 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-mcftb 81bc17a548438fea381f6545019d224918a53fe07d6bca49081ae9891508d473 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:ea:99:ea:00:c9\",\"name\":\"81bc17a548438fe\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1f5757b9-5ebe-4028-9a5c-23ad77ca4052\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 22:59:53.740784 3901 cni.go:403] [openshift-ingress/router-default-6bc69697db-p9rvv 6344fff0a95e393bc4ff00900cb3f40f8bfa12374cf03f7f64c7b15e5954a1e6 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-6bc69697db-p9rvv 6344fff0a95e393bc4ff00900cb3f40f8bfa12374cf03f7f64c7b15e5954a1e6 network default NAD default] I0327 22:59:53.749217 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-6bc69697db-p9rvv, hostIfaceName: 6344fff0a95e393, network: default, NAD default, SandboxID: "6344fff0a95e393bc4ff00900cb3f40f8bfa12374cf03f7f64c7b15e5954a1e6", PCI device ID: , UID: "59254dd4-7064-4a1f-b447-f78f0d944fbd", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0327 22:59:53.789721 3901 cni.go:424] [openshift-ingress/router-default-6bc69697db-p9rvv 6344fff0a95e393bc4ff00900cb3f40f8bfa12374cf03f7f64c7b15e5954a1e6 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-6bc69697db-p9rvv 6344fff0a95e393bc4ff00900cb3f40f8bfa12374cf03f7f64c7b15e5954a1e6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:af:f2:9f:a9:a8\",\"name\":\"6344fff0a95e393\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5011b6cd-0d08-41cb-b43b-30266f831ae6\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 22:59:53.937385 3901 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-65bqx f1ef86821231b247627c966be90d2c7a412f6613c908c835ba3e6eb2567180e2 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-65bqx f1ef86821231b247627c966be90d2c7a412f6613c908c835ba3e6eb2567180e2 network default NAD default] I0327 22:59:53.949487 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-65bqx, hostIfaceName: f1ef86821231b24, network: default, NAD default, SandboxID: "f1ef86821231b247627c966be90d2c7a412f6613c908c835ba3e6eb2567180e2", PCI device ID: , UID: "7b8f0a60-347b-44b2-9f93-0a6bb6960016", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0327 22:59:53.995153 3901 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-65bqx f1ef86821231b247627c966be90d2c7a412f6613c908c835ba3e6eb2567180e2 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-65bqx f1ef86821231b247627c966be90d2c7a412f6613c908c835ba3e6eb2567180e2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:36:4a:b6:cc:02\",\"name\":\"f1ef86821231b24\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f9a08df3-cdbb-4eda-b4c8-878b89c75a32\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 22:59:54.029917 3901 cni.go:403] [openshift-ingress/router-default-85bf9b9667-mm9hl 90fc0ee8b23794294821aad6724546e2ad47011034b57eac007e8a3d16315838 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-85bf9b9667-mm9hl 90fc0ee8b23794294821aad6724546e2ad47011034b57eac007e8a3d16315838 network default NAD default] I0327 22:59:54.039600 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-85bf9b9667-mm9hl, hostIfaceName: 90fc0ee8b237942, network: default, NAD default, SandboxID: "90fc0ee8b23794294821aad6724546e2ad47011034b57eac007e8a3d16315838", PCI device ID: , UID: "7de59b45-3300-4f4a-8e0d-eb750edd1a51", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0327 22:59:54.080861 3901 cni.go:424] [openshift-ingress/router-default-85bf9b9667-mm9hl 90fc0ee8b23794294821aad6724546e2ad47011034b57eac007e8a3d16315838 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-85bf9b9667-mm9hl 90fc0ee8b23794294821aad6724546e2ad47011034b57eac007e8a3d16315838 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:1a:10:a0:a8:60\",\"name\":\"90fc0ee8b237942\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cccc68b3-0a0d-48e9-90e5-4a7d2a3cd4d7\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 22:59:55.097358 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0327 22:59:55.097377 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 22:59:55.097555 3901 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0327 22:59:55.103719 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 22:59:55.103739 3901 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 22:59:55.103746 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 22:59:55.107813 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 22:59:55.270181 3901 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-w4gsn] creating logical port openshift-insights_insights-runtime-extractor-w4gsn for pod on switch ip-10-0-2-195.ec2.internal I0327 22:59:55.270342 3901 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-w4gsn I0327 22:59:55.288024 3901 pod.go:62] [openshift-insights/insights-runtime-extractor-w4gsn] pod update took 17.732251ms I0327 22:59:55.288050 3901 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-w4gsn] addLogicalPort annotation time took 17.76102ms I0327 22:59:55.289331 3901 pods.go:271] [openshift-insights/insights-runtime-extractor-w4gsn] addLogicalPort took 19.166558ms, libovsdb time 909.194µs W0327 22:59:55.466977 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0327 22:59:55.467007 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0327 22:59:55.552303 3901 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0327 22:59:55.564388 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 22:59:55.564407 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 22:59:55.564412 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default I0327 22:59:55.633839 3901 cni.go:403] [openshift-insights/insights-runtime-extractor-w4gsn a818b70d80bb299e738f41495baa93da2c280ae4b472910a653653b9c5e4e568 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-w4gsn a818b70d80bb299e738f41495baa93da2c280ae4b472910a653653b9c5e4e568 network default NAD default] I0327 22:59:55.641532 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-w4gsn, hostIfaceName: a818b70d80bb299, network: default, NAD default, SandboxID: "a818b70d80bb299e738f41495baa93da2c280ae4b472910a653653b9c5e4e568", PCI device ID: , UID: "abf057c0-51ce-44a4-8c55-be7b5687a113", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0327 22:59:55.643644 3901 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 22:59:55.682362 3901 cni.go:424] [openshift-insights/insights-runtime-extractor-w4gsn a818b70d80bb299e738f41495baa93da2c280ae4b472910a653653b9c5e4e568 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-w4gsn a818b70d80bb299e738f41495baa93da2c280ae4b472910a653653b9c5e4e568 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:ae:6a:5f:a2:65\",\"name\":\"a818b70d80bb299\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ae19c457-1952-4613-a1b6-6368443d7a61\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 22:59:56.471340 3901 obj_retry.go:464] Detected object openshift-image-registry/image-registry-656cf7696d-955wr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 22:59:56.471970 3901 obj_retry.go:464] Detected object openshift-image-registry/image-registry-656cf7696d-955wr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 22:59:56.488725 3901 obj_retry.go:458] Detected object openshift-image-registry/image-registry-656cf7696d-955wr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:56.488745 3901 obj_retry.go:458] Detected object openshift-image-registry/image-registry-656cf7696d-955wr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:56.493633 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-656cf7696d-955wr I0327 22:59:56.493653 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-656cf7696d-955wr I0327 22:59:57.149003 3901 obj_retry.go:464] Detected object openshift-image-registry/image-registry-57996bf879-5hf57 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 22:59:57.149529 3901 obj_retry.go:464] Detected object openshift-image-registry/image-registry-57996bf879-5hf57 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 22:59:57.162926 3901 obj_retry.go:458] Detected object openshift-image-registry/image-registry-57996bf879-5hf57 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:57.162943 3901 obj_retry.go:458] Detected object openshift-image-registry/image-registry-57996bf879-5hf57 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:57.173755 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-57996bf879-5hf57 I0327 22:59:57.173781 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-57996bf879-5hf57 W0327 22:59:57.478698 3901 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0327 22:59:58.142569 3901 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0327 22:59:59.139958 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0327 23:00:02.397075 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0327 23:00:06.307569 3901 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-789f65cff9-d8r7v] creating logical port openshift-monitoring_thanos-querier-789f65cff9-d8r7v for pod on switch ip-10-0-2-195.ec2.internal I0327 23:00:06.307703 3901 kube.go:256] Updating pod openshift-monitoring/thanos-querier-789f65cff9-d8r7v I0327 23:00:06.324112 3901 pod.go:62] [openshift-monitoring/thanos-querier-789f65cff9-d8r7v] pod update took 16.431255ms I0327 23:00:06.324141 3901 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-789f65cff9-d8r7v] addLogicalPort annotation time took 16.471112ms I0327 23:00:06.325551 3901 pods.go:271] [openshift-monitoring/thanos-querier-789f65cff9-d8r7v] addLogicalPort took 17.996174ms, libovsdb time 1.020613ms I0327 23:00:06.663994 3901 cni.go:403] [openshift-monitoring/thanos-querier-789f65cff9-d8r7v 73abc8aaa35f7f45477094c2c294cd748ea0050845f3194afffabbdf7ebaaba3 network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-789f65cff9-d8r7v 73abc8aaa35f7f45477094c2c294cd748ea0050845f3194afffabbdf7ebaaba3 network default NAD default] I0327 23:00:06.672191 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-789f65cff9-d8r7v, hostIfaceName: 73abc8aaa35f7f4, network: default, NAD default, SandboxID: "73abc8aaa35f7f45477094c2c294cd748ea0050845f3194afffabbdf7ebaaba3", PCI device ID: , UID: "a141c40d-a0e9-4191-b6b4-7dccb5954180", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0327 23:00:06.713272 3901 cni.go:424] [openshift-monitoring/thanos-querier-789f65cff9-d8r7v 73abc8aaa35f7f45477094c2c294cd748ea0050845f3194afffabbdf7ebaaba3 network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-789f65cff9-d8r7v 73abc8aaa35f7f45477094c2c294cd748ea0050845f3194afffabbdf7ebaaba3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:a1:40:5e:8f:88\",\"name\":\"73abc8aaa35f7f4\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4a9124ca-d5c2-4b86-adf1-bb34be56da0d\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:00:07.579021 3901 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:07.579042 3901 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:07.579048 3901 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 23:00:08.745936 3901 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-66586f8fb9-j4d8c] creating logical port openshift-monitoring_metrics-server-66586f8fb9-j4d8c for pod on switch ip-10-0-2-195.ec2.internal I0327 23:00:08.746034 3901 kube.go:256] Updating pod openshift-monitoring/metrics-server-66586f8fb9-j4d8c I0327 23:00:08.764432 3901 pod.go:62] [openshift-monitoring/metrics-server-66586f8fb9-j4d8c] pod update took 18.420808ms I0327 23:00:08.764453 3901 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-66586f8fb9-j4d8c] addLogicalPort annotation time took 18.447882ms I0327 23:00:08.765530 3901 pods.go:271] [openshift-monitoring/metrics-server-66586f8fb9-j4d8c] addLogicalPort took 19.608827ms, libovsdb time 794.964µs I0327 23:00:09.100927 3901 cni.go:403] [openshift-monitoring/metrics-server-66586f8fb9-j4d8c c0003ac398a33580c4b0c9d2afc4a6fca4831076833880cc05637a5551dcec9c network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-66586f8fb9-j4d8c c0003ac398a33580c4b0c9d2afc4a6fca4831076833880cc05637a5551dcec9c network default NAD default] I0327 23:00:09.108890 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-66586f8fb9-j4d8c, hostIfaceName: c0003ac398a3358, network: default, NAD default, SandboxID: "c0003ac398a33580c4b0c9d2afc4a6fca4831076833880cc05637a5551dcec9c", PCI device ID: , UID: "f685b938-8116-4371-9c93-c43179b8ff9d", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0327 23:00:09.150595 3901 cni.go:424] [openshift-monitoring/metrics-server-66586f8fb9-j4d8c c0003ac398a33580c4b0c9d2afc4a6fca4831076833880cc05637a5551dcec9c network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-66586f8fb9-j4d8c c0003ac398a33580c4b0c9d2afc4a6fca4831076833880cc05637a5551dcec9c network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:7f:18:c2:ce:e7\",\"name\":\"c0003ac398a3358\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0506fbd1-f23a-4584-850b-2b989aa2298e\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:00:09.214062 3901 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:09.214083 3901 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:09.214089 3901 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:10.513312 3901 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.513334 3901 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.513341 3901 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.513346 3901 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier I0327 23:00:10.574838 3901 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-2-195.ec2.internal I0327 23:00:10.574954 3901 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0327 23:00:10.612090 3901 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 37.182346ms I0327 23:00:10.612112 3901 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 37.209857ms I0327 23:00:10.613183 3901 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 38.361404ms, libovsdb time 764.741µs W0327 23:00:10.649067 3901 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.649080 3901 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.649087 3901 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.649091 3901 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier I0327 23:00:10.936844 3901 cni.go:403] [openshift-monitoring/prometheus-k8s-0 20c350d3c653608015f0152252bfe4d65cd1b32ead7bd5125d4f0e8d2fbb4c80 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 20c350d3c653608015f0152252bfe4d65cd1b32ead7bd5125d4f0e8d2fbb4c80 network default NAD default] I0327 23:00:10.945723 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 20c350d3c653608, network: default, NAD default, SandboxID: "20c350d3c653608015f0152252bfe4d65cd1b32ead7bd5125d4f0e8d2fbb4c80", PCI device ID: , UID: "5ee5a659-d195-4c06-846a-aff9649c1a9a", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0327 23:00:10.987200 3901 cni.go:424] [openshift-monitoring/prometheus-k8s-0 20c350d3c653608015f0152252bfe4d65cd1b32ead7bd5125d4f0e8d2fbb4c80 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 20c350d3c653608015f0152252bfe4d65cd1b32ead7bd5125d4f0e8d2fbb4c80 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:bc:53:da:44:56\",\"name\":\"20c350d3c653608\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6bcb9d31-7cda-4fd7-8e26-c0c5ae00b831\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:00:11.217932 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0327 23:00:11.514823 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 23:00:11.571630 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 23:00:11.612755 3901 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 23:00:11.612777 3901 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 23:00:12.529935 3901 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 23:00:12.529950 3901 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0327 23:00:13.102309 3901 cni.go:403] [openshift-multus/network-metrics-daemon-996pn 252aace63b09eb8efe0d555a439cf964f265760c8c636f27f4c43783bb5363b6 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-996pn 252aace63b09eb8efe0d555a439cf964f265760c8c636f27f4c43783bb5363b6 network default NAD default] I0327 23:00:13.111406 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-996pn, hostIfaceName: 252aace63b09eb8, network: default, NAD default, SandboxID: "252aace63b09eb8efe0d555a439cf964f265760c8c636f27f4c43783bb5363b6", PCI device ID: , UID: "61cbd592-bfa1-4070-b23f-4e6c640e6a58", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0327 23:00:13.158146 3901 cni.go:424] [openshift-multus/network-metrics-daemon-996pn 252aace63b09eb8efe0d555a439cf964f265760c8c636f27f4c43783bb5363b6 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-996pn 252aace63b09eb8efe0d555a439cf964f265760c8c636f27f4c43783bb5363b6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:0c:f3:25:b6:cf\",\"name\":\"252aace63b09eb8\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0d23e2c-e7f7-4b07-91af-82a7234bd5af\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:00:15.621489 3901 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-d4fhx] creating logical port openshift-console_downloads-56d45c498b-d4fhx for pod on switch ip-10-0-2-195.ec2.internal I0327 23:00:15.621645 3901 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-d4fhx I0327 23:00:15.639058 3901 pod.go:62] [openshift-console/downloads-56d45c498b-d4fhx] pod update took 17.446871ms I0327 23:00:15.639089 3901 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-d4fhx] addLogicalPort annotation time took 17.481989ms I0327 23:00:15.642914 3901 pods.go:271] [openshift-console/downloads-56d45c498b-d4fhx] addLogicalPort took 21.454873ms, libovsdb time 3.368611ms I0327 23:00:15.980354 3901 cni.go:403] [openshift-console/downloads-56d45c498b-d4fhx c5ace83c73cad5466a00004c458a63dc266725f940369ab40a1eddd36b4bb6ea network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-d4fhx c5ace83c73cad5466a00004c458a63dc266725f940369ab40a1eddd36b4bb6ea network default NAD default] I0327 23:00:15.995723 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-d4fhx, hostIfaceName: c5ace83c73cad54, network: default, NAD default, SandboxID: "c5ace83c73cad5466a00004c458a63dc266725f940369ab40a1eddd36b4bb6ea", PCI device ID: , UID: "cb310da0-2381-4827-98fe-3145ea9f2c9e", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0327 23:00:16.039145 3901 cni.go:424] [openshift-console/downloads-56d45c498b-d4fhx c5ace83c73cad5466a00004c458a63dc266725f940369ab40a1eddd36b4bb6ea network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-d4fhx c5ace83c73cad5466a00004c458a63dc266725f940369ab40a1eddd36b4bb6ea network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:32:e0:ea:9d:cf\",\"name\":\"c5ace83c73cad54\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8e960787-8f66-4224-93b1-9e47676577b4\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:00:16.524080 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:16.524101 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:16.524107 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:16.524112 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0327 23:00:25.152315 3901 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 23:00:25.650516 3901 base_network_controller_pods.go:487] [default/openshift-console/console-8665c6c979-b5t6s] creating logical port openshift-console_console-8665c6c979-b5t6s for pod on switch ip-10-0-2-195.ec2.internal I0327 23:00:25.650635 3901 kube.go:256] Updating pod openshift-console/console-8665c6c979-b5t6s I0327 23:00:25.666449 3901 pod.go:62] [openshift-console/console-8665c6c979-b5t6s] pod update took 15.833494ms I0327 23:00:25.666492 3901 base_network_controller_pods.go:937] [default/openshift-console/console-8665c6c979-b5t6s] addLogicalPort annotation time took 15.881827ms I0327 23:00:25.667689 3901 pods.go:271] [openshift-console/console-8665c6c979-b5t6s] addLogicalPort took 17.192234ms, libovsdb time 913.704µs I0327 23:00:32.178750 3901 cni.go:403] [openshift-console/console-8665c6c979-b5t6s 7d73bd774eaf8429c8cf82b5561a6c540fe5d114597ad1496bfde62cf451f720 network default NAD default] ADD starting CNI request [openshift-console/console-8665c6c979-b5t6s 7d73bd774eaf8429c8cf82b5561a6c540fe5d114597ad1496bfde62cf451f720 network default NAD default] I0327 23:00:32.186948 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-8665c6c979-b5t6s, hostIfaceName: 7d73bd774eaf842, network: default, NAD default, SandboxID: "7d73bd774eaf8429c8cf82b5561a6c540fe5d114597ad1496bfde62cf451f720", PCI device ID: , UID: "b837a980-7ff4-48f9-953e-3d5e68243bfc", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0327 23:00:32.229454 3901 cni.go:424] [openshift-console/console-8665c6c979-b5t6s 7d73bd774eaf8429c8cf82b5561a6c540fe5d114597ad1496bfde62cf451f720 network default NAD default] ADD finished CNI request [openshift-console/console-8665c6c979-b5t6s 7d73bd774eaf8429c8cf82b5561a6c540fe5d114597ad1496bfde62cf451f720 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:50:db:fd:a3:f1\",\"name\":\"7d73bd774eaf842\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ff0becc1-0b63-4c14-9eee-fc7a4d498d28\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:00:32.520202 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 23:00:32.605912 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0327 23:00:32.631178 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0327 23:00:35.621815 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 23:00:36.320961 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 23:00:36.636328 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 23:00:43.361152 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0327 23:00:43.688124 3901 cni.go:403] [openshift-image-registry/image-registry-656cf7696d-mgnhg 25c06dfe12518602f59fd64301df4ce5f3ee57c4c4aea659b02d4603a2072dda network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-656cf7696d-mgnhg 25c06dfe12518602f59fd64301df4ce5f3ee57c4c4aea659b02d4603a2072dda network default NAD default] I0327 23:00:43.770401 3901 cni.go:424] [openshift-image-registry/image-registry-656cf7696d-mgnhg 25c06dfe12518602f59fd64301df4ce5f3ee57c4c4aea659b02d4603a2072dda network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-656cf7696d-mgnhg 25c06dfe12518602f59fd64301df4ce5f3ee57c4c4aea659b02d4603a2072dda network default NAD default], result "{}", err I0327 23:00:43.834979 3901 obj_retry.go:464] Detected object openshift-image-registry/image-registry-656cf7696d-mgnhg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:00:43.835016 3901 pods.go:176] Deleting pod: openshift-image-registry/image-registry-656cf7696d-mgnhg I0327 23:00:43.836188 3901 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-656cf7696d-mgnhg, ips: 10.129.0.6 I0327 23:00:43.836222 3901 obj_retry.go:464] Detected object openshift-image-registry/image-registry-656cf7696d-mgnhg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:00:44.651695 3901 obj_retry.go:458] Detected object openshift-image-registry/image-registry-656cf7696d-mgnhg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:00:44.651723 3901 obj_retry.go:458] Detected object openshift-image-registry/image-registry-656cf7696d-mgnhg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:00:44.657629 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-656cf7696d-mgnhg I0327 23:00:44.657662 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-656cf7696d-mgnhg W0327 23:00:53.403712 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 23:00:59.428797 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0327 23:01:01.652401 3901 obj_retry.go:464] Detected object openshift-console/console-8665c6c979-4sq4c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:01.652964 3901 obj_retry.go:464] Detected object openshift-console/console-8665c6c979-4sq4c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:02.465285 3901 obj_retry.go:458] Detected object openshift-console/console-8665c6c979-4sq4c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:02.465316 3901 obj_retry.go:458] Detected object openshift-console/console-8665c6c979-4sq4c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:02.465342 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-8665c6c979-4sq4c I0327 23:01:02.465354 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-8665c6c979-4sq4c I0327 23:01:09.942204 3901 cni.go:403] [openshift-console/console-8665c6c979-b5t6s 7d73bd774eaf8429c8cf82b5561a6c540fe5d114597ad1496bfde62cf451f720 network default NAD default] DEL starting CNI request [openshift-console/console-8665c6c979-b5t6s 7d73bd774eaf8429c8cf82b5561a6c540fe5d114597ad1496bfde62cf451f720 network default NAD default] I0327 23:01:10.018454 3901 cni.go:424] [openshift-console/console-8665c6c979-b5t6s 7d73bd774eaf8429c8cf82b5561a6c540fe5d114597ad1496bfde62cf451f720 network default NAD default] DEL finished CNI request [openshift-console/console-8665c6c979-b5t6s 7d73bd774eaf8429c8cf82b5561a6c540fe5d114597ad1496bfde62cf451f720 network default NAD default], result "{}", err I0327 23:01:10.084732 3901 obj_retry.go:464] Detected object openshift-console/console-8665c6c979-b5t6s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:10.084766 3901 pods.go:176] Deleting pod: openshift-console/console-8665c6c979-b5t6s I0327 23:01:10.086130 3901 pods.go:236] Attempting to release IPs for pod: openshift-console/console-8665c6c979-b5t6s, ips: 10.129.0.19 I0327 23:01:10.086155 3901 obj_retry.go:464] Detected object openshift-console/console-8665c6c979-b5t6s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:10.753121 3901 obj_retry.go:458] Detected object openshift-console/console-8665c6c979-b5t6s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:10.753147 3901 obj_retry.go:458] Detected object openshift-console/console-8665c6c979-b5t6s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:10.759025 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-8665c6c979-b5t6s I0327 23:01:10.759043 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-8665c6c979-b5t6s I0327 23:01:11.247217 3901 cni.go:403] [openshift-ingress/router-default-85bf9b9667-mm9hl 90fc0ee8b23794294821aad6724546e2ad47011034b57eac007e8a3d16315838 network default NAD default] DEL starting CNI request [openshift-ingress/router-default-85bf9b9667-mm9hl 90fc0ee8b23794294821aad6724546e2ad47011034b57eac007e8a3d16315838 network default NAD default] I0327 23:01:11.335523 3901 cni.go:424] [openshift-ingress/router-default-85bf9b9667-mm9hl 90fc0ee8b23794294821aad6724546e2ad47011034b57eac007e8a3d16315838 network default NAD default] DEL finished CNI request [openshift-ingress/router-default-85bf9b9667-mm9hl 90fc0ee8b23794294821aad6724546e2ad47011034b57eac007e8a3d16315838 network default NAD default], result "{}", err I0327 23:01:11.401609 3901 obj_retry.go:464] Detected object openshift-ingress/router-default-85bf9b9667-mm9hl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:11.401637 3901 pods.go:176] Deleting pod: openshift-ingress/router-default-85bf9b9667-mm9hl I0327 23:01:11.402880 3901 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-85bf9b9667-mm9hl, ips: 10.129.0.10 I0327 23:01:11.402905 3901 obj_retry.go:464] Detected object openshift-ingress/router-default-85bf9b9667-mm9hl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:11.759028 3901 obj_retry.go:458] Detected object openshift-ingress/router-default-85bf9b9667-mm9hl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:11.759054 3901 obj_retry.go:458] Detected object openshift-ingress/router-default-85bf9b9667-mm9hl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:11.764557 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-85bf9b9667-mm9hl I0327 23:01:11.764580 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-85bf9b9667-mm9hl W0327 23:01:11.808223 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 23:01:11.808252 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0327 23:01:25.986746 3901 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:25.987293 3901 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:26.549592 3901 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:26.549612 3901 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:26.556153 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0327 23:01:26.556173 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0327 23:01:30.370519 3901 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:30.371030 3901 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:30.902008 3901 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:30.902036 3901 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:30.908544 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0327 23:01:30.908569 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0327 23:01:31.878159 3901 base_network_controller_pods.go:487] [default/openshift-console/console-84546c86b9-jkpqt] creating logical port openshift-console_console-84546c86b9-jkpqt for pod on switch ip-10-0-2-195.ec2.internal I0327 23:01:31.878255 3901 kube.go:256] Updating pod openshift-console/console-84546c86b9-jkpqt I0327 23:01:31.900869 3901 pod.go:62] [openshift-console/console-84546c86b9-jkpqt] pod update took 22.641899ms I0327 23:01:31.900890 3901 base_network_controller_pods.go:937] [default/openshift-console/console-84546c86b9-jkpqt] addLogicalPort annotation time took 22.668445ms I0327 23:01:31.901931 3901 pods.go:271] [openshift-console/console-84546c86b9-jkpqt] addLogicalPort took 23.791266ms, libovsdb time 706.065µs I0327 23:01:32.242898 3901 cni.go:403] [openshift-console/console-84546c86b9-jkpqt f4fc3cf8bb390f4b0dc55d1fd6b82d63a481b75194a87cd31cf32220efe98bbf network default NAD default] ADD starting CNI request [openshift-console/console-84546c86b9-jkpqt f4fc3cf8bb390f4b0dc55d1fd6b82d63a481b75194a87cd31cf32220efe98bbf network default NAD default] I0327 23:01:32.251159 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-84546c86b9-jkpqt, hostIfaceName: f4fc3cf8bb390f4, network: default, NAD default, SandboxID: "f4fc3cf8bb390f4b0dc55d1fd6b82d63a481b75194a87cd31cf32220efe98bbf", PCI device ID: , UID: "2f3af793-45e6-413a-81f5-c3b94b930d8a", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0327 23:01:32.292226 3901 cni.go:424] [openshift-console/console-84546c86b9-jkpqt f4fc3cf8bb390f4b0dc55d1fd6b82d63a481b75194a87cd31cf32220efe98bbf network default NAD default] ADD finished CNI request [openshift-console/console-84546c86b9-jkpqt f4fc3cf8bb390f4b0dc55d1fd6b82d63a481b75194a87cd31cf32220efe98bbf network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:52:fe:ba:ef:ed\",\"name\":\"f4fc3cf8bb390f4\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9f4b2824-d80e-48fb-9407-9e7f4cb3b137\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:01:36.058191 3901 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-7mjmh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:36.058726 3901 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-7mjmh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:37.093207 3901 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-7mjmh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:37.093231 3901 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-7mjmh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 23:01:42.905485 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 23:01:57.174302 3901 obj_retry.go:464] Detected object openshift-console/console-759495f8d9-2zdq6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:57.174869 3901 obj_retry.go:464] Detected object openshift-console/console-759495f8d9-2zdq6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:57.684743 3901 obj_retry.go:458] Detected object openshift-console/console-759495f8d9-2zdq6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:57.684770 3901 obj_retry.go:458] Detected object openshift-console/console-759495f8d9-2zdq6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:57.690778 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-759495f8d9-2zdq6 I0327 23:01:57.690803 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-759495f8d9-2zdq6 I0327 23:01:58.314947 3901 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:58.315569 3901 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:58.801529 3901 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 23:01:59.039835 3901 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:59.039861 3901 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:59.045744 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0327 23:01:59.045795 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0327 23:02:08.243687 3901 obj_retry.go:464] Detected object openshift-console/console-759495f8d9-x5c28 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:08.244276 3901 obj_retry.go:464] Detected object openshift-console/console-759495f8d9-x5c28 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:09.087487 3901 obj_retry.go:458] Detected object openshift-console/console-759495f8d9-x5c28 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:02:09.087509 3901 obj_retry.go:458] Detected object openshift-console/console-759495f8d9-x5c28 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:02:09.089636 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-759495f8d9-x5c28 I0327 23:02:09.089659 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-759495f8d9-x5c28 I0327 23:02:33.378688 3901 cni.go:403] [openshift-monitoring/prometheus-k8s-0 20c350d3c653608015f0152252bfe4d65cd1b32ead7bd5125d4f0e8d2fbb4c80 network default NAD default] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 20c350d3c653608015f0152252bfe4d65cd1b32ead7bd5125d4f0e8d2fbb4c80 network default NAD default] I0327 23:02:33.463113 3901 cni.go:424] [openshift-monitoring/prometheus-k8s-0 20c350d3c653608015f0152252bfe4d65cd1b32ead7bd5125d4f0e8d2fbb4c80 network default NAD default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 20c350d3c653608015f0152252bfe4d65cd1b32ead7bd5125d4f0e8d2fbb4c80 network default NAD default], result "{}", err I0327 23:02:33.527405 3901 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:33.527443 3901 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0327 23:02:33.528636 3901 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.129.0.17 I0327 23:02:33.528666 3901 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:34.074825 3901 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:02:34.074915 3901 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:02:34.081060 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0327 23:02:34.081095 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0327 23:02:34.120511 3901 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-2-195.ec2.internal I0327 23:02:34.120614 3901 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0327 23:02:34.141838 3901 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 21.26414ms I0327 23:02:34.141867 3901 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 21.294755ms I0327 23:02:34.143186 3901 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 22.695764ms, libovsdb time 928.932µs I0327 23:02:34.478860 3901 cni.go:403] [openshift-monitoring/prometheus-k8s-0 2153f55b029a6314fd105007ab1478ca82f8bcfc7e5574f99604ee68b6cece7a network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 2153f55b029a6314fd105007ab1478ca82f8bcfc7e5574f99604ee68b6cece7a network default NAD default] I0327 23:02:34.487160 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 2153f55b029a631, network: default, NAD default, SandboxID: "2153f55b029a6314fd105007ab1478ca82f8bcfc7e5574f99604ee68b6cece7a", PCI device ID: , UID: "3de98ffa-a9f4-49f5-b9fe-025e0a1b7e28", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0327 23:02:34.532168 3901 cni.go:424] [openshift-monitoring/prometheus-k8s-0 2153f55b029a6314fd105007ab1478ca82f8bcfc7e5574f99604ee68b6cece7a network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 2153f55b029a6314fd105007ab1478ca82f8bcfc7e5574f99604ee68b6cece7a network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:f8:89:31:64:51\",\"name\":\"2153f55b029a631\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/135b6484-e5c5-432f-88fc-1ff407b46429\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:03:33.393763 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=4 W0327 23:03:35.349455 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 23:03:35.349495 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0327 23:03:39.388254 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0327 23:03:50.377643 3901 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0327 23:03:58.414855 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0327 23:04:04.416660 3901 obj_retry.go:464] Detected object openshift-console/console-84546c86b9-j852z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:04.417243 3901 obj_retry.go:464] Detected object openshift-console/console-84546c86b9-j852z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:05.194818 3901 obj_retry.go:458] Detected object openshift-console/console-84546c86b9-j852z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:05.194860 3901 obj_retry.go:458] Detected object openshift-console/console-84546c86b9-j852z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:05.200816 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-84546c86b9-j852z I0327 23:04:05.200848 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-84546c86b9-j852z W0327 23:04:11.239553 3901 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 23:04:14.398810 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0327 23:04:14.690612 3901 cni.go:403] [openshift-console/console-84546c86b9-jkpqt f4fc3cf8bb390f4b0dc55d1fd6b82d63a481b75194a87cd31cf32220efe98bbf network default NAD default] DEL starting CNI request [openshift-console/console-84546c86b9-jkpqt f4fc3cf8bb390f4b0dc55d1fd6b82d63a481b75194a87cd31cf32220efe98bbf network default NAD default] I0327 23:04:14.769259 3901 cni.go:424] [openshift-console/console-84546c86b9-jkpqt f4fc3cf8bb390f4b0dc55d1fd6b82d63a481b75194a87cd31cf32220efe98bbf network default NAD default] DEL finished CNI request [openshift-console/console-84546c86b9-jkpqt f4fc3cf8bb390f4b0dc55d1fd6b82d63a481b75194a87cd31cf32220efe98bbf network default NAD default], result "{}", err I0327 23:04:14.812017 3901 obj_retry.go:464] Detected object openshift-console/console-84546c86b9-jkpqt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:14.812054 3901 pods.go:176] Deleting pod: openshift-console/console-84546c86b9-jkpqt I0327 23:04:14.813132 3901 pods.go:236] Attempting to release IPs for pod: openshift-console/console-84546c86b9-jkpqt, ips: 10.129.0.20 I0327 23:04:14.813161 3901 obj_retry.go:464] Detected object openshift-console/console-84546c86b9-jkpqt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:15.458635 3901 obj_retry.go:458] Detected object openshift-console/console-84546c86b9-jkpqt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:15.458658 3901 obj_retry.go:458] Detected object openshift-console/console-84546c86b9-jkpqt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:15.463814 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-84546c86b9-jkpqt I0327 23:04:15.463841 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-84546c86b9-jkpqt I0327 23:04:22.271206 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0327 23:04:22.271265 3901 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-28bsv IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-28bsv: no pod IPs found I0327 23:04:52.296380 3901 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0327 23:05:06.415752 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0327 23:05:08.409628 3901 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0327 23:05:24.378527 3901 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0327 23:05:31.302165 3901 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=71 I0327 23:05:55.309125 3901 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=625 I0327 23:06:06.063529 3901 namespace.go:98] [tsf] adding namespace I0327 23:06:06.064502 3901 namespace.go:102] [tsf] adding namespace took 946.778µs I0327 23:06:06.079833 3901 namespace.go:142] [tsf] updating namespace I0327 23:06:06.090745 3901 namespace.go:142] [tsf] updating namespace I0327 23:06:09.797798 3901 namespace.go:98] [rhtpa-operator] adding namespace I0327 23:06:09.798629 3901 namespace.go:102] [rhtpa-operator] adding namespace took 804.846µs I0327 23:06:09.802514 3901 namespace.go:98] [konflux-ui] adding namespace I0327 23:06:09.802554 3901 namespace.go:98] [tsf-keycloak] adding namespace I0327 23:06:09.802717 3901 namespace.go:98] [cert-manager-operator] adding namespace I0327 23:06:09.802775 3901 namespace.go:98] [tsf-tas] adding namespace I0327 23:06:09.803296 3901 namespace.go:102] [konflux-ui] adding namespace took 762.148µs I0327 23:06:09.803326 3901 namespace.go:98] [konflux-operator] adding namespace I0327 23:06:09.803839 3901 namespace.go:102] [konflux-operator] adding namespace took 506.511µs I0327 23:06:09.804327 3901 namespace.go:102] [tsf-keycloak] adding namespace took 1.759689ms I0327 23:06:09.804790 3901 namespace.go:102] [cert-manager-operator] adding namespace took 2.058189ms I0327 23:06:09.805070 3901 namespace.go:98] [rhbk-operator] adding namespace I0327 23:06:09.805101 3901 namespace.go:98] [tsf-tpa] adding namespace I0327 23:06:09.805421 3901 namespace.go:102] [tsf-tas] adding namespace took 2.63362ms I0327 23:06:09.805968 3901 namespace.go:102] [rhbk-operator] adding namespace took 884.346µs I0327 23:06:09.806585 3901 namespace.go:102] [tsf-tpa] adding namespace took 1.465675ms I0327 23:06:09.815667 3901 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:06:09.829770 3901 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:06:09.850663 3901 namespace.go:142] [konflux-ui] updating namespace I0327 23:06:09.856396 3901 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:06:09.860433 3901 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:06:09.864412 3901 namespace.go:142] [tsf-tas] updating namespace I0327 23:06:09.884509 3901 namespace.go:142] [konflux-ui] updating namespace I0327 23:06:09.884569 3901 namespace.go:142] [konflux-operator] updating namespace I0327 23:06:09.884693 3901 namespace.go:142] [rhbk-operator] updating namespace I0327 23:06:09.889581 3901 namespace.go:142] [tsf-tpa] updating namespace I0327 23:06:09.889603 3901 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:06:09.948719 3901 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:06:09.967405 3901 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:06:09.980692 3901 namespace.go:142] [tsf-tas] updating namespace I0327 23:06:10.000078 3901 namespace.go:142] [tsf-tas] updating namespace I0327 23:06:10.011528 3901 namespace.go:142] [konflux-ui] updating namespace I0327 23:06:10.021752 3901 namespace.go:142] [konflux-operator] updating namespace I0327 23:06:10.029425 3901 namespace.go:142] [konflux-operator] updating namespace I0327 23:06:10.049502 3901 namespace.go:142] [rhbk-operator] updating namespace I0327 23:06:10.058500 3901 namespace.go:142] [rhbk-operator] updating namespace I0327 23:06:10.088618 3901 namespace.go:142] [tsf-tpa] updating namespace I0327 23:06:10.109514 3901 namespace.go:142] [tsf-tpa] updating namespace I0327 23:06:10.109546 3901 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:06:10.141457 3901 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:06:11.139142 3901 namespace.go:142] [tsf-tpa] updating namespace I0327 23:06:11.162610 3901 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:06:11.349762 3901 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-2-195.ec2.internal I0327 23:06:11.349883 3901 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0327 23:06:11.375843 3901 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 25.989407ms I0327 23:06:11.375863 3901 base_network_controller_pods.go:937] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 26.013142ms I0327 23:06:11.377047 3901 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 27.303981ms, libovsdb time 861.395µs I0327 23:06:11.715531 3901 cni.go:403] [tsf/test-tsf-subscriptions 7246647b4080cb387c201ee15913970369d754c2c1b9e89a9ce5145d3f2af847 network default NAD default] ADD starting CNI request [tsf/test-tsf-subscriptions 7246647b4080cb387c201ee15913970369d754c2c1b9e89a9ce5145d3f2af847 network default NAD default] I0327 23:06:11.724580 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: 7246647b4080cb3, network: default, NAD default, SandboxID: "7246647b4080cb387c201ee15913970369d754c2c1b9e89a9ce5145d3f2af847", PCI device ID: , UID: "abe532df-2f9a-485a-ad57-9de6165a2e88", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0327 23:06:11.766948 3901 cni.go:424] [tsf/test-tsf-subscriptions 7246647b4080cb387c201ee15913970369d754c2c1b9e89a9ce5145d3f2af847 network default NAD default] ADD finished CNI request [tsf/test-tsf-subscriptions 7246647b4080cb387c201ee15913970369d754c2c1b9e89a9ce5145d3f2af847 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:c8:5e:6d:d6:41\",\"name\":\"7246647b4080cb3\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b685993e-1fdd-4392-89ee-bdffb31d5244\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:06:12.781145 3901 base_network_controller_pods.go:487] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9] creating logical port openshift-marketplace_925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 for pod on switch ip-10-0-2-195.ec2.internal I0327 23:06:12.781321 3901 kube.go:256] Updating pod openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 I0327 23:06:12.782014 3901 base_network_controller_pods.go:487] [default/openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7] creating logical port openshift-marketplace_ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 for pod on switch ip-10-0-2-195.ec2.internal I0327 23:06:12.782129 3901 kube.go:256] Updating pod openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 I0327 23:06:12.797344 3901 pod.go:62] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9] pod update took 16.053546ms I0327 23:06:12.797368 3901 base_network_controller_pods.go:937] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9] addLogicalPort annotation time took 16.082715ms I0327 23:06:12.798705 3901 pods.go:271] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9] addLogicalPort took 17.579772ms, libovsdb time 966.282µs I0327 23:06:12.799052 3901 pod.go:62] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7] pod update took 16.952048ms I0327 23:06:12.799116 3901 base_network_controller_pods.go:937] [default/openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7] addLogicalPort annotation time took 17.01674ms I0327 23:06:12.800711 3901 pods.go:271] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7] addLogicalPort took 18.707278ms, libovsdb time 1.065312ms I0327 23:06:13.336151 3901 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 581f20a2b34694fd70216f2ce44a7a64f5d430ae99dcd70b68b9e7da3aa1e770 network default NAD default] ADD starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 581f20a2b34694fd70216f2ce44a7a64f5d430ae99dcd70b68b9e7da3aa1e770 network default NAD default] I0327 23:06:13.336170 3901 cni.go:403] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 555e23136c2b4b91870875542a201f86ef5dfd2945b45024dfce3e973363c739 network default NAD default] ADD starting CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 555e23136c2b4b91870875542a201f86ef5dfd2945b45024dfce3e973363c739 network default NAD default] I0327 23:06:13.346333 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7, hostIfaceName: 555e23136c2b4b9, network: default, NAD default, SandboxID: "555e23136c2b4b91870875542a201f86ef5dfd2945b45024dfce3e973363c739", PCI device ID: , UID: "92d53a66-45c5-410e-bfa8-77e26fe22771", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0327 23:06:13.346337 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9, hostIfaceName: 581f20a2b34694f, network: default, NAD default, SandboxID: "581f20a2b34694fd70216f2ce44a7a64f5d430ae99dcd70b68b9e7da3aa1e770", PCI device ID: , UID: "d4b92859-f3b4-4a48-a6c6-ea99c1789c00", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0327 23:06:13.389931 3901 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 581f20a2b34694fd70216f2ce44a7a64f5d430ae99dcd70b68b9e7da3aa1e770 network default NAD default] ADD finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 581f20a2b34694fd70216f2ce44a7a64f5d430ae99dcd70b68b9e7da3aa1e770 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:f8:1f:2c:f2:c1\",\"name\":\"581f20a2b34694f\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/745bb8f3-88c8-4389-b8e4-f32a3cfb1529\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:06:13.614294 3901 cni.go:424] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 555e23136c2b4b91870875542a201f86ef5dfd2945b45024dfce3e973363c739 network default NAD default] ADD finished CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 555e23136c2b4b91870875542a201f86ef5dfd2945b45024dfce3e973363c739 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:f2:98:06:00:07\",\"name\":\"555e23136c2b4b9\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bd86d362-4968-4907-8a5a-95003d32c333\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:06:15.698883 3901 base_network_controller_pods.go:487] [default/openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v] creating logical port openshift-marketplace_867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v for pod on switch ip-10-0-2-195.ec2.internal I0327 23:06:15.699024 3901 kube.go:256] Updating pod openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v I0327 23:06:15.711881 3901 pod.go:62] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v] pod update took 12.88403ms I0327 23:06:15.711910 3901 base_network_controller_pods.go:937] [default/openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v] addLogicalPort annotation time took 12.918819ms I0327 23:06:15.713434 3901 pods.go:271] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v] addLogicalPort took 14.573053ms, libovsdb time 1.084293ms I0327 23:06:16.931669 3901 cni.go:403] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 555e23136c2b4b91870875542a201f86ef5dfd2945b45024dfce3e973363c739 network default NAD default] DEL starting CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 555e23136c2b4b91870875542a201f86ef5dfd2945b45024dfce3e973363c739 network default NAD default] I0327 23:06:17.006872 3901 cni.go:424] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 555e23136c2b4b91870875542a201f86ef5dfd2945b45024dfce3e973363c739 network default NAD default] DEL finished CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 555e23136c2b4b91870875542a201f86ef5dfd2945b45024dfce3e973363c739 network default NAD default], result "{}", err I0327 23:06:17.112632 3901 cni.go:403] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v e21aac6b73a2ec0e58d562b8369d7395fab49b99f1b25d8715ce5b914b9ab09b network default NAD default] ADD starting CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v e21aac6b73a2ec0e58d562b8369d7395fab49b99f1b25d8715ce5b914b9ab09b network default NAD default] I0327 23:06:17.117881 3901 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:17.117947 3901 pods.go:176] Deleting pod: openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 I0327 23:06:17.119691 3901 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7, ips: 10.129.0.24 I0327 23:06:17.119727 3901 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:17.119751 3901 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:17.119760 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:06:17.120002 3901 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:17.120023 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:06:17.123942 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v, hostIfaceName: e21aac6b73a2ec0, network: default, NAD default, SandboxID: "e21aac6b73a2ec0e58d562b8369d7395fab49b99f1b25d8715ce5b914b9ab09b", PCI device ID: , UID: "d7788a33-49dc-4c57-9a15-e9c0d9222ad7", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0327 23:06:17.173818 3901 cni.go:424] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v e21aac6b73a2ec0e58d562b8369d7395fab49b99f1b25d8715ce5b914b9ab09b network default NAD default] ADD finished CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v e21aac6b73a2ec0e58d562b8369d7395fab49b99f1b25d8715ce5b914b9ab09b network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:73:21:19:1e:63\",\"name\":\"e21aac6b73a2ec0\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/835fe9c8-6952-48ba-9e36-78e2797c02b1\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:06:17.510056 3901 base_network_controller_pods.go:487] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz] creating logical port openshift-marketplace_3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz for pod on switch ip-10-0-2-195.ec2.internal I0327 23:06:17.510176 3901 kube.go:256] Updating pod openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz I0327 23:06:17.524250 3901 pod.go:62] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz] pod update took 14.112947ms I0327 23:06:17.524278 3901 base_network_controller_pods.go:937] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz] addLogicalPort annotation time took 14.14867ms I0327 23:06:17.525576 3901 pods.go:271] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz] addLogicalPort took 15.53621ms, libovsdb time 939.232µs I0327 23:06:18.141686 3901 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:18.141715 3901 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:18.141732 3901 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:18.141759 3901 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:19.161402 3901 base_network_controller_pods.go:487] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg] creating logical port openshift-marketplace_30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg for pod on switch ip-10-0-2-195.ec2.internal I0327 23:06:19.161537 3901 kube.go:256] Updating pod openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg I0327 23:06:19.174724 3901 pod.go:62] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg] pod update took 13.223647ms I0327 23:06:19.174789 3901 base_network_controller_pods.go:937] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg] addLogicalPort annotation time took 13.293612ms I0327 23:06:19.176114 3901 pods.go:271] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg] addLogicalPort took 14.735196ms, libovsdb time 994.614µs I0327 23:06:19.270483 3901 base_network_controller_pods.go:487] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg] creating logical port openshift-marketplace_18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg for pod on switch ip-10-0-2-195.ec2.internal I0327 23:06:19.270601 3901 kube.go:256] Updating pod openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg I0327 23:06:19.283696 3901 pod.go:62] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg] pod update took 13.120185ms I0327 23:06:19.283784 3901 base_network_controller_pods.go:937] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg] addLogicalPort annotation time took 13.213044ms I0327 23:06:19.285320 3901 pods.go:271] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg] addLogicalPort took 14.862068ms, libovsdb time 1.036982ms I0327 23:06:19.688252 3901 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg b4bfa0fa6f64fbee938f14f9c76b2dbc89e5936b7ff55a25f370ce5fdf385c54 network default NAD default] ADD starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg b4bfa0fa6f64fbee938f14f9c76b2dbc89e5936b7ff55a25f370ce5fdf385c54 network default NAD default] I0327 23:06:19.688287 3901 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz fda7fe4a206f850a8f90243f8e92d70997efae418e751758b049b41eba3aa94b network default NAD default] ADD starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz fda7fe4a206f850a8f90243f8e92d70997efae418e751758b049b41eba3aa94b network default NAD default] I0327 23:06:19.690149 3901 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg 17f82325df36b9f3a49095bff79813a313876602e32c7bb861af0897cd1013fc network default NAD default] ADD starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg 17f82325df36b9f3a49095bff79813a313876602e32c7bb861af0897cd1013fc network default NAD default] I0327 23:06:19.703191 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg, hostIfaceName: b4bfa0fa6f64fbe, network: default, NAD default, SandboxID: "b4bfa0fa6f64fbee938f14f9c76b2dbc89e5936b7ff55a25f370ce5fdf385c54", PCI device ID: , UID: "21e6b1f3-5f48-46df-ab1f-13c4fb47d289", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0327 23:06:19.703377 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg, hostIfaceName: 17f82325df36b9f, network: default, NAD default, SandboxID: "17f82325df36b9f3a49095bff79813a313876602e32c7bb861af0897cd1013fc", PCI device ID: , UID: "91d9530c-4e0b-4738-8dc8-a0473e2b736c", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0327 23:06:19.708566 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz, hostIfaceName: fda7fe4a206f850, network: default, NAD default, SandboxID: "fda7fe4a206f850a8f90243f8e92d70997efae418e751758b049b41eba3aa94b", PCI device ID: , UID: "e3d75d68-fb42-4521-a149-ac096c9d65b8", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0327 23:06:19.752146 3901 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg b4bfa0fa6f64fbee938f14f9c76b2dbc89e5936b7ff55a25f370ce5fdf385c54 network default NAD default] ADD finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg b4bfa0fa6f64fbee938f14f9c76b2dbc89e5936b7ff55a25f370ce5fdf385c54 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:62:a9:70:1f:b3\",\"name\":\"b4bfa0fa6f64fbe\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/865ceaad-aafb-4280-bc64-783402633477\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:06:19.770200 3901 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg 17f82325df36b9f3a49095bff79813a313876602e32c7bb861af0897cd1013fc network default NAD default] ADD finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg 17f82325df36b9f3a49095bff79813a313876602e32c7bb861af0897cd1013fc network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:40:ea:ef:e5:e8\",\"name\":\"17f82325df36b9f\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/75004bf1-b908-477c-944b-33ea057d47d8\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:06:19.790326 3901 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz fda7fe4a206f850a8f90243f8e92d70997efae418e751758b049b41eba3aa94b network default NAD default] ADD finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz fda7fe4a206f850a8f90243f8e92d70997efae418e751758b049b41eba3aa94b network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:c1:1f:fe:a0:31\",\"name\":\"fda7fe4a206f850\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/68cf5f4d-7d90-47c5-9077-daf522d414ee\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:06:20.969259 3901 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 581f20a2b34694fd70216f2ce44a7a64f5d430ae99dcd70b68b9e7da3aa1e770 network default NAD default] DEL starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 581f20a2b34694fd70216f2ce44a7a64f5d430ae99dcd70b68b9e7da3aa1e770 network default NAD default] I0327 23:06:21.040046 3901 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 581f20a2b34694fd70216f2ce44a7a64f5d430ae99dcd70b68b9e7da3aa1e770 network default NAD default] DEL finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 581f20a2b34694fd70216f2ce44a7a64f5d430ae99dcd70b68b9e7da3aa1e770 network default NAD default], result "{}", err I0327 23:06:21.075478 3901 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:21.075517 3901 pods.go:176] Deleting pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 I0327 23:06:21.076961 3901 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9, ips: 10.129.0.23 I0327 23:06:21.076999 3901 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:21.077022 3901 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:21.077031 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:06:21.077240 3901 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:21.077256 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:06:22.099563 3901 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:22.099592 3901 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:22.099613 3901 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:22.099628 3901 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:22.382612 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0327 23:06:22.992686 3901 cni.go:403] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v e21aac6b73a2ec0e58d562b8369d7395fab49b99f1b25d8715ce5b914b9ab09b network default NAD default] DEL starting CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v e21aac6b73a2ec0e58d562b8369d7395fab49b99f1b25d8715ce5b914b9ab09b network default NAD default] I0327 23:06:23.057745 3901 cni.go:424] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v e21aac6b73a2ec0e58d562b8369d7395fab49b99f1b25d8715ce5b914b9ab09b network default NAD default] DEL finished CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v e21aac6b73a2ec0e58d562b8369d7395fab49b99f1b25d8715ce5b914b9ab09b network default NAD default], result "{}", err I0327 23:06:23.106447 3901 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:23.106506 3901 pods.go:176] Deleting pod: openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v I0327 23:06:23.108007 3901 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v, ips: 10.129.0.25 I0327 23:06:23.108044 3901 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:23.108079 3901 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:23.108092 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:06:23.108298 3901 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:23.108313 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:06:24.005957 3901 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg 17f82325df36b9f3a49095bff79813a313876602e32c7bb861af0897cd1013fc network default NAD default] DEL starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg 17f82325df36b9f3a49095bff79813a313876602e32c7bb861af0897cd1013fc network default NAD default] I0327 23:06:24.006555 3901 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz fda7fe4a206f850a8f90243f8e92d70997efae418e751758b049b41eba3aa94b network default NAD default] DEL starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz fda7fe4a206f850a8f90243f8e92d70997efae418e751758b049b41eba3aa94b network default NAD default] I0327 23:06:24.074403 3901 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg 17f82325df36b9f3a49095bff79813a313876602e32c7bb861af0897cd1013fc network default NAD default] DEL finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg 17f82325df36b9f3a49095bff79813a313876602e32c7bb861af0897cd1013fc network default NAD default], result "{}", err I0327 23:06:24.103459 3901 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz fda7fe4a206f850a8f90243f8e92d70997efae418e751758b049b41eba3aa94b network default NAD default] DEL finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz fda7fe4a206f850a8f90243f8e92d70997efae418e751758b049b41eba3aa94b network default NAD default], result "{}", err I0327 23:06:24.120871 3901 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.120904 3901 pods.go:176] Deleting pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg I0327 23:06:24.122365 3901 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg, ips: 10.129.0.28 I0327 23:06:24.122394 3901 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.122413 3901 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.122421 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:06:24.122670 3901 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.122686 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:06:24.135763 3901 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:24.135786 3901 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:24.135806 3901 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:24.135820 3901 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:24.148205 3901 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.148232 3901 pods.go:176] Deleting pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz I0327 23:06:24.149407 3901 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz, ips: 10.129.0.26 I0327 23:06:24.149446 3901 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.149462 3901 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.149481 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:06:24.149680 3901 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.149694 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:06:25.008236 3901 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg b4bfa0fa6f64fbee938f14f9c76b2dbc89e5936b7ff55a25f370ce5fdf385c54 network default NAD default] DEL starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg b4bfa0fa6f64fbee938f14f9c76b2dbc89e5936b7ff55a25f370ce5fdf385c54 network default NAD default] I0327 23:06:25.074385 3901 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg b4bfa0fa6f64fbee938f14f9c76b2dbc89e5936b7ff55a25f370ce5fdf385c54 network default NAD default] DEL finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg b4bfa0fa6f64fbee938f14f9c76b2dbc89e5936b7ff55a25f370ce5fdf385c54 network default NAD default], result "{}", err I0327 23:06:25.124985 3901 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:25.125016 3901 pods.go:176] Deleting pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg I0327 23:06:25.126441 3901 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg, ips: 10.129.0.27 I0327 23:06:25.126486 3901 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:25.126506 3901 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:25.126516 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:06:25.128039 3901 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:25.128170 3901 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:06:25.147493 3901 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.147532 3901 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.147543 3901 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.147552 3901 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.180897 3901 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.180915 3901 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.180924 3901 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.180932 3901 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:26.147860 3901 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:26.147888 3901 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:26.147902 3901 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:26.147912 3901 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0327 23:06:28.826158 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0327 23:06:29.169954 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0327 23:06:29.178062 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 23:06:37.786343 3901 namespace.go:98] [cert-manager] adding namespace I0327 23:06:37.787306 3901 namespace.go:102] [cert-manager] adding namespace took 936.779µs I0327 23:06:37.808037 3901 namespace.go:142] [cert-manager] updating namespace I0327 23:06:37.822486 3901 namespace.go:142] [cert-manager] updating namespace I0327 23:06:43.481516 3901 namespace.go:142] [openshift-operators] updating namespace I0327 23:06:43.500612 3901 namespace.go:142] [openshift-operators] updating namespace W0327 23:06:45.928788 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 23:06:47.536851 3901 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0327 23:06:47.536875 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook I0327 23:06:53.191035 3901 namespace.go:98] [trusted-artifact-signer] adding namespace I0327 23:06:53.191882 3901 namespace.go:102] [trusted-artifact-signer] adding namespace took 823.436µs I0327 23:06:53.206681 3901 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:06:53.217584 3901 namespace.go:142] [trusted-artifact-signer] updating namespace W0327 23:06:53.985752 3901 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 23:06:56.258334 3901 namespace.go:142] [openshift-operators] updating namespace I0327 23:07:06.299720 3901 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=164 I0327 23:07:12.151828 3901 namespace.go:98] [openshift-pipelines] adding namespace I0327 23:07:12.152784 3901 namespace.go:102] [openshift-pipelines] adding namespace took 935.604µs I0327 23:07:12.170161 3901 namespace.go:142] [openshift-pipelines] updating namespace I0327 23:07:12.855070 3901 namespace.go:142] [cert-manager] updating namespace I0327 23:07:13.056457 3901 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:07:13.457685 3901 namespace.go:142] [konflux-operator] updating namespace I0327 23:07:13.657196 3901 namespace.go:142] [konflux-ui] updating namespace I0327 23:07:13.856600 3901 namespace.go:142] [open-cluster-management-2pabjk1o0k4lirtjul81bm4bi3nfk4gj] updating namespace I0327 23:07:14.261139 3901 namespace.go:142] [rhbk-operator] updating namespace I0327 23:07:14.462601 3901 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:07:14.663340 3901 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:07:14.864185 3901 namespace.go:142] [tsf] updating namespace I0327 23:07:15.065238 3901 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:07:15.265766 3901 namespace.go:142] [tsf-tas] updating namespace I0327 23:07:15.468373 3901 namespace.go:142] [tsf-tpa] updating namespace I0327 23:07:15.606523 3901 namespace.go:142] [cert-manager] updating namespace I0327 23:07:15.616520 3901 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:07:15.624464 3901 namespace.go:142] [default] updating namespace I0327 23:07:15.633933 3901 namespace.go:142] [konflux-operator] updating namespace I0327 23:07:15.642555 3901 namespace.go:142] [konflux-ui] updating namespace I0327 23:07:15.651808 3901 namespace.go:142] [open-cluster-management-2pabjk1o0k4lirtjul81bm4bi3nfk4gj] updating namespace I0327 23:07:15.660830 3901 namespace.go:142] [openshift] updating namespace I0327 23:07:15.671571 3901 namespace.go:142] [rhbk-operator] updating namespace I0327 23:07:15.679683 3901 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:07:15.689995 3901 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:07:15.698841 3901 namespace.go:142] [tsf] updating namespace I0327 23:07:15.710186 3901 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:07:15.731257 3901 namespace.go:142] [tsf-tas] updating namespace I0327 23:07:15.749154 3901 namespace.go:142] [tsf-tpa] updating namespace I0327 23:07:15.851938 3901 namespace.go:142] [cert-manager] updating namespace I0327 23:07:15.950575 3901 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:07:16.050188 3901 namespace.go:142] [default] updating namespace I0327 23:07:16.156629 3901 namespace.go:142] [konflux-operator] updating namespace I0327 23:07:16.250568 3901 namespace.go:142] [konflux-ui] updating namespace I0327 23:07:16.355185 3901 namespace.go:142] [open-cluster-management-2pabjk1o0k4lirtjul81bm4bi3nfk4gj] updating namespace I0327 23:07:16.450281 3901 namespace.go:142] [openshift] updating namespace I0327 23:07:16.550169 3901 namespace.go:142] [rhbk-operator] updating namespace I0327 23:07:16.651856 3901 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:07:16.752218 3901 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:07:16.849817 3901 namespace.go:142] [tsf] updating namespace I0327 23:07:16.951679 3901 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:07:17.052411 3901 namespace.go:142] [tsf-tas] updating namespace I0327 23:07:17.152303 3901 namespace.go:142] [tsf-tpa] updating namespace I0327 23:07:17.448839 3901 namespace.go:142] [openshift-pipelines] updating namespace I0327 23:07:23.291251 3901 cni.go:403] [tsf/test-tsf-subscriptions 7246647b4080cb387c201ee15913970369d754c2c1b9e89a9ce5145d3f2af847 network default NAD default] DEL starting CNI request [tsf/test-tsf-subscriptions 7246647b4080cb387c201ee15913970369d754c2c1b9e89a9ce5145d3f2af847 network default NAD default] I0327 23:07:23.359583 3901 cni.go:424] [tsf/test-tsf-subscriptions 7246647b4080cb387c201ee15913970369d754c2c1b9e89a9ce5145d3f2af847 network default NAD default] DEL finished CNI request [tsf/test-tsf-subscriptions 7246647b4080cb387c201ee15913970369d754c2c1b9e89a9ce5145d3f2af847 network default NAD default], result "{}", err I0327 23:07:23.409572 3901 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:07:23.409606 3901 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0327 23:07:23.410898 3901 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.129.0.22 I0327 23:07:23.410925 3901 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:07:23.430960 3901 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:07:23.430976 3901 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:07:23.436165 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0327 23:07:23.436184 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0327 23:07:24.749701 3901 base_network_controller_pods.go:487] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-2-195.ec2.internal I0327 23:07:24.749790 3901 kube.go:256] Updating pod tsf/test-tsf-infrastructure I0327 23:07:24.764578 3901 pod.go:62] [tsf/test-tsf-infrastructure] pod update took 14.814139ms I0327 23:07:24.764596 3901 base_network_controller_pods.go:937] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 14.838217ms I0327 23:07:24.765770 3901 pods.go:271] [tsf/test-tsf-infrastructure] addLogicalPort took 16.084073ms, libovsdb time 846.667µs I0327 23:07:25.131038 3901 cni.go:403] [tsf/test-tsf-infrastructure 9b04a49d219dd82a55ece8b30d2b727f88891b2b811a873cff283345e8239374 network default NAD default] ADD starting CNI request [tsf/test-tsf-infrastructure 9b04a49d219dd82a55ece8b30d2b727f88891b2b811a873cff283345e8239374 network default NAD default] I0327 23:07:25.139678 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: 9b04a49d219dd82, network: default, NAD default, SandboxID: "9b04a49d219dd82a55ece8b30d2b727f88891b2b811a873cff283345e8239374", PCI device ID: , UID: "e89c03e3-431d-48f7-8dd5-8fef493c7d67", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0327 23:07:25.180960 3901 cni.go:424] [tsf/test-tsf-infrastructure 9b04a49d219dd82a55ece8b30d2b727f88891b2b811a873cff283345e8239374 network default NAD default] ADD finished CNI request [tsf/test-tsf-infrastructure 9b04a49d219dd82a55ece8b30d2b727f88891b2b811a873cff283345e8239374 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:cc:21:11:63:44\",\"name\":\"9b04a49d219dd82\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/131a98ad-a460-45f9-8c93-1a128459b5e1\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:07:25.404428 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0327 23:07:27.646424 3901 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-8mmg4] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-8mmg4 for pod on switch ip-10-0-2-195.ec2.internal I0327 23:07:27.646528 3901 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-8mmg4 I0327 23:07:27.655158 3901 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-g6mdm] creating logical port tsf-keycloak_keycloak-pgsql-bee-744f6fc4d9-g6mdm for pod on switch ip-10-0-2-195.ec2.internal I0327 23:07:27.655263 3901 kube.go:256] Updating pod tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-g6mdm I0327 23:07:27.660664 3901 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-8mmg4] pod update took 14.160586ms I0327 23:07:27.660685 3901 base_network_controller_pods.go:937] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-8mmg4] addLogicalPort annotation time took 14.186856ms I0327 23:07:27.662127 3901 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-8mmg4] addLogicalPort took 15.723666ms, libovsdb time 1.075913ms I0327 23:07:27.670005 3901 pod.go:62] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-g6mdm] pod update took 14.773807ms I0327 23:07:27.670075 3901 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-g6mdm] addLogicalPort annotation time took 14.846309ms I0327 23:07:27.671340 3901 pods.go:271] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-g6mdm] addLogicalPort took 16.198281ms, libovsdb time 864.947µs W0327 23:07:30.473814 3901 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:07:30.473832 3901 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:07:30.473837 3901 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:07:30.473842 3901 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:07:31.233625 3901 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 23:07:31.233653 3901 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 23:07:31.233660 3901 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 23:07:31.239998 3901 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 23:07:31.240008 3901 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 23:07:31.240012 3901 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0327 23:07:36.297876 3901 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=107 I0327 23:07:37.930443 3901 cni.go:403] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-g6mdm c2b4843546b6162ff6ad038df062d1a22e64b5a6b60cd378b19b105d7ae0a90b network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-g6mdm c2b4843546b6162ff6ad038df062d1a22e64b5a6b60cd378b19b105d7ae0a90b network default NAD default] I0327 23:07:37.930545 3901 cni.go:403] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-8mmg4 f126b93abbb8406e5b80652ee97220e8c57252706bf7c4f730faeaa766d937b6 network default NAD default] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-8mmg4 f126b93abbb8406e5b80652ee97220e8c57252706bf7c4f730faeaa766d937b6 network default NAD default] I0327 23:07:37.939803 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-pgsql-bee-744f6fc4d9-g6mdm, hostIfaceName: c2b4843546b6162, network: default, NAD default, SandboxID: "c2b4843546b6162ff6ad038df062d1a22e64b5a6b60cd378b19b105d7ae0a90b", PCI device ID: , UID: "ca258df0-0d0b-4b45-8b96-3726a85917a5", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0327 23:07:37.939892 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-8mmg4, hostIfaceName: f126b93abbb8406, network: default, NAD default, SandboxID: "f126b93abbb8406e5b80652ee97220e8c57252706bf7c4f730faeaa766d937b6", PCI device ID: , UID: "0e02ed14-a2cd-4d45-8944-dab31b2a209f", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0327 23:07:37.982786 3901 cni.go:424] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-g6mdm c2b4843546b6162ff6ad038df062d1a22e64b5a6b60cd378b19b105d7ae0a90b network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-g6mdm c2b4843546b6162ff6ad038df062d1a22e64b5a6b60cd378b19b105d7ae0a90b network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:f9:28:98:cf:83\",\"name\":\"c2b4843546b6162\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/be5bfa65-8460-4828-a149-c7131303c479\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:07:38.000318 3901 cni.go:424] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-8mmg4 f126b93abbb8406e5b80652ee97220e8c57252706bf7c4f730faeaa766d937b6 network default NAD default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-8mmg4 f126b93abbb8406e5b80652ee97220e8c57252706bf7c4f730faeaa766d937b6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:bd:df:d9:35:2f\",\"name\":\"f126b93abbb8406\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/866da952-178f-48fe-96a8-1748b22569f9\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:07:38.370336 3901 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0327 23:07:38.394622 3901 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0327 23:07:53.302082 3901 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-hpjtn] creating logical port openshift-pipelines_tekton-triggers-controller-68564d75ff-hpjtn for pod on switch ip-10-0-2-195.ec2.internal I0327 23:07:53.302194 3901 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-68564d75ff-hpjtn I0327 23:07:53.320447 3901 pod.go:62] [openshift-pipelines/tekton-triggers-controller-68564d75ff-hpjtn] pod update took 18.282791ms I0327 23:07:53.320482 3901 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-hpjtn] addLogicalPort annotation time took 18.311885ms I0327 23:07:53.321670 3901 pods.go:271] [openshift-pipelines/tekton-triggers-controller-68564d75ff-hpjtn] addLogicalPort took 19.610422ms, libovsdb time 884.319µs I0327 23:07:53.464355 3901 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-sx9z7] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-8bb674894-sx9z7 for pod on switch ip-10-0-2-195.ec2.internal I0327 23:07:53.464440 3901 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-sx9z7 I0327 23:07:53.477758 3901 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-sx9z7] pod update took 13.337974ms I0327 23:07:53.477777 3901 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-sx9z7] addLogicalPort annotation time took 13.36479ms I0327 23:07:53.479030 3901 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-sx9z7] addLogicalPort took 14.687257ms, libovsdb time 936.809µs I0327 23:07:53.661740 3901 cni.go:403] [openshift-pipelines/tekton-triggers-controller-68564d75ff-hpjtn 527d90cf045f1ff5dbc5887fbe33a430e577e6d08bcd52ca36bdafecd994c0b8 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-hpjtn 527d90cf045f1ff5dbc5887fbe33a430e577e6d08bcd52ca36bdafecd994c0b8 network default NAD default] I0327 23:07:53.670776 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-68564d75ff-hpjtn, hostIfaceName: 527d90cf045f1ff, network: default, NAD default, SandboxID: "527d90cf045f1ff5dbc5887fbe33a430e577e6d08bcd52ca36bdafecd994c0b8", PCI device ID: , UID: "df7dc349-8fbc-47fe-acea-5f836bb47139", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0327 23:07:53.716008 3901 cni.go:424] [openshift-pipelines/tekton-triggers-controller-68564d75ff-hpjtn 527d90cf045f1ff5dbc5887fbe33a430e577e6d08bcd52ca36bdafecd994c0b8 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-hpjtn 527d90cf045f1ff5dbc5887fbe33a430e577e6d08bcd52ca36bdafecd994c0b8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:f2:3e:d8:0f:8a\",\"name\":\"527d90cf045f1ff\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2df33bd9-3555-43f8-9310-fb654f90d8f6\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:07:53.823941 3901 cni.go:403] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-sx9z7 20aacaca64a42f229e9d4d49cce61e9040192980d9fb7660ed848e6e786a4923 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-sx9z7 20aacaca64a42f229e9d4d49cce61e9040192980d9fb7660ed848e6e786a4923 network default NAD default] I0327 23:07:53.832517 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-8bb674894-sx9z7, hostIfaceName: 20aacaca64a42f2, network: default, NAD default, SandboxID: "20aacaca64a42f229e9d4d49cce61e9040192980d9fb7660ed848e6e786a4923", PCI device ID: , UID: "a9f9b9ae-7c0f-41e4-af0d-2a7ee880fb79", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0327 23:07:53.873852 3901 cni.go:424] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-sx9z7 20aacaca64a42f229e9d4d49cce61e9040192980d9fb7660ed848e6e786a4923 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-sx9z7 20aacaca64a42f229e9d4d49cce61e9040192980d9fb7660ed848e6e786a4923 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:ba:46:6f:78:36\",\"name\":\"20aacaca64a42f2\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3518c071-5701-4771-939f-c5d2d7f75875\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:07:57.489829 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0327 23:07:59.378059 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0327 23:07:59.396812 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 5432: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0327 23:08:02.500898 3901 cni.go:403] [tsf/test-tsf-infrastructure 9b04a49d219dd82a55ece8b30d2b727f88891b2b811a873cff283345e8239374 network default NAD default] DEL starting CNI request [tsf/test-tsf-infrastructure 9b04a49d219dd82a55ece8b30d2b727f88891b2b811a873cff283345e8239374 network default NAD default] I0327 23:08:02.597808 3901 cni.go:424] [tsf/test-tsf-infrastructure 9b04a49d219dd82a55ece8b30d2b727f88891b2b811a873cff283345e8239374 network default NAD default] DEL finished CNI request [tsf/test-tsf-infrastructure 9b04a49d219dd82a55ece8b30d2b727f88891b2b811a873cff283345e8239374 network default NAD default], result "{}", err I0327 23:08:02.645132 3901 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:08:02.645174 3901 pods.go:176] Deleting pod: tsf/test-tsf-infrastructure I0327 23:08:02.646375 3901 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.129.0.29 I0327 23:08:02.646398 3901 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:08:02.669852 3901 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:08:02.669871 3901 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:08:02.675727 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0327 23:08:02.675751 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0327 23:08:03.830401 3901 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-2-195.ec2.internal I0327 23:08:03.830539 3901 kube.go:256] Updating pod tsf/test-tsf-iam I0327 23:08:03.849058 3901 pod.go:62] [tsf/test-tsf-iam] pod update took 18.543642ms I0327 23:08:03.849077 3901 base_network_controller_pods.go:937] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 18.567726ms I0327 23:08:03.850186 3901 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 19.801313ms, libovsdb time 814.411µs I0327 23:08:04.198499 3901 cni.go:403] [tsf/test-tsf-iam e3cd40d68338578f4a9f5279d2886a0432cebffe2b3e4ed1bf2badfa77981ba1 network default NAD default] ADD starting CNI request [tsf/test-tsf-iam e3cd40d68338578f4a9f5279d2886a0432cebffe2b3e4ed1bf2badfa77981ba1 network default NAD default] I0327 23:08:04.206680 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: e3cd40d68338578, network: default, NAD default, SandboxID: "e3cd40d68338578f4a9f5279d2886a0432cebffe2b3e4ed1bf2badfa77981ba1", PCI device ID: , UID: "213cfd9f-1601-470b-91b1-1d02176a0327", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0327 23:08:04.248606 3901 cni.go:424] [tsf/test-tsf-iam e3cd40d68338578f4a9f5279d2886a0432cebffe2b3e4ed1bf2badfa77981ba1 network default NAD default] ADD finished CNI request [tsf/test-tsf-iam e3cd40d68338578f4a9f5279d2886a0432cebffe2b3e4ed1bf2badfa77981ba1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:cf:7c:fe:8d:f4\",\"name\":\"e3cd40d68338578\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f091d388-89e2-4283-a9c9-ade6371df2b7\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:08:08.479939 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0327 23:08:15.386241 3901 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 23:08:18.461612 3901 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-2-195.ec2.internal I0327 23:08:18.461758 3901 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0327 23:08:18.474156 3901 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 12.421883ms I0327 23:08:18.474178 3901 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 12.450807ms I0327 23:08:18.475354 3901 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 13.761943ms, libovsdb time 821.721µs I0327 23:08:23.298638 3901 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=100 W0327 23:08:24.417366 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:08:24.417389 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:08:24.419351 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 23:08:27.821054 3901 cni.go:403] [openshift-pipelines/tekton-results-postgres-0 ca6bf41673220440f65fddd3600db14bd7f2447c57091d184772d6888e288e80 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 ca6bf41673220440f65fddd3600db14bd7f2447c57091d184772d6888e288e80 network default NAD default] I0327 23:08:27.829387 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: ca6bf4167322044, network: default, NAD default, SandboxID: "ca6bf41673220440f65fddd3600db14bd7f2447c57091d184772d6888e288e80", PCI device ID: , UID: "557754db-6afa-4af9-9abb-16d927955193", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0327 23:08:27.872837 3901 cni.go:424] [openshift-pipelines/tekton-results-postgres-0 ca6bf41673220440f65fddd3600db14bd7f2447c57091d184772d6888e288e80 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 ca6bf41673220440f65fddd3600db14bd7f2447c57091d184772d6888e288e80 network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:2b:3e:fe:0e:7d\",\"name\":\"ca6bf4167322044\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1d3eca7b-50a5-4fca-98c2-85332426aaf9\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:08:33.695383 3901 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 23:08:35.038997 3901 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-gpnxt] creating logical port openshift-pipelines_tekton-results-api-7b4b7bb78c-gpnxt for pod on switch ip-10-0-2-195.ec2.internal I0327 23:08:35.039091 3901 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-7b4b7bb78c-gpnxt I0327 23:08:35.053654 3901 pod.go:62] [openshift-pipelines/tekton-results-api-7b4b7bb78c-gpnxt] pod update took 14.580132ms I0327 23:08:35.053681 3901 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-gpnxt] addLogicalPort annotation time took 14.617127ms I0327 23:08:35.055061 3901 pods.go:271] [openshift-pipelines/tekton-results-api-7b4b7bb78c-gpnxt] addLogicalPort took 16.087771ms, libovsdb time 909.281µs I0327 23:08:35.408459 3901 cni.go:403] [openshift-pipelines/tekton-results-api-7b4b7bb78c-gpnxt 6d8f817110054c947058198dcc3ddaa54b90a83e78d7160aae6f995e3ba9a3e7 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-gpnxt 6d8f817110054c947058198dcc3ddaa54b90a83e78d7160aae6f995e3ba9a3e7 network default NAD default] I0327 23:08:35.417342 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-7b4b7bb78c-gpnxt, hostIfaceName: 6d8f817110054c9, network: default, NAD default, SandboxID: "6d8f817110054c947058198dcc3ddaa54b90a83e78d7160aae6f995e3ba9a3e7", PCI device ID: , UID: "2fa4f6e4-2ea8-4224-b42a-5d0a83eb55b3", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0327 23:08:35.463632 3901 cni.go:424] [openshift-pipelines/tekton-results-api-7b4b7bb78c-gpnxt 6d8f817110054c947058198dcc3ddaa54b90a83e78d7160aae6f995e3ba9a3e7 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-gpnxt 6d8f817110054c947058198dcc3ddaa54b90a83e78d7160aae6f995e3ba9a3e7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:52:6a:09:c7:6f\",\"name\":\"6d8f817110054c9\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0e74043b-5b4d-4392-82cb-624184ccfb94\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:08:43.711687 3901 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 23:08:43.711706 3901 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 23:08:43.711713 3901 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0327 23:08:46.792682 3901 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:08:46.793272 3901 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:08:47.556903 3901 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:08:47.556934 3901 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:08:47.562715 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 23:08:47.562731 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0327 23:08:49.577177 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:08:49.577199 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:08:49.577348 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 23:08:55.413291 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 23:08:55.413317 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 23:08:55.413324 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0327 23:09:01.459448 3901 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-nxwqx] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-nxwqx for pod on switch ip-10-0-2-195.ec2.internal I0327 23:09:01.459587 3901 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-nxwqx I0327 23:09:01.472534 3901 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-nxwqx] pod update took 12.982618ms I0327 23:09:01.472560 3901 base_network_controller_pods.go:937] [default/openshift-pipelines/tkn-cli-serve-784b566968-nxwqx] addLogicalPort annotation time took 13.013799ms I0327 23:09:01.473755 3901 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-nxwqx] addLogicalPort took 14.328565ms, libovsdb time 846.329µs I0327 23:09:01.818289 3901 cni.go:403] [openshift-pipelines/tkn-cli-serve-784b566968-nxwqx 1103633a0ed7d9544d1bd9e7e469c39df4818cc9341b29903bda2e5abd6c729d network default NAD default] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-nxwqx 1103633a0ed7d9544d1bd9e7e469c39df4818cc9341b29903bda2e5abd6c729d network default NAD default] I0327 23:09:01.826891 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-nxwqx, hostIfaceName: 1103633a0ed7d95, network: default, NAD default, SandboxID: "1103633a0ed7d9544d1bd9e7e469c39df4818cc9341b29903bda2e5abd6c729d", PCI device ID: , UID: "44480530-a26a-479a-9c8b-28731be21475", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0327 23:09:01.870305 3901 cni.go:424] [openshift-pipelines/tkn-cli-serve-784b566968-nxwqx 1103633a0ed7d9544d1bd9e7e469c39df4818cc9341b29903bda2e5abd6c729d network default NAD default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-nxwqx 1103633a0ed7d9544d1bd9e7e469c39df4818cc9341b29903bda2e5abd6c729d network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:ee:18:28:25:f4\",\"name\":\"1103633a0ed7d95\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/666e5bd6-c0cf-45d5-add1-ddd02f2170eb\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:09:04.832218 3901 cni.go:403] [tsf/test-tsf-iam e3cd40d68338578f4a9f5279d2886a0432cebffe2b3e4ed1bf2badfa77981ba1 network default NAD default] DEL starting CNI request [tsf/test-tsf-iam e3cd40d68338578f4a9f5279d2886a0432cebffe2b3e4ed1bf2badfa77981ba1 network default NAD default] I0327 23:09:04.904546 3901 cni.go:424] [tsf/test-tsf-iam e3cd40d68338578f4a9f5279d2886a0432cebffe2b3e4ed1bf2badfa77981ba1 network default NAD default] DEL finished CNI request [tsf/test-tsf-iam e3cd40d68338578f4a9f5279d2886a0432cebffe2b3e4ed1bf2badfa77981ba1 network default NAD default], result "{}", err I0327 23:09:05.482019 3901 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:09:05.482097 3901 pods.go:176] Deleting pod: tsf/test-tsf-iam I0327 23:09:05.483622 3901 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.129.0.34 I0327 23:09:05.483654 3901 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:09:22.904002 3901 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-gvqnc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:09:22.904548 3901 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-gvqnc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:09:23.931833 3901 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-gvqnc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:09:23.931855 3901 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-gvqnc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:09:24.273523 3901 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:09:24.274071 3901 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:09:24.784606 3901 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:09:24.784637 3901 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:09:24.790591 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 23:09:24.790614 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0327 23:09:25.713031 3901 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x9j7p] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-x9j7p for pod on switch ip-10-0-2-195.ec2.internal I0327 23:09:25.713123 3901 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x9j7p I0327 23:09:25.727564 3901 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x9j7p] pod update took 14.466565ms I0327 23:09:25.727585 3901 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x9j7p] addLogicalPort annotation time took 14.492547ms I0327 23:09:25.728798 3901 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x9j7p] addLogicalPort took 15.782152ms, libovsdb time 890.588µs I0327 23:09:25.811246 3901 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-5pd67] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-5pd67 for pod on switch ip-10-0-2-195.ec2.internal I0327 23:09:25.811323 3901 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-5pd67 I0327 23:09:25.824422 3901 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-5pd67] pod update took 13.119894ms I0327 23:09:25.824449 3901 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-5pd67] addLogicalPort annotation time took 13.150082ms I0327 23:09:25.825634 3901 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-5pd67] addLogicalPort took 14.401052ms, libovsdb time 790.804µs I0327 23:09:25.920596 3901 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-b5bfs] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-b5bfs for pod on switch ip-10-0-2-195.ec2.internal I0327 23:09:25.920676 3901 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-b5bfs I0327 23:09:25.936012 3901 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b5bfs] pod update took 15.352592ms I0327 23:09:25.936039 3901 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-b5bfs] addLogicalPort annotation time took 15.384289ms I0327 23:09:25.937771 3901 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b5bfs] addLogicalPort took 17.188163ms, libovsdb time 1.261504ms I0327 23:09:26.074782 3901 cni.go:403] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x9j7p 56c436ad8e780612f7dc6230c3301ebaf8b1bbbccf75d4a62ca083e556fe311d network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x9j7p 56c436ad8e780612f7dc6230c3301ebaf8b1bbbccf75d4a62ca083e556fe311d network default NAD default] I0327 23:09:26.082978 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-x9j7p, hostIfaceName: 56c436ad8e78061, network: default, NAD default, SandboxID: "56c436ad8e780612f7dc6230c3301ebaf8b1bbbccf75d4a62ca083e556fe311d", PCI device ID: , UID: "ecac24c4-7fc5-458c-b1ec-4f68356fed6c", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0327 23:09:26.126654 3901 cni.go:424] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x9j7p 56c436ad8e780612f7dc6230c3301ebaf8b1bbbccf75d4a62ca083e556fe311d network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x9j7p 56c436ad8e780612f7dc6230c3301ebaf8b1bbbccf75d4a62ca083e556fe311d network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:1a:2c:96:0e:39\",\"name\":\"56c436ad8e78061\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3001f6c8-7a5b-485f-a240-3cf4cd4e4b41\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:09:26.171252 3901 cni.go:403] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-5pd67 df7aa5feda59e9cbc27a83596d66ffef7a35ba1fcdd8e8e767f03aa7f1973b41 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-5pd67 df7aa5feda59e9cbc27a83596d66ffef7a35ba1fcdd8e8e767f03aa7f1973b41 network default NAD default] I0327 23:09:26.179779 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-5pd67, hostIfaceName: df7aa5feda59e9c, network: default, NAD default, SandboxID: "df7aa5feda59e9cbc27a83596d66ffef7a35ba1fcdd8e8e767f03aa7f1973b41", PCI device ID: , UID: "b9eb7fc4-49ec-479d-8dc9-35af23238099", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0327 23:09:26.221231 3901 cni.go:424] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-5pd67 df7aa5feda59e9cbc27a83596d66ffef7a35ba1fcdd8e8e767f03aa7f1973b41 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-5pd67 df7aa5feda59e9cbc27a83596d66ffef7a35ba1fcdd8e8e767f03aa7f1973b41 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:8d:2e:a5:c2:b9\",\"name\":\"df7aa5feda59e9c\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d574ad17-defe-4d93-8d3d-6d011b711468\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:09:26.284483 3901 cni.go:403] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b5bfs b33a4d4666166793fd7fbb2f0802c0d31f398512e1fe4d11dd30f77e4cad2628 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b5bfs b33a4d4666166793fd7fbb2f0802c0d31f398512e1fe4d11dd30f77e4cad2628 network default NAD default] I0327 23:09:26.293107 3901 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-b5bfs, hostIfaceName: b33a4d466616679, network: default, NAD default, SandboxID: "b33a4d4666166793fd7fbb2f0802c0d31f398512e1fe4d11dd30f77e4cad2628", PCI device ID: , UID: "ce6f9c07-b7c5-4946-b77c-23f46cb05d18", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0327 23:09:26.335839 3901 cni.go:424] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b5bfs b33a4d4666166793fd7fbb2f0802c0d31f398512e1fe4d11dd30f77e4cad2628 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b5bfs b33a4d4666166793fd7fbb2f0802c0d31f398512e1fe4d11dd30f77e4cad2628 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:c2:01:a3:e8:b7\",\"name\":\"b33a4d466616679\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2460231e-8be8-43ee-9885-e98a24fe8f89\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:09:26.810915 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:09:26.810945 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:09:26.816562 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 23:09:28.956822 3901 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0327 23:09:29.013753 3901 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 23:09:29.013769 3901 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 23:09:29.968858 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8082: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 23:09:29.968877 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller I0327 23:09:31.371329 3901 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-48.ec2.internal I0327 23:09:31.371364 3901 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-48.ec2.internal: 42.045µs I0327 23:09:31.371361 3901 egressfirewall.go:628] Syncing node "ip-10-0-0-48.ec2.internal" for egress firewall W0327 23:09:39.971598 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0327 23:10:04.337247 3901 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-zcpwn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:04.337972 3901 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-zcpwn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:05.221132 3901 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-zcpwn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:05.221155 3901 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-zcpwn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:05.598422 3901 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:05.598451 3901 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:05.606197 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 23:10:05.606223 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 23:10:05.720540 3901 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-2-195.ec2.internal I0327 23:10:05.720673 3901 kube.go:256] Updating pod tsf/test-tsf-iam I0327 23:10:05.735610 3901 pod.go:62] [tsf/test-tsf-iam] pod update took 14.962491ms I0327 23:10:05.735632 3901 base_network_controller_pods.go:937] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 14.987651ms I0327 23:10:05.736894 3901 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 16.369308ms, libovsdb time 927.433µs I0327 23:10:06.081329 3901 cni.go:403] [tsf/test-tsf-iam 34cbeb7aebe50a294c8385f6f6c90c4dde16ca4e53c4ac554900351ffb4832a5 network default NAD default] ADD starting CNI request [tsf/test-tsf-iam 34cbeb7aebe50a294c8385f6f6c90c4dde16ca4e53c4ac554900351ffb4832a5 network default NAD default] I0327 23:10:06.089705 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 34cbeb7aebe50a2, network: default, NAD default, SandboxID: "34cbeb7aebe50a294c8385f6f6c90c4dde16ca4e53c4ac554900351ffb4832a5", PCI device ID: , UID: "37fe6ed7-ca10-4e82-a10c-d65f10e2e2d2", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0327 23:10:06.134253 3901 cni.go:424] [tsf/test-tsf-iam 34cbeb7aebe50a294c8385f6f6c90c4dde16ca4e53c4ac554900351ffb4832a5 network default NAD default] ADD finished CNI request [tsf/test-tsf-iam 34cbeb7aebe50a294c8385f6f6c90c4dde16ca4e53c4ac554900351ffb4832a5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:89:0f:fe:80:90\",\"name\":\"34cbeb7aebe50a2\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/811d2808-1251-48ba-9bf6-9e3a74f4c4b0\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:10:10.381815 3901 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0327 23:10:14.204741 3901 cni.go:403] [tsf/test-tsf-iam 34cbeb7aebe50a294c8385f6f6c90c4dde16ca4e53c4ac554900351ffb4832a5 network default NAD default] DEL starting CNI request [tsf/test-tsf-iam 34cbeb7aebe50a294c8385f6f6c90c4dde16ca4e53c4ac554900351ffb4832a5 network default NAD default] I0327 23:10:14.284109 3901 cni.go:424] [tsf/test-tsf-iam 34cbeb7aebe50a294c8385f6f6c90c4dde16ca4e53c4ac554900351ffb4832a5 network default NAD default] DEL finished CNI request [tsf/test-tsf-iam 34cbeb7aebe50a294c8385f6f6c90c4dde16ca4e53c4ac554900351ffb4832a5 network default NAD default], result "{}", err I0327 23:10:14.334894 3901 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:14.334921 3901 pods.go:176] Deleting pod: tsf/test-tsf-iam I0327 23:10:14.336278 3901 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.129.0.41 I0327 23:10:14.336308 3901 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:14.355380 3901 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:14.355402 3901 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:14.365227 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 23:10:14.365251 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 23:10:15.175781 3901 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-2-195.ec2.internal I0327 23:10:15.175875 3901 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0327 23:10:15.193310 3901 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 17.463469ms I0327 23:10:15.193330 3901 base_network_controller_pods.go:937] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 17.489157ms I0327 23:10:15.194530 3901 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 18.765945ms, libovsdb time 821.861µs I0327 23:10:15.532369 3901 cni.go:403] [tsf-tas/test-tsf-tas f7b6b0b255b34fd877ac3c895756cbbb54e20340cf79f043e18b375e7901263f network default NAD default] ADD starting CNI request [tsf-tas/test-tsf-tas f7b6b0b255b34fd877ac3c895756cbbb54e20340cf79f043e18b375e7901263f network default NAD default] I0327 23:10:15.541142 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: f7b6b0b255b34fd, network: default, NAD default, SandboxID: "f7b6b0b255b34fd877ac3c895756cbbb54e20340cf79f043e18b375e7901263f", PCI device ID: , UID: "0725a3f7-83df-479b-9f4e-6c36bd4b0755", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0327 23:10:15.583531 3901 cni.go:424] [tsf-tas/test-tsf-tas f7b6b0b255b34fd877ac3c895756cbbb54e20340cf79f043e18b375e7901263f network default NAD default] ADD finished CNI request [tsf-tas/test-tsf-tas f7b6b0b255b34fd877ac3c895756cbbb54e20340cf79f043e18b375e7901263f network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:11:3f:28:13:92\",\"name\":\"f7b6b0b255b34fd\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cb309186-9269-4660-ac21-7365af5dc2bc\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:10:17.608946 3901 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-wpwm6-x5grc] creating logical port tsf-tas_rekor-createtree-job-wpwm6-x5grc for pod on switch ip-10-0-2-195.ec2.internal I0327 23:10:17.609031 3901 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-wpwm6-x5grc I0327 23:10:17.622393 3901 pod.go:62] [tsf-tas/rekor-createtree-job-wpwm6-x5grc] pod update took 13.380226ms I0327 23:10:17.622413 3901 base_network_controller_pods.go:937] [default/tsf-tas/rekor-createtree-job-wpwm6-x5grc] addLogicalPort annotation time took 13.405968ms I0327 23:10:17.623658 3901 pods.go:271] [tsf-tas/rekor-createtree-job-wpwm6-x5grc] addLogicalPort took 14.724823ms, libovsdb time 875.733µs I0327 23:10:17.969162 3901 cni.go:403] [tsf-tas/rekor-createtree-job-wpwm6-x5grc 51be6e0b23708ee47bdf3d4a47ac70ee856e348b7bfb1aad340b58ac1d59a628 network default NAD default] ADD starting CNI request [tsf-tas/rekor-createtree-job-wpwm6-x5grc 51be6e0b23708ee47bdf3d4a47ac70ee856e348b7bfb1aad340b58ac1d59a628 network default NAD default] I0327 23:10:17.977350 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-wpwm6-x5grc, hostIfaceName: 51be6e0b23708ee, network: default, NAD default, SandboxID: "51be6e0b23708ee47bdf3d4a47ac70ee856e348b7bfb1aad340b58ac1d59a628", PCI device ID: , UID: "c2aedf45-c8a5-4983-a761-0c9d787605e5", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0327 23:10:18.019945 3901 cni.go:424] [tsf-tas/rekor-createtree-job-wpwm6-x5grc 51be6e0b23708ee47bdf3d4a47ac70ee856e348b7bfb1aad340b58ac1d59a628 network default NAD default] ADD finished CNI request [tsf-tas/rekor-createtree-job-wpwm6-x5grc 51be6e0b23708ee47bdf3d4a47ac70ee856e348b7bfb1aad340b58ac1d59a628 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:9f:bc:e7:56:22\",\"name\":\"51be6e0b23708ee\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/002389f3-bf12-4772-a82f-5619af79fc8e\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:10:22.112603 3901 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0327 23:10:22.112622 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0327 23:10:25.316671 3901 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0327 23:10:28.154186 3901 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0327 23:10:32.351240 3901 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0327 23:10:36.420264 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0327 23:10:41.374319 3901 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-vvwth-lxlpn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:41.374895 3901 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-vvwth-lxlpn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:42.239182 3901 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-vvwth-lxlpn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:42.239207 3901 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-vvwth-lxlpn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:57.387311 3901 namespace.go:142] [openshift-pipelines] updating namespace I0327 23:11:04.471434 3901 cni.go:403] [tsf-tas/test-tsf-tas f7b6b0b255b34fd877ac3c895756cbbb54e20340cf79f043e18b375e7901263f network default NAD default] DEL starting CNI request [tsf-tas/test-tsf-tas f7b6b0b255b34fd877ac3c895756cbbb54e20340cf79f043e18b375e7901263f network default NAD default] I0327 23:11:04.543299 3901 cni.go:424] [tsf-tas/test-tsf-tas f7b6b0b255b34fd877ac3c895756cbbb54e20340cf79f043e18b375e7901263f network default NAD default] DEL finished CNI request [tsf-tas/test-tsf-tas f7b6b0b255b34fd877ac3c895756cbbb54e20340cf79f043e18b375e7901263f network default NAD default], result "{}", err I0327 23:11:04.595334 3901 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:04.595371 3901 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0327 23:11:04.596591 3901 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.129.0.42 I0327 23:11:04.596622 3901 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:04.614967 3901 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:04.614986 3901 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:04.620569 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0327 23:11:04.620587 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0327 23:11:05.266564 3901 base_network_controller_pods.go:487] [default/tsf/patch-tekton-config-l4pwv] creating logical port tsf_patch-tekton-config-l4pwv for pod on switch ip-10-0-2-195.ec2.internal I0327 23:11:05.266663 3901 kube.go:256] Updating pod tsf/patch-tekton-config-l4pwv I0327 23:11:05.278365 3901 pod.go:62] [tsf/patch-tekton-config-l4pwv] pod update took 11.730031ms I0327 23:11:05.278382 3901 base_network_controller_pods.go:937] [default/tsf/patch-tekton-config-l4pwv] addLogicalPort annotation time took 11.753461ms I0327 23:11:05.279642 3901 pods.go:271] [tsf/patch-tekton-config-l4pwv] addLogicalPort took 13.096922ms, libovsdb time 845.434µs I0327 23:11:05.627898 3901 cni.go:403] [tsf/patch-tekton-config-l4pwv f66a92212c3c50b58fe3c3fff37a2aa1eb194c7e5a91161cadc61506b590685b network default NAD default] ADD starting CNI request [tsf/patch-tekton-config-l4pwv f66a92212c3c50b58fe3c3fff37a2aa1eb194c7e5a91161cadc61506b590685b network default NAD default] I0327 23:11:05.636320 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: patch-tekton-config-l4pwv, hostIfaceName: f66a92212c3c50b, network: default, NAD default, SandboxID: "f66a92212c3c50b58fe3c3fff37a2aa1eb194c7e5a91161cadc61506b590685b", PCI device ID: , UID: "af665654-393c-49b2-aa82-a4c9e093ee8b", MAC: 0a:58:0a:81:00:2c, IPs: [10.129.0.44/23] I0327 23:11:05.679607 3901 cni.go:424] [tsf/patch-tekton-config-l4pwv f66a92212c3c50b58fe3c3fff37a2aa1eb194c7e5a91161cadc61506b590685b network default NAD default] ADD finished CNI request [tsf/patch-tekton-config-l4pwv f66a92212c3c50b58fe3c3fff37a2aa1eb194c7e5a91161cadc61506b590685b network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:cf:27:d1:f2:82\",\"name\":\"f66a92212c3c50b\"},{\"mac\":\"0a:58:0a:81:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3f5d72ab-3837-42a0-baa1-8f9dfefed61e\"}],\"ips\":[{\"address\":\"10.129.0.44/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:11:12.522791 3901 cni.go:403] [tsf/patch-tekton-config-l4pwv f66a92212c3c50b58fe3c3fff37a2aa1eb194c7e5a91161cadc61506b590685b network default NAD default] DEL starting CNI request [tsf/patch-tekton-config-l4pwv f66a92212c3c50b58fe3c3fff37a2aa1eb194c7e5a91161cadc61506b590685b network default NAD default] I0327 23:11:12.604962 3901 cni.go:424] [tsf/patch-tekton-config-l4pwv f66a92212c3c50b58fe3c3fff37a2aa1eb194c7e5a91161cadc61506b590685b network default NAD default] DEL finished CNI request [tsf/patch-tekton-config-l4pwv f66a92212c3c50b58fe3c3fff37a2aa1eb194c7e5a91161cadc61506b590685b network default NAD default], result "{}", err I0327 23:11:12.652606 3901 obj_retry.go:464] Detected object tsf/patch-tekton-config-l4pwv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:12.652638 3901 pods.go:176] Deleting pod: tsf/patch-tekton-config-l4pwv I0327 23:11:12.653970 3901 pods.go:236] Attempting to release IPs for pod: tsf/patch-tekton-config-l4pwv, ips: 10.129.0.44 I0327 23:11:12.654002 3901 obj_retry.go:464] Detected object tsf/patch-tekton-config-l4pwv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:13.685737 3901 obj_retry.go:458] Detected object tsf/patch-tekton-config-l4pwv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:13.685764 3901 obj_retry.go:458] Detected object tsf/patch-tekton-config-l4pwv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:15.539540 3901 cni.go:403] [tsf-tas/rekor-createtree-job-wpwm6-x5grc 51be6e0b23708ee47bdf3d4a47ac70ee856e348b7bfb1aad340b58ac1d59a628 network default NAD default] DEL starting CNI request [tsf-tas/rekor-createtree-job-wpwm6-x5grc 51be6e0b23708ee47bdf3d4a47ac70ee856e348b7bfb1aad340b58ac1d59a628 network default NAD default] I0327 23:11:15.619953 3901 cni.go:424] [tsf-tas/rekor-createtree-job-wpwm6-x5grc 51be6e0b23708ee47bdf3d4a47ac70ee856e348b7bfb1aad340b58ac1d59a628 network default NAD default] DEL finished CNI request [tsf-tas/rekor-createtree-job-wpwm6-x5grc 51be6e0b23708ee47bdf3d4a47ac70ee856e348b7bfb1aad340b58ac1d59a628 network default NAD default], result "{}", err I0327 23:11:15.667590 3901 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-wpwm6-x5grc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:15.667622 3901 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-wpwm6-x5grc I0327 23:11:15.669220 3901 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-wpwm6-x5grc, ips: 10.129.0.43 I0327 23:11:15.669321 3901 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-wpwm6-x5grc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:16.305933 3901 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=58 I0327 23:11:16.552298 3901 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-wpwm6-x5grc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:16.552325 3901 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-wpwm6-x5grc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:17.550940 3901 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-m9vwn-l2rxq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:17.551605 3901 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-m9vwn-l2rxq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:18.441281 3901 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-m9vwn-l2rxq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:18.441310 3901 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-m9vwn-l2rxq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:18.626546 3901 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-7444b77bd8-qfpz9] creating logical port tsf-tas_rekor-search-ui-7444b77bd8-qfpz9 for pod on switch ip-10-0-2-195.ec2.internal I0327 23:11:18.626631 3901 kube.go:256] Updating pod tsf-tas/rekor-search-ui-7444b77bd8-qfpz9 I0327 23:11:18.643033 3901 pod.go:62] [tsf-tas/rekor-search-ui-7444b77bd8-qfpz9] pod update took 16.426532ms I0327 23:11:18.643056 3901 base_network_controller_pods.go:937] [default/tsf-tas/rekor-search-ui-7444b77bd8-qfpz9] addLogicalPort annotation time took 16.456216ms I0327 23:11:18.644198 3901 pods.go:271] [tsf-tas/rekor-search-ui-7444b77bd8-qfpz9] addLogicalPort took 17.665356ms, libovsdb time 782.812µs I0327 23:11:18.987129 3901 cni.go:403] [tsf-tas/rekor-search-ui-7444b77bd8-qfpz9 0120c8f6d0693df2f130e56e51377e789e51ac97f12abee3ae9b352df349c8b3 network default NAD default] ADD starting CNI request [tsf-tas/rekor-search-ui-7444b77bd8-qfpz9 0120c8f6d0693df2f130e56e51377e789e51ac97f12abee3ae9b352df349c8b3 network default NAD default] I0327 23:11:18.995791 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-7444b77bd8-qfpz9, hostIfaceName: 0120c8f6d0693df, network: default, NAD default, SandboxID: "0120c8f6d0693df2f130e56e51377e789e51ac97f12abee3ae9b352df349c8b3", PCI device ID: , UID: "760c5254-09c3-4839-83bf-ee446ee632b6", MAC: 0a:58:0a:81:00:2d, IPs: [10.129.0.45/23] I0327 23:11:19.245070 3901 cni.go:424] [tsf-tas/rekor-search-ui-7444b77bd8-qfpz9 0120c8f6d0693df2f130e56e51377e789e51ac97f12abee3ae9b352df349c8b3 network default NAD default] ADD finished CNI request [tsf-tas/rekor-search-ui-7444b77bd8-qfpz9 0120c8f6d0693df2f130e56e51377e789e51ac97f12abee3ae9b352df349c8b3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:0f:c3:dd:cd:4b\",\"name\":\"0120c8f6d0693df\"},{\"mac\":\"0a:58:0a:81:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7fefc89f-acf5-46d7-927c-35c3f58190da\"}],\"ips\":[{\"address\":\"10.129.0.45/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:11:21.239836 3901 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-66c488c8cd-5wlsh] creating logical port tsf-tas_rekor-server-66c488c8cd-5wlsh for pod on switch ip-10-0-2-195.ec2.internal I0327 23:11:21.239972 3901 kube.go:256] Updating pod tsf-tas/rekor-server-66c488c8cd-5wlsh I0327 23:11:21.254684 3901 pod.go:62] [tsf-tas/rekor-server-66c488c8cd-5wlsh] pod update took 14.734008ms I0327 23:11:21.254716 3901 base_network_controller_pods.go:937] [default/tsf-tas/rekor-server-66c488c8cd-5wlsh] addLogicalPort annotation time took 14.771374ms I0327 23:11:21.256143 3901 pods.go:271] [tsf-tas/rekor-server-66c488c8cd-5wlsh] addLogicalPort took 16.330492ms, libovsdb time 1.019941ms W0327 23:11:21.460003 3901 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0327 23:11:22.391750 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0327 23:11:22.442458 3901 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-5fbcd4678c-rfxtf] creating logical port tsf-tas_ctlog-5fbcd4678c-rfxtf for pod on switch ip-10-0-2-195.ec2.internal I0327 23:11:22.442565 3901 kube.go:256] Updating pod tsf-tas/ctlog-5fbcd4678c-rfxtf I0327 23:11:22.457634 3901 pod.go:62] [tsf-tas/ctlog-5fbcd4678c-rfxtf] pod update took 15.093191ms I0327 23:11:22.457659 3901 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-5fbcd4678c-rfxtf] addLogicalPort annotation time took 15.12312ms I0327 23:11:22.458839 3901 pods.go:271] [tsf-tas/ctlog-5fbcd4678c-rfxtf] addLogicalPort took 16.398114ms, libovsdb time 808.258µs W0327 23:11:22.578128 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0327 23:11:22.755896 3901 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-rn4wv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:22.756450 3901 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-rn4wv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:22.801524 3901 cni.go:403] [tsf-tas/ctlog-5fbcd4678c-rfxtf 0bcd645e056924276e4f2365dc356c910f1b87966cfd0952da326791093a9161 network default NAD default] ADD starting CNI request [tsf-tas/ctlog-5fbcd4678c-rfxtf 0bcd645e056924276e4f2365dc356c910f1b87966cfd0952da326791093a9161 network default NAD default] I0327 23:11:22.809753 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-5fbcd4678c-rfxtf, hostIfaceName: 0bcd645e0569242, network: default, NAD default, SandboxID: "0bcd645e056924276e4f2365dc356c910f1b87966cfd0952da326791093a9161", PCI device ID: , UID: "6744bb09-21ad-404c-999b-5d234176122f", MAC: 0a:58:0a:81:00:2f, IPs: [10.129.0.47/23] I0327 23:11:23.060461 3901 cni.go:424] [tsf-tas/ctlog-5fbcd4678c-rfxtf 0bcd645e056924276e4f2365dc356c910f1b87966cfd0952da326791093a9161 network default NAD default] ADD finished CNI request [tsf-tas/ctlog-5fbcd4678c-rfxtf 0bcd645e056924276e4f2365dc356c910f1b87966cfd0952da326791093a9161 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:f5:91:9d:96:16\",\"name\":\"0bcd645e0569242\"},{\"mac\":\"0a:58:0a:81:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/470983fb-1261-48d0-8270-002385223cf9\"}],\"ips\":[{\"address\":\"10.129.0.47/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:11:23.773957 3901 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-rn4wv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:23.773978 3901 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-rn4wv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:23.835097 3901 obj_retry.go:458] Detected object tsf/patch-tekton-config-l4pwv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:23.835118 3901 obj_retry.go:458] Detected object tsf/patch-tekton-config-l4pwv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:23.839334 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-l4pwv I0327 23:11:23.839354 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-l4pwv W0327 23:11:26.599575 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0327 23:11:31.501620 3901 cni.go:403] [tsf-tas/rekor-server-66c488c8cd-5wlsh 30deed31f658b7cace702261167ddd8990654822a01d423ff1a554f655ff589f network default NAD default] ADD starting CNI request [tsf-tas/rekor-server-66c488c8cd-5wlsh 30deed31f658b7cace702261167ddd8990654822a01d423ff1a554f655ff589f network default NAD default] I0327 23:11:31.509814 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-66c488c8cd-5wlsh, hostIfaceName: 30deed31f658b7c, network: default, NAD default, SandboxID: "30deed31f658b7cace702261167ddd8990654822a01d423ff1a554f655ff589f", PCI device ID: , UID: "91af8851-837e-4162-8ba4-8bcf2cd57abb", MAC: 0a:58:0a:81:00:2e, IPs: [10.129.0.46/23] I0327 23:11:31.553851 3901 cni.go:424] [tsf-tas/rekor-server-66c488c8cd-5wlsh 30deed31f658b7cace702261167ddd8990654822a01d423ff1a554f655ff589f network default NAD default] ADD finished CNI request [tsf-tas/rekor-server-66c488c8cd-5wlsh 30deed31f658b7cace702261167ddd8990654822a01d423ff1a554f655ff589f network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:a9:84:9b:34:f1\",\"name\":\"30deed31f658b7c\"},{\"mac\":\"0a:58:0a:81:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a11d4eea-ce9c-4d77-9852-5e88ab289823\"}],\"ips\":[{\"address\":\"10.129.0.46/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:11:34.636520 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui W0327 23:11:34.660627 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0327 23:11:34.678593 3901 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:34.679271 3901 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:34.703745 3901 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:34.703772 3901 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:34.710202 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0327 23:11:34.710222 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0327 23:11:35.311101 3901 base_network_controller_pods.go:487] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-2-195.ec2.internal I0327 23:11:35.311190 3901 kube.go:256] Updating pod tsf-tpa/pre-install-tsf-tpa I0327 23:11:35.324401 3901 pod.go:62] [tsf-tpa/pre-install-tsf-tpa] pod update took 13.235005ms I0327 23:11:35.324420 3901 base_network_controller_pods.go:937] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 13.258171ms I0327 23:11:35.325861 3901 pods.go:271] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 14.7798ms, libovsdb time 964.837µs I0327 23:11:35.671144 3901 cni.go:403] [tsf-tpa/pre-install-tsf-tpa fd040cf481c66021d128685197f1e01b84c606aa1d584955dd7eab9cc9b0be52 network default NAD default] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa fd040cf481c66021d128685197f1e01b84c606aa1d584955dd7eab9cc9b0be52 network default NAD default] I0327 23:11:35.679489 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: fd040cf481c6602, network: default, NAD default, SandboxID: "fd040cf481c66021d128685197f1e01b84c606aa1d584955dd7eab9cc9b0be52", PCI device ID: , UID: "20615769-6e10-4cf4-991d-412c4a425ee1", MAC: 0a:58:0a:81:00:30, IPs: [10.129.0.48/23] I0327 23:11:35.723544 3901 cni.go:424] [tsf-tpa/pre-install-tsf-tpa fd040cf481c66021d128685197f1e01b84c606aa1d584955dd7eab9cc9b0be52 network default NAD default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa fd040cf481c66021d128685197f1e01b84c606aa1d584955dd7eab9cc9b0be52 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:a4:95:aa:ed:dd\",\"name\":\"fd040cf481c6602\"},{\"mac\":\"0a:58:0a:81:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e37ef785-cdc8-410e-a1bc-11fd90468794\"}],\"ips\":[{\"address\":\"10.129.0.48/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0327 23:11:37.610568 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog I0327 23:11:38.678694 3901 cni.go:403] [tsf-tpa/pre-install-tsf-tpa fd040cf481c66021d128685197f1e01b84c606aa1d584955dd7eab9cc9b0be52 network default NAD default] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa fd040cf481c66021d128685197f1e01b84c606aa1d584955dd7eab9cc9b0be52 network default NAD default] I0327 23:11:38.754173 3901 cni.go:424] [tsf-tpa/pre-install-tsf-tpa fd040cf481c66021d128685197f1e01b84c606aa1d584955dd7eab9cc9b0be52 network default NAD default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa fd040cf481c66021d128685197f1e01b84c606aa1d584955dd7eab9cc9b0be52 network default NAD default], result "{}", err I0327 23:11:38.807065 3901 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:38.807090 3901 pods.go:176] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0327 23:11:38.808555 3901 pods.go:236] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.129.0.48 I0327 23:11:38.808589 3901 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:38.828659 3901 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:38.828689 3901 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:38.830834 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0327 23:11:38.830855 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0327 23:11:39.115400 3901 base_network_controller_pods.go:487] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-2-195.ec2.internal I0327 23:11:39.115540 3901 kube.go:256] Updating pod tsf-tpa/test-tsf-tpa I0327 23:11:39.134521 3901 pod.go:62] [tsf-tpa/test-tsf-tpa] pod update took 19.007577ms I0327 23:11:39.134541 3901 base_network_controller_pods.go:937] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 19.033007ms I0327 23:11:39.135795 3901 pods.go:271] [tsf-tpa/test-tsf-tpa] addLogicalPort took 20.41443ms, libovsdb time 882.007µs I0327 23:11:39.484440 3901 cni.go:403] [tsf-tpa/test-tsf-tpa 4156509da6856863b5adbc781d4a1ad25017f263c239f68dcd4cfd676c409f8c network default NAD default] ADD starting CNI request [tsf-tpa/test-tsf-tpa 4156509da6856863b5adbc781d4a1ad25017f263c239f68dcd4cfd676c409f8c network default NAD default] I0327 23:11:39.492986 3901 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: 4156509da685686, network: default, NAD default, SandboxID: "4156509da6856863b5adbc781d4a1ad25017f263c239f68dcd4cfd676c409f8c", PCI device ID: , UID: "d9a7d701-90a0-4420-a58d-a3e0eb6184a6", MAC: 0a:58:0a:81:00:31, IPs: [10.129.0.49/23] I0327 23:11:39.538937 3901 cni.go:424] [tsf-tpa/test-tsf-tpa 4156509da6856863b5adbc781d4a1ad25017f263c239f68dcd4cfd676c409f8c network default NAD default] ADD finished CNI request [tsf-tpa/test-tsf-tpa 4156509da6856863b5adbc781d4a1ad25017f263c239f68dcd4cfd676c409f8c network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:96:3d:b3:3e:fe\",\"name\":\"4156509da685686\"},{\"mac\":\"0a:58:0a:81:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c4de3340-74a0-4b8c-96f7-c6b57c11f6dc\"}],\"ips\":[{\"address\":\"10.129.0.49/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:11:44.746996 3901 obj_retry.go:464] Detected object tsf-tpa/create-db-47htb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:44.747557 3901 obj_retry.go:464] Detected object tsf-tpa/create-db-47htb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 23:11:45.653147 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0327 23:11:45.777894 3901 obj_retry.go:458] Detected object tsf-tpa/create-db-47htb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:45.777915 3901 obj_retry.go:458] Detected object tsf-tpa/create-db-47htb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:46.397690 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0327 23:11:48.766693 3901 obj_retry.go:464] Detected object tsf-tpa/migrate-db-v94m6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:48.767257 3901 obj_retry.go:464] Detected object tsf-tpa/migrate-db-v94m6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:48.900202 3901 obj_retry.go:458] Detected object tsf-tpa/migrate-db-v94m6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:48.900223 3901 obj_retry.go:458] Detected object tsf-tpa/migrate-db-v94m6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:51.784939 3901 obj_retry.go:464] Detected object tsf-tpa/create-importers-tgp5h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:51.785592 3901 obj_retry.go:464] Detected object tsf-tpa/create-importers-tgp5h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:52.806189 3901 obj_retry.go:458] Detected object tsf-tpa/create-importers-tgp5h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:52.806236 3901 obj_retry.go:458] Detected object tsf-tpa/create-importers-tgp5h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:53.385989 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0327 23:11:57.918881 3901 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-qkrpw-29dcv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:57.919498 3901 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-qkrpw-29dcv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:58.313324 3901 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=567 I0327 23:11:58.814628 3901 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-qkrpw-29dcv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:58.814653 3901 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-qkrpw-29dcv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 23:12:05.853351 3901 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0327 23:12:13.895733 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0327 23:12:24.418938 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0327 23:12:33.991966 3901 cni.go:403] [tsf-tpa/test-tsf-tpa 4156509da6856863b5adbc781d4a1ad25017f263c239f68dcd4cfd676c409f8c network default NAD default] DEL starting CNI request [tsf-tpa/test-tsf-tpa 4156509da6856863b5adbc781d4a1ad25017f263c239f68dcd4cfd676c409f8c network default NAD default] I0327 23:12:34.063544 3901 cni.go:424] [tsf-tpa/test-tsf-tpa 4156509da6856863b5adbc781d4a1ad25017f263c239f68dcd4cfd676c409f8c network default NAD default] DEL finished CNI request [tsf-tpa/test-tsf-tpa 4156509da6856863b5adbc781d4a1ad25017f263c239f68dcd4cfd676c409f8c network default NAD default], result "{}", err I0327 23:12:34.115390 3901 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:12:34.115420 3901 pods.go:176] Deleting pod: tsf-tpa/test-tsf-tpa I0327 23:12:34.116959 3901 pods.go:236] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.129.0.49 I0327 23:12:34.116987 3901 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:12:34.136716 3901 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:12:34.136739 3901 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:12:34.140011 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0327 23:12:34.140038 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0327 23:12:35.102246 3901 namespace.go:98] [integration-service] adding namespace I0327 23:12:35.103035 3901 namespace.go:102] [integration-service] adding namespace took 768.9µs I0327 23:12:35.103068 3901 namespace.go:98] [image-controller] adding namespace I0327 23:12:35.103496 3901 namespace.go:102] [image-controller] adding namespace took 421.108µs I0327 23:12:35.103876 3901 namespace.go:98] [build-service] adding namespace I0327 23:12:35.104543 3901 namespace.go:102] [build-service] adding namespace took 653.041µs I0327 23:12:35.133845 3901 namespace.go:142] [integration-service] updating namespace I0327 23:12:35.152778 3901 namespace.go:142] [integration-service] updating namespace I0327 23:12:35.172967 3901 namespace.go:142] [image-controller] updating namespace I0327 23:12:35.205555 3901 namespace.go:142] [image-controller] updating namespace I0327 23:12:35.227021 3901 namespace.go:142] [build-service] updating namespace I0327 23:12:35.256833 3901 namespace.go:142] [build-service] updating namespace I0327 23:12:35.351335 3901 namespace.go:142] [build-service] updating namespace I0327 23:12:35.366702 3901 namespace.go:142] [integration-service] updating namespace I0327 23:12:35.390560 3901 namespace.go:98] [release-service] adding namespace I0327 23:12:35.391508 3901 namespace.go:102] [release-service] adding namespace took 932.287µs I0327 23:12:35.417272 3901 namespace.go:142] [release-service] updating namespace I0327 23:12:35.422095 3901 namespace.go:142] [konflux-ui] updating namespace I0327 23:12:35.471791 3901 namespace.go:142] [release-service] updating namespace I0327 23:12:35.623609 3901 namespace.go:98] [konflux-info] adding namespace I0327 23:12:35.624565 3901 namespace.go:102] [konflux-info] adding namespace took 933.766µs I0327 23:12:35.759303 3901 namespace.go:142] [konflux-info] updating namespace I0327 23:12:35.813331 3901 namespace.go:98] [namespace-lister] adding namespace I0327 23:12:35.814047 3901 namespace.go:102] [namespace-lister] adding namespace took 701.449µs I0327 23:12:35.916015 3901 namespace.go:142] [konflux-info] updating namespace I0327 23:12:35.937544 3901 namespace.go:98] [enterprise-contract-service] adding namespace I0327 23:12:35.938296 3901 namespace.go:102] [enterprise-contract-service] adding namespace took 737.57µs I0327 23:12:35.972073 3901 namespace.go:142] [namespace-lister] updating namespace I0327 23:12:35.975640 3901 namespace.go:142] [image-controller] updating namespace I0327 23:12:36.220660 3901 namespace.go:142] [namespace-lister] updating namespace I0327 23:12:36.444716 3901 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:12:36.560896 3901 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:12:36.736856 3901 namespace.go:98] [default-tenant] adding namespace I0327 23:12:36.737746 3901 namespace.go:102] [default-tenant] adding namespace took 869.791µs I0327 23:12:36.921107 3901 namespace.go:98] [segment-bridge] adding namespace I0327 23:12:36.921879 3901 namespace.go:102] [segment-bridge] adding namespace took 756.154µs I0327 23:12:36.970796 3901 namespace.go:142] [default-tenant] updating namespace I0327 23:12:37.024644 3901 namespace.go:142] [default-tenant] updating namespace I0327 23:12:37.144520 3901 namespace.go:142] [segment-bridge] updating namespace I0327 23:12:37.187352 3901 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-67655c4674-x2gm5] creating logical port namespace-lister_namespace-lister-67655c4674-x2gm5 for pod on switch ip-10-0-2-195.ec2.internal I0327 23:12:37.187432 3901 kube.go:256] Updating pod namespace-lister/namespace-lister-67655c4674-x2gm5 I0327 23:12:37.225778 3901 namespace.go:142] [segment-bridge] updating namespace I0327 23:12:37.288181 3901 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0327 23:12:37.807719 3901 namespace.go:142] [build-service] updating namespace I0327 23:12:38.213419 3901 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:12:38.440388 3901 kube.go:256] Updating pod namespace-lister/namespace-lister-67655c4674-x2gm5 I0327 23:12:38.479948 3901 pod.go:62] [namespace-lister/namespace-lister-67655c4674-x2gm5] pod update took 1.292539357s I0327 23:12:38.479973 3901 base_network_controller_pods.go:937] [default/namespace-lister/namespace-lister-67655c4674-x2gm5] addLogicalPort annotation time took 1.292567093s I0327 23:12:38.481227 3901 pods.go:271] [namespace-lister/namespace-lister-67655c4674-x2gm5] addLogicalPort took 1.293891316s, libovsdb time 782.792µs I0327 23:12:38.481333 3901 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0327 23:12:38.481354 3901 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0327 23:12:38.481440 3901 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-759977d69f-c8pzp] creating logical port integration-service_integration-service-controller-manager-759977d69f-c8pzp for pod on switch ip-10-0-2-195.ec2.internal I0327 23:12:38.481448 3901 base_network_controller_pods.go:487] [default/konflux-ui/proxy-58bb569fbd-fksns] creating logical port konflux-ui_proxy-58bb569fbd-fksns for pod on switch ip-10-0-2-195.ec2.internal I0327 23:12:38.481545 3901 kube.go:256] Updating pod integration-service/integration-service-controller-manager-759977d69f-c8pzp I0327 23:12:38.481580 3901 kube.go:256] Updating pod konflux-ui/proxy-58bb569fbd-fksns I0327 23:12:38.525049 3901 kube.go:256] Updating pod integration-service/integration-service-controller-manager-759977d69f-c8pzp I0327 23:12:38.542867 3901 namespace.go:142] [image-controller] updating namespace I0327 23:12:38.544633 3901 pod.go:62] [konflux-ui/proxy-58bb569fbd-fksns] pod update took 63.107771ms I0327 23:12:38.544651 3901 base_network_controller_pods.go:937] [default/konflux-ui/proxy-58bb569fbd-fksns] addLogicalPort annotation time took 63.129882ms I0327 23:12:38.545713 3901 pods.go:271] [konflux-ui/proxy-58bb569fbd-fksns] addLogicalPort took 64.271058ms, libovsdb time 718.335µs I0327 23:12:38.555029 3901 pod.go:62] [integration-service/integration-service-controller-manager-759977d69f-c8pzp] pod update took 73.504064ms I0327 23:12:38.555052 3901 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-759977d69f-c8pzp] addLogicalPort annotation time took 73.529277ms I0327 23:12:38.556272 3901 pods.go:271] [integration-service/integration-service-controller-manager-759977d69f-c8pzp] addLogicalPort took 74.839586ms, libovsdb time 765.608µs I0327 23:12:38.556747 3901 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0327 23:12:38.556779 3901 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0327 23:12:38.556793 3901 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0327 23:12:38.556864 3901 base_network_controller_pods.go:487] [default/konflux-ui/dex-7b65f9676c-f92r5] creating logical port konflux-ui_dex-7b65f9676c-f92r5 for pod on switch ip-10-0-2-195.ec2.internal I0327 23:12:38.556940 3901 kube.go:256] Updating pod konflux-ui/dex-7b65f9676c-f92r5 I0327 23:12:38.579783 3901 pod.go:62] [konflux-ui/dex-7b65f9676c-f92r5] pod update took 22.873387ms I0327 23:12:38.579799 3901 base_network_controller_pods.go:937] [default/konflux-ui/dex-7b65f9676c-f92r5] addLogicalPort annotation time took 22.891447ms I0327 23:12:38.580677 3901 pods.go:271] [konflux-ui/dex-7b65f9676c-f92r5] addLogicalPort took 23.82484ms, libovsdb time 586.82µs I0327 23:12:38.581077 3901 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0327 23:12:38.600412 3901 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-854f775895-jgbkj] creating logical port release-service_release-service-controller-manager-854f775895-jgbkj for pod on switch ip-10-0-2-195.ec2.internal I0327 23:12:38.600508 3901 kube.go:256] Updating pod release-service/release-service-controller-manager-854f775895-jgbkj I0327 23:12:38.630805 3901 pod.go:62] [release-service/release-service-controller-manager-854f775895-jgbkj] pod update took 30.31692ms I0327 23:12:38.630823 3901 base_network_controller_pods.go:937] [default/release-service/release-service-controller-manager-854f775895-jgbkj] addLogicalPort annotation time took 30.355381ms I0327 23:12:38.631979 3901 pods.go:271] [release-service/release-service-controller-manager-854f775895-jgbkj] addLogicalPort took 31.57834ms, libovsdb time 781.581µs I0327 23:12:38.824745 3901 namespace.go:142] [integration-service] updating namespace I0327 23:12:39.029694 3901 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-vpgth of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:12:39.030300 3901 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-vpgth of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:12:39.151527 3901 namespace.go:142] [konflux-info] updating namespace I0327 23:12:39.341343 3901 namespace.go:142] [namespace-lister] updating namespace I0327 23:12:39.511684 3901 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-65d5cf98c-r5n5x] creating logical port integration-service_integration-service-controller-manager-65d5cf98c-r5n5x for pod on switch ip-10-0-2-195.ec2.internal I0327 23:12:39.511769 3901 kube.go:256] Updating pod integration-service/integration-service-controller-manager-65d5cf98c-r5n5x I0327 23:12:39.527038 3901 pod.go:62] [integration-service/integration-service-controller-manager-65d5cf98c-r5n5x] pod update took 15.291893ms I0327 23:12:39.527060 3901 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-65d5cf98c-r5n5x] addLogicalPort annotation time took 15.320783ms I0327 23:12:39.528268 3901 pods.go:271] [integration-service/integration-service-controller-manager-65d5cf98c-r5n5x] addLogicalPort took 16.600672ms, libovsdb time 858.067µs I0327 23:12:39.545748 3901 namespace.go:142] [release-service] updating namespace I0327 23:12:39.703943 3901 namespace.go:142] [build-service] updating namespace I0327 23:12:39.720655 3901 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:12:39.734586 3901 namespace.go:142] [image-controller] updating namespace I0327 23:12:39.746774 3901 namespace.go:142] [integration-service] updating namespace I0327 23:12:39.762191 3901 namespace.go:142] [konflux-info] updating namespace I0327 23:12:39.780601 3901 namespace.go:142] [namespace-lister] updating namespace I0327 23:12:39.796432 3901 namespace.go:142] [release-service] updating namespace I0327 23:12:39.833858 3901 namespace.go:142] [build-service] updating namespace I0327 23:12:39.880922 3901 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:12:39.948768 3901 namespace.go:142] [image-controller] updating namespace I0327 23:12:40.046291 3901 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-vpgth of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:12:40.046313 3901 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-vpgth of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:12:40.050407 3901 namespace.go:142] [integration-service] updating namespace I0327 23:12:40.149670 3901 namespace.go:142] [konflux-info] updating namespace I0327 23:12:40.216234 3901 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-2-195.ec2.internal I0327 23:12:40.216327 3901 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0327 23:12:40.233500 3901 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 17.195894ms I0327 23:12:40.233517 3901 base_network_controller_pods.go:937] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 17.216477ms I0327 23:12:40.234708 3901 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 18.48835ms, libovsdb time 823.406µs I0327 23:12:40.252214 3901 namespace.go:142] [namespace-lister] updating namespace I0327 23:12:40.345333 3901 namespace.go:142] [release-service] updating namespace I0327 23:12:40.575986 3901 cni.go:403] [konflux-ui/test-tsf-konflux b4969115c867998842f858c7d106ccfb1a9da8960aed4844e09556c24a3ae0e3 network default NAD default] ADD starting CNI request [konflux-ui/test-tsf-konflux b4969115c867998842f858c7d106ccfb1a9da8960aed4844e09556c24a3ae0e3 network default NAD default] I0327 23:12:40.587529 3901 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: b4969115c867998, network: default, NAD default, SandboxID: "b4969115c867998842f858c7d106ccfb1a9da8960aed4844e09556c24a3ae0e3", PCI device ID: , UID: "71b84ed7-edcb-48f2-ada3-926068978398", MAC: 0a:58:0a:81:00:38, IPs: [10.129.0.56/23] I0327 23:12:40.637415 3901 cni.go:424] [konflux-ui/test-tsf-konflux b4969115c867998842f858c7d106ccfb1a9da8960aed4844e09556c24a3ae0e3 network default NAD default] ADD finished CNI request [konflux-ui/test-tsf-konflux b4969115c867998842f858c7d106ccfb1a9da8960aed4844e09556c24a3ae0e3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:be:e3:dc:c3:70\",\"name\":\"b4969115c867998\"},{\"mac\":\"0a:58:0a:81:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/530252bd-5de6-4d80-92f5-a63398987d8c\"}],\"ips\":[{\"address\":\"10.129.0.56/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:12:41.536601 3901 namespace.go:142] [default-tenant] updating namespace I0327 23:12:41.738174 3901 namespace.go:142] [segment-bridge] updating namespace I0327 23:12:42.047949 3901 namespace.go:142] [default-tenant] updating namespace I0327 23:12:42.052116 3901 cni.go:403] [namespace-lister/namespace-lister-67655c4674-x2gm5 3de06250dbcff4777cd04cf67db65761cfa4dbff68649adba14998480fb171f9 network default NAD default] ADD starting CNI request [namespace-lister/namespace-lister-67655c4674-x2gm5 3de06250dbcff4777cd04cf67db65761cfa4dbff68649adba14998480fb171f9 network default NAD default] I0327 23:12:42.060789 3901 helper_linux.go:450] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-67655c4674-x2gm5, hostIfaceName: 3de06250dbcff47, network: default, NAD default, SandboxID: "3de06250dbcff4777cd04cf67db65761cfa4dbff68649adba14998480fb171f9", PCI device ID: , UID: "16f3f02f-f975-4c76-ab7a-2f543e4b0f83", MAC: 0a:58:0a:81:00:32, IPs: [10.129.0.50/23] I0327 23:12:42.069322 3901 namespace.go:142] [segment-bridge] updating namespace I0327 23:12:42.105936 3901 cni.go:424] [namespace-lister/namespace-lister-67655c4674-x2gm5 3de06250dbcff4777cd04cf67db65761cfa4dbff68649adba14998480fb171f9 network default NAD default] ADD finished CNI request [namespace-lister/namespace-lister-67655c4674-x2gm5 3de06250dbcff4777cd04cf67db65761cfa4dbff68649adba14998480fb171f9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:56:fb:3f:ea:62\",\"name\":\"3de06250dbcff47\"},{\"mac\":\"0a:58:0a:81:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1bd7486d-eeb1-4ed3-8cde-f8d912883c6d\"}],\"ips\":[{\"address\":\"10.129.0.50/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:12:42.170181 3901 namespace.go:142] [default-tenant] updating namespace I0327 23:12:42.269334 3901 namespace.go:142] [segment-bridge] updating namespace W0327 23:12:42.930760 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0327 23:12:42.952290 3901 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0327 23:12:46.065142 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0327 23:12:46.085976 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister I0327 23:12:46.382268 3901 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0327 23:12:48.824287 3901 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-c8pzp 0ce089e63ec8476149a996a999949a1e8e057b84bca06d776d0c5dbc53860802 network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-759977d69f-c8pzp 0ce089e63ec8476149a996a999949a1e8e057b84bca06d776d0c5dbc53860802 network default NAD default] I0327 23:12:48.833521 3901 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-759977d69f-c8pzp, hostIfaceName: 0ce089e63ec8476, network: default, NAD default, SandboxID: "0ce089e63ec8476149a996a999949a1e8e057b84bca06d776d0c5dbc53860802", PCI device ID: , UID: "330ef77a-9aa0-40c4-b819-d26a1f3cfe5b", MAC: 0a:58:0a:81:00:34, IPs: [10.129.0.52/23] I0327 23:12:48.877600 3901 cni.go:403] [release-service/release-service-controller-manager-854f775895-jgbkj 92d0429caf7e0a13f15f113a800549d7db52ed947a1d6e61cdb004a63ec30604 network default NAD default] ADD starting CNI request [release-service/release-service-controller-manager-854f775895-jgbkj 92d0429caf7e0a13f15f113a800549d7db52ed947a1d6e61cdb004a63ec30604 network default NAD default] I0327 23:12:48.880097 3901 cni.go:403] [integration-service/integration-service-controller-manager-65d5cf98c-r5n5x efd2843fe47f5c9848f99ffe96011636c67bc7f968fa02de9a136c5a96f63c8e network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-65d5cf98c-r5n5x efd2843fe47f5c9848f99ffe96011636c67bc7f968fa02de9a136c5a96f63c8e network default NAD default] I0327 23:12:48.886300 3901 cni.go:403] [konflux-ui/proxy-58bb569fbd-fksns 5ddcb245903d2e72d3bd341f1742a1b37a4725c43f580486828d366475a3dc6f network default NAD default] ADD starting CNI request [konflux-ui/proxy-58bb569fbd-fksns 5ddcb245903d2e72d3bd341f1742a1b37a4725c43f580486828d366475a3dc6f network default NAD default] I0327 23:12:48.886841 3901 cni.go:403] [konflux-ui/dex-7b65f9676c-f92r5 09b5598f76ec5e8bb4cc49d32da77ba259093caf3851af191bcde5159caa8068 network default NAD default] ADD starting CNI request [konflux-ui/dex-7b65f9676c-f92r5 09b5598f76ec5e8bb4cc49d32da77ba259093caf3851af191bcde5159caa8068 network default NAD default] I0327 23:12:48.892031 3901 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-c8pzp 0ce089e63ec8476149a996a999949a1e8e057b84bca06d776d0c5dbc53860802 network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-759977d69f-c8pzp 0ce089e63ec8476149a996a999949a1e8e057b84bca06d776d0c5dbc53860802 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:30:b8:a2:aa:92\",\"name\":\"0ce089e63ec8476\"},{\"mac\":\"0a:58:0a:81:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3f392693-a577-44f2-b68a-35a6ec1aed21\"}],\"ips\":[{\"address\":\"10.129.0.52/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:12:48.892210 3901 helper_linux.go:450] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-854f775895-jgbkj, hostIfaceName: 92d0429caf7e0a1, network: default, NAD default, SandboxID: "92d0429caf7e0a13f15f113a800549d7db52ed947a1d6e61cdb004a63ec30604", PCI device ID: , UID: "af585e31-2082-49cf-912a-6d6fcdb32519", MAC: 0a:58:0a:81:00:36, IPs: [10.129.0.54/23] I0327 23:12:48.892537 3901 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-65d5cf98c-r5n5x, hostIfaceName: efd2843fe47f5c9, network: default, NAD default, SandboxID: "efd2843fe47f5c9848f99ffe96011636c67bc7f968fa02de9a136c5a96f63c8e", PCI device ID: , UID: "5d2c2fbc-f887-4ff1-bb1c-28d5fc65378c", MAC: 0a:58:0a:81:00:37, IPs: [10.129.0.55/23] I0327 23:12:48.900360 3901 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-58bb569fbd-fksns, hostIfaceName: 5ddcb245903d2e7, network: default, NAD default, SandboxID: "5ddcb245903d2e72d3bd341f1742a1b37a4725c43f580486828d366475a3dc6f", PCI device ID: , UID: "5b633f60-807c-486e-b7e7-b7313051ba95", MAC: 0a:58:0a:81:00:33, IPs: [10.129.0.51/23] I0327 23:12:48.901498 3901 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: dex-7b65f9676c-f92r5, hostIfaceName: 09b5598f76ec5e8, network: default, NAD default, SandboxID: "09b5598f76ec5e8bb4cc49d32da77ba259093caf3851af191bcde5159caa8068", PCI device ID: , UID: "23f85fb0-bd87-41e0-998b-89accb1f97b9", MAC: 0a:58:0a:81:00:35, IPs: [10.129.0.53/23] I0327 23:12:48.946607 3901 cni.go:424] [release-service/release-service-controller-manager-854f775895-jgbkj 92d0429caf7e0a13f15f113a800549d7db52ed947a1d6e61cdb004a63ec30604 network default NAD default] ADD finished CNI request [release-service/release-service-controller-manager-854f775895-jgbkj 92d0429caf7e0a13f15f113a800549d7db52ed947a1d6e61cdb004a63ec30604 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:2d:e3:02:af:4f\",\"name\":\"92d0429caf7e0a1\"},{\"mac\":\"0a:58:0a:81:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/22576ad8-9a6b-4335-9dde-322a2d9ab2b9\"}],\"ips\":[{\"address\":\"10.129.0.54/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:12:48.967101 3901 cni.go:424] [integration-service/integration-service-controller-manager-65d5cf98c-r5n5x efd2843fe47f5c9848f99ffe96011636c67bc7f968fa02de9a136c5a96f63c8e network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-65d5cf98c-r5n5x efd2843fe47f5c9848f99ffe96011636c67bc7f968fa02de9a136c5a96f63c8e network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:13:7d:41:43:3c\",\"name\":\"efd2843fe47f5c9\"},{\"mac\":\"0a:58:0a:81:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/024d99d5-896c-45a2-9072-3eccff0dd030\"}],\"ips\":[{\"address\":\"10.129.0.55/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:12:49.192916 3901 cni.go:424] [konflux-ui/dex-7b65f9676c-f92r5 09b5598f76ec5e8bb4cc49d32da77ba259093caf3851af191bcde5159caa8068 network default NAD default] ADD finished CNI request [konflux-ui/dex-7b65f9676c-f92r5 09b5598f76ec5e8bb4cc49d32da77ba259093caf3851af191bcde5159caa8068 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:10:f6:1a:74:3a\",\"name\":\"09b5598f76ec5e8\"},{\"mac\":\"0a:58:0a:81:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cf50b355-1801-4b1e-94a9-a96d33c95480\"}],\"ips\":[{\"address\":\"10.129.0.53/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:12:49.193009 3901 cni.go:424] [konflux-ui/proxy-58bb569fbd-fksns 5ddcb245903d2e72d3bd341f1742a1b37a4725c43f580486828d366475a3dc6f network default NAD default] ADD finished CNI request [konflux-ui/proxy-58bb569fbd-fksns 5ddcb245903d2e72d3bd341f1742a1b37a4725c43f580486828d366475a3dc6f network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:8e:1e:6a:4a:9d\",\"name\":\"5ddcb245903d2e7\"},{\"mac\":\"0a:58:0a:81:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/402e51c2-5697-4b42-ae2b-819ca4e0cda7\"}],\"ips\":[{\"address\":\"10.129.0.51/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:12:50.301419 3901 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=66 W0327 23:12:52.001829 3901 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0327 23:12:52.001849 3901 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 23:12:56.140275 3901 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0327 23:12:56.158147 3901 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 23:12:56.161711 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 23:12:56.186350 3901 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 23:12:56.186368 3901 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0327 23:12:56.205125 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 23:12:56.208636 3901 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 23:12:56.234582 3901 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0327 23:12:56.235087 3901 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service I0327 23:13:05.402687 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 W0327 23:13:07.165792 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 23:13:07.165993 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 23:13:07.184724 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0327 23:13:07.184827 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service I0327 23:13:07.350582 3901 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-c8pzp 0ce089e63ec8476149a996a999949a1e8e057b84bca06d776d0c5dbc53860802 network default NAD default] DEL starting CNI request [integration-service/integration-service-controller-manager-759977d69f-c8pzp 0ce089e63ec8476149a996a999949a1e8e057b84bca06d776d0c5dbc53860802 network default NAD default] I0327 23:13:07.424022 3901 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-c8pzp 0ce089e63ec8476149a996a999949a1e8e057b84bca06d776d0c5dbc53860802 network default NAD default] DEL finished CNI request [integration-service/integration-service-controller-manager-759977d69f-c8pzp 0ce089e63ec8476149a996a999949a1e8e057b84bca06d776d0c5dbc53860802 network default NAD default], result "{}", err I0327 23:13:07.486407 3901 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-c8pzp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:07.486437 3901 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-759977d69f-c8pzp I0327 23:13:07.487696 3901 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-759977d69f-c8pzp, ips: 10.129.0.52 I0327 23:13:07.487724 3901 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-c8pzp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:08.241432 3901 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-c8pzp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:08.241455 3901 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-c8pzp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:08.247591 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-c8pzp I0327 23:13:08.247617 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-c8pzp W0327 23:13:09.175938 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex I0327 23:13:16.413284 3901 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 W0327 23:13:29.062020 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 23:13:29.062050 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0327 23:13:29.062067 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 23:13:29.062074 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0327 23:13:33.215090 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 23:13:33.215109 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0327 23:13:58.302591 3901 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=50 I0327 23:13:59.204900 3901 cni.go:403] [konflux-ui/proxy-58bb569fbd-fksns 5ddcb245903d2e72d3bd341f1742a1b37a4725c43f580486828d366475a3dc6f network default NAD default] DEL starting CNI request [konflux-ui/proxy-58bb569fbd-fksns 5ddcb245903d2e72d3bd341f1742a1b37a4725c43f580486828d366475a3dc6f network default NAD default] I0327 23:13:59.294691 3901 cni.go:424] [konflux-ui/proxy-58bb569fbd-fksns 5ddcb245903d2e72d3bd341f1742a1b37a4725c43f580486828d366475a3dc6f network default NAD default] DEL finished CNI request [konflux-ui/proxy-58bb569fbd-fksns 5ddcb245903d2e72d3bd341f1742a1b37a4725c43f580486828d366475a3dc6f network default NAD default], result "{}", err I0327 23:13:59.358097 3901 obj_retry.go:464] Detected object konflux-ui/proxy-58bb569fbd-fksns of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:59.358125 3901 pods.go:176] Deleting pod: konflux-ui/proxy-58bb569fbd-fksns I0327 23:13:59.359550 3901 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-58bb569fbd-fksns, ips: 10.129.0.51 I0327 23:13:59.359585 3901 obj_retry.go:464] Detected object konflux-ui/proxy-58bb569fbd-fksns of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:59.536526 3901 obj_retry.go:458] Detected object konflux-ui/proxy-58bb569fbd-fksns of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:59.536547 3901 obj_retry.go:458] Detected object konflux-ui/proxy-58bb569fbd-fksns of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:59.540438 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-58bb569fbd-fksns I0327 23:13:59.540496 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-58bb569fbd-fksns I0327 23:14:10.605492 3901 cni.go:403] [konflux-ui/test-tsf-konflux b4969115c867998842f858c7d106ccfb1a9da8960aed4844e09556c24a3ae0e3 network default NAD default] DEL starting CNI request [konflux-ui/test-tsf-konflux b4969115c867998842f858c7d106ccfb1a9da8960aed4844e09556c24a3ae0e3 network default NAD default] I0327 23:14:10.676873 3901 cni.go:424] [konflux-ui/test-tsf-konflux b4969115c867998842f858c7d106ccfb1a9da8960aed4844e09556c24a3ae0e3 network default NAD default] DEL finished CNI request [konflux-ui/test-tsf-konflux b4969115c867998842f858c7d106ccfb1a9da8960aed4844e09556c24a3ae0e3 network default NAD default], result "{}", err I0327 23:14:10.744632 3901 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:10.744679 3901 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0327 23:14:10.745998 3901 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.129.0.56 I0327 23:14:10.746024 3901 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:10.788981 3901 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:10.789032 3901 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:10.792952 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0327 23:14:10.792981 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0327 23:14:40.903800 3901 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:40.904405 3901 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:40.943381 3901 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-56b684c87-cjfhj] creating logical port image-controller_image-controller-controller-manager-56b684c87-cjfhj for pod on switch ip-10-0-2-195.ec2.internal I0327 23:14:40.943508 3901 kube.go:256] Updating pod image-controller/image-controller-controller-manager-56b684c87-cjfhj I0327 23:14:40.960025 3901 pod.go:62] [image-controller/image-controller-controller-manager-56b684c87-cjfhj] pod update took 16.560607ms I0327 23:14:40.960049 3901 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-56b684c87-cjfhj] addLogicalPort annotation time took 16.587328ms I0327 23:14:40.961234 3901 pods.go:271] [image-controller/image-controller-controller-manager-56b684c87-cjfhj] addLogicalPort took 17.867547ms, libovsdb time 818.135µs I0327 23:14:41.305406 3901 cni.go:403] [image-controller/image-controller-controller-manager-56b684c87-cjfhj bfb1ecb32b4746f853e69d29b00281fa1a4e12b2d90bcfa84196bc6b55eb4a60 network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-56b684c87-cjfhj bfb1ecb32b4746f853e69d29b00281fa1a4e12b2d90bcfa84196bc6b55eb4a60 network default NAD default] I0327 23:14:41.313865 3901 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-56b684c87-cjfhj, hostIfaceName: bfb1ecb32b4746f, network: default, NAD default, SandboxID: "bfb1ecb32b4746f853e69d29b00281fa1a4e12b2d90bcfa84196bc6b55eb4a60", PCI device ID: , UID: "4ad5928c-cc70-45e3-bade-ee0a97b077ba", MAC: 0a:58:0a:81:00:39, IPs: [10.129.0.57/23] I0327 23:14:41.565933 3901 cni.go:424] [image-controller/image-controller-controller-manager-56b684c87-cjfhj bfb1ecb32b4746f853e69d29b00281fa1a4e12b2d90bcfa84196bc6b55eb4a60 network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-56b684c87-cjfhj bfb1ecb32b4746f853e69d29b00281fa1a4e12b2d90bcfa84196bc6b55eb4a60 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:9f:06:df:a3:15\",\"name\":\"bfb1ecb32b4746f\"},{\"mac\":\"0a:58:0a:81:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2e3ffdc7-e207-4fc8-a4ab-4cb3f3d7cfff\"}],\"ips\":[{\"address\":\"10.129.0.57/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 23:14:41.616828 3901 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:41.616847 3901 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:41.623783 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 I0327 23:14:41.623806 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 I0327 23:14:42.299444 3901 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=12 I0327 23:14:47.407577 3901 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 W0327 23:14:55.809016 3901 util.go:959] Failed to build node endpoints for node ip-10-0-2-195.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0327 23:14:56.115611 3901 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:56.116188 3901 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:56.703430 3901 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:56.703454 3901 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:56.706761 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 I0327 23:14:56.706788 3901 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8