+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-121.ec2.internal ++ [[ -n ip-10-0-0-121.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-121.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0429 19:38:58.673198529 - disable conntrack on geneve port' I0429 19:38:58.673198529 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0429 19:38:58.698481247 - starting ovnkube-node + echo 'I0429 19:38:58.698481247 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-121.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-121.ec2.internal --init-node ip-10-0-0-121.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-121.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0429 19:38:58.747401 3658 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0429 19:38:58.747460 3658 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-c689bfa631.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0429 19:38:58.749032 3658 kube.go:431] Waiting for certificate I0429 19:38:58.749063 3658 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0429 19:38:58.749095 3658 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0429 19:38:58.749822 3658 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0429 19:38:58.782800 3658 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0429 19:38:58.782829 3658 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0429 19:38:58.785845 3658 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0429 19:38:58.785942 3658 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-5qz52" I0429 19:38:58.792073 3658 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-5qz52" I0429 19:38:58.792147 3658 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0429 19:38:59.749840 3658 kube.go:438] Certificate found I0429 19:38:59.750447 3658 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0429 19:38:59.751828 3658 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0429 19:38:59.815040 3658 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-30 19:33:58 +0000 UTC" deadline="2026-04-30 14:25:48.305624354 +0000 UTC" I0429 19:38:59.815075 3658 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="18h46m48.490560751s" I0429 19:38:59.822610 3658 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0429 19:38:59.833724 3658 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0429 19:38:59.833886 3658 server.go:101] MetricServer registers OVS metrics I0429 19:38:59.834275 3658 server.go:105] MetricServer registers OVN DB metrics I0429 19:38:59.834863 3658 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0429 19:38:59.834929 3658 factory.go:561] Starting watch factory I0429 19:38:59.835021 3658 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.835037 3658 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.835051 3658 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.835066 3658 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.835129 3658 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.835164 3658 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.835198 3658 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.835207 3658 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.835898 3658 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.835927 3658 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.836350 3658 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.836430 3658 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.839370 3658 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0429 19:38:59.839396 3658 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0429 19:38:59.841294 3658 controller_manager.go:370] Starting the ovnkube controller I0429 19:38:59.841303 3658 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-121.ec2.internal I0429 19:38:59.841343 3658 controller_manager.go:395] NBDB zone sync took: 32.484µs I0429 19:38:59.841351 3658 factory.go:561] Starting watch factory I0429 19:38:59.852775 3658 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0429 19:38:59.852799 3658 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0429 19:38:59.856589 3658 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0429 19:38:59.856643 3658 server.go:109] MetricServer registers OVN Controller metrics I0429 19:38:59.858479 3658 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.858825 3658 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.858888 3658 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.859221 3658 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.861643 3658 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.861731 3658 server.go:113] MetricServer registers OVN Northd metrics I0429 19:38:59.866502 3658 metrics.go:519] OVN Metrics Server starts to run ... I0429 19:38:59.868119 3658 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0429 19:38:59.940820 3658 factory.go:1977] *v1.Node informer cache synced successfully I0429 19:38:59.940844 3658 factory.go:1977] *v1.Service informer cache synced successfully I0429 19:38:59.940848 3658 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0429 19:38:59.940851 3658 factory.go:1977] *v1.Pod informer cache synced successfully I0429 19:38:59.940854 3658 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0429 19:38:59.940857 3658 factory.go:1977] *v1.Namespace informer cache synced successfully I0429 19:38:59.940948 3658 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 19:38:59.940948 3658 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 19:38:59.940962 3658 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 19:38:59.940965 3658 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 19:38:59.944778 3658 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 19:38:59.944779 3658 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 19:38:59.946638 3658 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0429 19:38:59.946649 3658 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0429 19:38:59.946651 3658 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0429 19:38:59.946657 3658 factory.go:1977] *v1.Pod informer cache synced successfully I0429 19:38:59.946661 3658 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0429 19:38:59.946664 3658 factory.go:1977] *v1.Namespace informer cache synced successfully I0429 19:38:59.946667 3658 factory.go:1977] *v1.Node informer cache synced successfully I0429 19:38:59.946669 3658 factory.go:1977] *v1.Service informer cache synced successfully I0429 19:38:59.946699 3658 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.946710 3658 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.949727 3658 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.951694 3658 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0429 19:38:59.951708 3658 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0429 19:38:59.951769 3658 factory.go:1977] *v1.EgressIP informer cache synced successfully I0429 19:38:59.951820 3658 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.951831 3658 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.954845 3658 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.956759 3658 factory.go:1977] *v1.EgressIP informer cache synced successfully I0429 19:38:59.956844 3658 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0429 19:38:59.956904 3658 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.956915 3658 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.959886 3658 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.961836 3658 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0429 19:38:59.961925 3658 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0429 19:38:59.961981 3658 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.961994 3658 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.964924 3658 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.966884 3658 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0429 19:38:59.966976 3658 factory.go:1977] *v1.EgressService informer cache synced successfully I0429 19:38:59.967042 3658 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.967060 3658 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.969986 3658 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.972823 3658 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0429 19:38:59.972831 3658 factory.go:1977] *v1.EgressService informer cache synced successfully I0429 19:38:59.973001 3658 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0429 19:38:59.973018 3658 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0429 19:38:59.976102 3658 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0429 19:38:59.978083 3658 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0429 19:38:59.978158 3658 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0429 19:38:59.978216 3658 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.978227 3658 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.978273 3658 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.978292 3658 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.981469 3658 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.981559 3658 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 19:38:59.983463 3658 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0429 19:38:59.983474 3658 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0429 19:38:59.983476 3658 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0429 19:38:59.983481 3658 factory.go:680] Watch Factory start up complete, took: 148.557935ms I0429 19:38:59.983557 3658 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0429 19:38:59.989078 3658 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0429 19:38:59.989089 3658 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0429 19:38:59.989096 3658 factory.go:680] Watch Factory start up complete, took: 147.743293ms I0429 19:38:59.989103 3658 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-0-121.ec2.internal" zone I0429 19:39:00.039273 3658 default_node_network_controller.go:710] Initializing the default node network controller I0429 19:39:00.122731 3658 udn_isolation.go:101] Starting UDN host isolation manager I0429 19:39:00.128733 3658 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0429 19:39:00.183248 3658 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0429 19:39:00.183324 3658 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0429 19:39:00.183340 3658 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0429 19:39:00.218160 3658 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0429 19:39:00.218230 3658 default_node_network_controller.go:809] Node ip-10-0-0-121.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0429 19:39:00.224955 3658 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.121"] k8s.ovn.org/zone-name:ip-10-0-0-121.ec2.internal] on node ip-10-0-0-121.ec2.internal I0429 19:39:00.246826 3658 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0429 19:39:00.253341 3658 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0429 19:39:00.253469 3658 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0429 19:39:00.253628 3658 gateway_init.go:263] Preparing Gateway I0429 19:39:00.253642 3658 gateway_shared_intf.go:1687] Creating new gateway I0429 19:39:00.362871 3658 iptables.go:108] Creating table: filter chain: FORWARD I0429 19:39:00.369770 3658 iptables.go:108] Creating table: filter chain: OUTPUT I0429 19:39:00.376649 3658 gateway_shared_intf.go:1816] Gateway Creation Complete I0429 19:39:00.376666 3658 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-121.ec2.internal","mac-address":"12:11:6d:78:9a:a1","ip-addresses":["10.0.0.121/24"],"ip-address":"10.0.0.121/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:3f771be6-3db9-4bf0-9376-8585c6b4eb85 k8s.ovn.org/node-encap-ips:["10.0.0.121"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.121/24"} k8s.ovn.org/zone-name:ip-10-0-0-121.ec2.internal] on node ip-10-0-0-121.ec2.internal I0429 19:39:00.397727 3658 default_node_network_controller.go:519] Node connection status = connected I0429 19:39:00.489155 3658 controller_manager.go:429] Waiting for node in zone sync took: 500.03625ms I0429 19:39:00.496966 3658 controller_manager.go:611] Switching to transit router for layer2 networks I0429 19:39:00.496982 3658 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-121.ec2.internal I0429 19:39:00.514350 3658 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0429 19:39:00.521536 3658 controller_manager.go:290] SCTP support detected in OVN I0429 19:39:00.522145 3658 services_controller.go:69] Creating services controller for network=default I0429 19:39:00.522238 3658 udn_enabled_svc.go:80] Waiting for service informer to sync I0429 19:39:00.522277 3658 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0429 19:39:00.522307 3658 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0429 19:39:00.522321 3658 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0429 19:39:00.522329 3658 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0429 19:39:00.522524 3658 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0429 19:39:00.522590 3658 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0429 19:39:00.522603 3658 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 47.322µs I0429 19:39:00.522636 3658 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 34.388µs I0429 19:39:00.522649 3658 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 58.858µs I0429 19:39:00.522657 3658 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0429 19:39:00.522669 3658 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0429 19:39:00.522684 3658 default_network_controller.go:325] Starting the default network controller I0429 19:39:00.522701 3658 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 16.087µs I0429 19:39:00.522788 3658 udn_enabled_svc.go:95] Performing full resync I0429 19:39:00.523113 3658 udn_enabled_svc.go:99] Waiting for handler to sync I0429 19:39:00.523123 3658 udn_enabled_svc.go:104] Starting worker I0429 19:39:00.533395 3658 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0429 19:39:00.533456 3658 repair.go:33] Syncing exgw routes took 47.306µs I0429 19:39:00.533480 3658 default_network_controller.go:414] Starting all the Watchers... I0429 19:39:00.533649 3658 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0429 19:39:00.533701 3658 namespace.go:98] [openshift-host-network] adding namespace I0429 19:39:00.533702 3658 namespace.go:98] [openshift-ingress-operator] adding namespace I0429 19:39:00.533706 3658 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0429 19:39:00.533716 3658 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0429 19:39:00.533721 3658 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0429 19:39:00.533722 3658 namespace.go:98] [openshift-monitoring] adding namespace I0429 19:39:00.533728 3658 namespace.go:98] [openshift-controller-manager] adding namespace I0429 19:39:00.533733 3658 namespace.go:98] [openshift-logging] adding namespace I0429 19:39:00.533744 3658 namespace.go:98] [openshift-service-ca-operator] adding namespace I0429 19:39:00.533746 3658 namespace.go:98] [openshift-package-operator] adding namespace I0429 19:39:00.533746 3658 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0429 19:39:00.533754 3658 namespace.go:98] [openshift-customer-monitoring] adding namespace I0429 19:39:00.533748 3658 namespace.go:98] [openshift-node] adding namespace I0429 19:39:00.533750 3658 namespace.go:98] [openshift-kube-scheduler] adding namespace I0429 19:39:00.534191 3658 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 530.933µs I0429 19:39:00.534206 3658 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0429 19:39:00.534741 3658 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 529.71µs I0429 19:39:00.534756 3658 namespace.go:98] [openshift-backplane-mobb] adding namespace I0429 19:39:00.535181 3658 namespace.go:102] [openshift-backplane-mobb] adding namespace took 418.968µs I0429 19:39:00.535196 3658 namespace.go:98] [openshift-config-operator] adding namespace I0429 19:39:00.535810 3658 namespace.go:102] [openshift-host-network] adding namespace took 2.095587ms I0429 19:39:00.535822 3658 namespace.go:98] [openshift-route-controller-manager] adding namespace I0429 19:39:00.536202 3658 namespace.go:102] [openshift-ingress-operator] adding namespace took 2.485966ms I0429 19:39:00.536212 3658 namespace.go:98] [openshift-ingress] adding namespace I0429 19:39:00.536749 3658 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 3.029114ms I0429 19:39:00.536764 3658 namespace.go:98] [kube-node-lease] adding namespace I0429 19:39:00.537212 3658 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 3.489631ms I0429 19:39:00.537221 3658 namespace.go:98] [openshift-network-console] adding namespace I0429 19:39:00.537716 3658 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 3.986204ms I0429 19:39:00.537731 3658 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0429 19:39:00.538117 3658 namespace.go:102] [openshift-logging] adding namespace took 4.378042ms I0429 19:39:00.538127 3658 namespace.go:98] [openshift-operators-redhat] adding namespace I0429 19:39:00.538661 3658 namespace.go:102] [openshift-monitoring] adding namespace took 4.927374ms I0429 19:39:00.538678 3658 namespace.go:98] [openshift-machine-config-operator] adding namespace I0429 19:39:00.539085 3658 namespace.go:102] [openshift-controller-manager] adding namespace took 5.350709ms I0429 19:39:00.539098 3658 namespace.go:98] [openshift-apiserver] adding namespace I0429 19:39:00.539511 3658 namespace.go:102] [openshift-service-ca-operator] adding namespace took 5.760235ms I0429 19:39:00.539523 3658 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0429 19:39:00.540015 3658 namespace.go:102] [openshift-package-operator] adding namespace took 6.261622ms I0429 19:39:00.540037 3658 namespace.go:98] [open-cluster-management-2q00ovcq3dp22etaglm2t8tqigpmb591] adding namespace I0429 19:39:00.540469 3658 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.707976ms I0429 19:39:00.540500 3658 namespace.go:98] [openshift-operators] adding namespace I0429 19:39:00.541008 3658 namespace.go:102] [openshift-node] adding namespace took 7.243886ms I0429 19:39:00.541021 3658 namespace.go:98] [openshift-dns] adding namespace I0429 19:39:00.541469 3658 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.710771ms I0429 19:39:00.541481 3658 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0429 19:39:00.541895 3658 namespace.go:102] [openshift-kube-scheduler] adding namespace took 8.127898ms I0429 19:39:00.541909 3658 namespace.go:98] [openshift-backplane-csm] adding namespace I0429 19:39:00.542508 3658 namespace.go:102] [openshift-config-operator] adding namespace took 7.305611ms I0429 19:39:00.542520 3658 namespace.go:98] [openshift-multus] adding namespace I0429 19:39:00.543015 3658 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.18886ms I0429 19:39:00.543026 3658 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0429 19:39:00.543630 3658 namespace.go:102] [openshift-ingress] adding namespace took 7.411632ms I0429 19:39:00.543650 3658 namespace.go:98] [openshift-cluster-version] adding namespace I0429 19:39:00.544139 3658 namespace.go:102] [kube-node-lease] adding namespace took 7.367912ms I0429 19:39:00.544154 3658 namespace.go:98] [openshift-authentication] adding namespace I0429 19:39:00.544595 3658 namespace.go:102] [openshift-network-console] adding namespace took 7.366342ms I0429 19:39:00.544613 3658 namespace.go:98] [openshift-backplane-cee] adding namespace I0429 19:39:00.545099 3658 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 7.362504ms I0429 19:39:00.545113 3658 namespace.go:98] [openshift-console-user-settings] adding namespace I0429 19:39:00.545464 3658 namespace.go:102] [openshift-operators-redhat] adding namespace took 7.330407ms I0429 19:39:00.545480 3658 namespace.go:98] [openshift-apiserver-operator] adding namespace I0429 19:39:00.545999 3658 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.314571ms I0429 19:39:00.546012 3658 namespace.go:98] [openshift-machine-api] adding namespace I0429 19:39:00.546351 3658 namespace.go:102] [openshift-apiserver] adding namespace took 7.246464ms I0429 19:39:00.546367 3658 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0429 19:39:00.546799 3658 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.270592ms I0429 19:39:00.546814 3658 namespace.go:98] [default] adding namespace I0429 19:39:00.547302 3658 namespace.go:102] [open-cluster-management-2q00ovcq3dp22etaglm2t8tqigpmb591] adding namespace took 7.257744ms I0429 19:39:00.547321 3658 namespace.go:98] [openshift-etcd] adding namespace I0429 19:39:00.548283 3658 namespace.go:102] [openshift-operators] adding namespace took 7.775682ms I0429 19:39:00.548298 3658 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0429 19:39:00.548727 3658 namespace.go:102] [openshift-dns] adding namespace took 7.699989ms I0429 19:39:00.548737 3658 namespace.go:98] [openshift-authentication-operator] adding namespace I0429 19:39:00.549150 3658 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.648013ms I0429 19:39:00.549167 3658 namespace.go:98] [openshift-config-managed] adding namespace W0429 19:39:00.549561 3658 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-wsr6s: pod openshift-multus/network-metrics-daemon-wsr6s: no pod IPs found W0429 19:39:00.549602 3658 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-b7p9m: pod openshift-multus/network-metrics-daemon-b7p9m: no pod IPs found I0429 19:39:00.549610 3658 namespace.go:102] [openshift-backplane-csm] adding namespace took 7.694531ms I0429 19:39:00.549624 3658 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0429 19:39:00.550098 3658 namespace.go:102] [openshift-multus] adding namespace took 7.571961ms I0429 19:39:00.550506 3658 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 7.474417ms I0429 19:39:00.550519 3658 namespace.go:98] [openshift-backplane] adding namespace I0429 19:39:00.550939 3658 namespace.go:102] [openshift-cluster-version] adding namespace took 7.283684ms I0429 19:39:00.550952 3658 namespace.go:98] [openshift-backplane-cse] adding namespace I0429 19:39:00.551351 3658 namespace.go:102] [openshift-authentication] adding namespace took 7.191321ms I0429 19:39:00.551363 3658 namespace.go:98] [openshift-console] adding namespace I0429 19:39:00.551771 3658 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.153253ms I0429 19:39:00.551785 3658 namespace.go:98] [openshift-network-operator] adding namespace I0429 19:39:00.552174 3658 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.055999ms I0429 19:39:00.552186 3658 namespace.go:98] [openshift-image-registry] adding namespace I0429 19:39:00.552580 3658 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.079872ms I0429 19:39:00.552592 3658 namespace.go:98] [openshift-kube-apiserver] adding namespace I0429 19:39:00.552992 3658 namespace.go:102] [openshift-machine-api] adding namespace took 6.974883ms I0429 19:39:00.553005 3658 namespace.go:98] [openshift-must-gather-operator] adding namespace I0429 19:39:00.553370 3658 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.998222ms I0429 19:39:00.553382 3658 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0429 19:39:00.553765 3658 namespace.go:102] [default] adding namespace took 6.945637ms I0429 19:39:00.553777 3658 namespace.go:98] [openshift-backplane-srep] adding namespace I0429 19:39:00.554150 3658 namespace.go:102] [openshift-etcd] adding namespace took 6.824354ms I0429 19:39:00.554162 3658 namespace.go:98] [openshift-backplane-tam] adding namespace I0429 19:39:00.554554 3658 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.251511ms I0429 19:39:00.554565 3658 namespace.go:98] [openshift-insights] adding namespace I0429 19:39:00.554965 3658 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.223487ms I0429 19:39:00.554976 3658 namespace.go:98] [dedicated-admin] adding namespace I0429 19:39:00.555353 3658 namespace.go:102] [openshift-config-managed] adding namespace took 6.180156ms I0429 19:39:00.555364 3658 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0429 19:39:00.555759 3658 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.129869ms I0429 19:39:00.555770 3658 namespace.go:98] [openshift-console-operator] adding namespace I0429 19:39:00.556149 3658 namespace.go:102] [openshift-backplane] adding namespace took 5.62589ms I0429 19:39:00.556160 3658 namespace.go:98] [openshift-config] adding namespace I0429 19:39:00.556553 3658 namespace.go:102] [openshift-backplane-cse] adding namespace took 5.596449ms I0429 19:39:00.556564 3658 namespace.go:98] [openshift-dns-operator] adding namespace I0429 19:39:00.556940 3658 namespace.go:102] [openshift-console] adding namespace took 5.572898ms I0429 19:39:00.556951 3658 namespace.go:98] [kube-system] adding namespace I0429 19:39:00.557365 3658 namespace.go:102] [openshift-network-operator] adding namespace took 5.576752ms I0429 19:39:00.557374 3658 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0429 19:39:00.557860 3658 namespace.go:102] [openshift-image-registry] adding namespace took 5.66848ms I0429 19:39:00.557879 3658 namespace.go:98] [openshift-ingress-canary] adding namespace I0429 19:39:00.558273 3658 namespace.go:102] [openshift-kube-apiserver] adding namespace took 5.67605ms I0429 19:39:00.558284 3658 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0429 19:39:00.558699 3658 namespace.go:102] [openshift-must-gather-operator] adding namespace took 5.687315ms I0429 19:39:00.558714 3658 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0429 19:39:00.559105 3658 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 5.71893ms I0429 19:39:00.559117 3658 namespace.go:98] [openshift] adding namespace I0429 19:39:00.559504 3658 namespace.go:102] [openshift-backplane-srep] adding namespace took 5.722834ms I0429 19:39:00.559513 3658 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0429 19:39:00.559979 3658 namespace.go:102] [openshift-backplane-tam] adding namespace took 5.811154ms I0429 19:39:00.559994 3658 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0429 19:39:00.560395 3658 namespace.go:102] [openshift-insights] adding namespace took 5.825596ms I0429 19:39:00.560411 3658 namespace.go:98] [openshift-marketplace] adding namespace I0429 19:39:00.560840 3658 namespace.go:102] [dedicated-admin] adding namespace took 5.858202ms I0429 19:39:00.560856 3658 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0429 19:39:00.561230 3658 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 5.859664ms I0429 19:39:00.561245 3658 namespace.go:98] [openshift-infra] adding namespace I0429 19:39:00.561715 3658 namespace.go:102] [openshift-console-operator] adding namespace took 5.939159ms I0429 19:39:00.561728 3658 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0429 19:39:00.562208 3658 namespace.go:102] [openshift-config] adding namespace took 6.043702ms I0429 19:39:00.562652 3658 namespace.go:102] [openshift-dns-operator] adding namespace took 6.08227ms I0429 19:39:00.562670 3658 namespace.go:98] [openshift-network-diagnostics] adding namespace I0429 19:39:00.563075 3658 namespace.go:102] [kube-system] adding namespace took 6.118551ms I0429 19:39:00.563444 3658 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.065235ms I0429 19:39:00.563936 3658 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.051417ms I0429 19:39:00.564313 3658 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.022988ms I0429 19:39:00.564327 3658 namespace.go:98] [kube-public] adding namespace I0429 19:39:00.564751 3658 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.032783ms I0429 19:39:00.565213 3658 namespace.go:102] [openshift] adding namespace took 6.090708ms I0429 19:39:00.565645 3658 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.127609ms I0429 19:39:00.565657 3658 namespace.go:98] [openshift-network-node-identity] adding namespace I0429 19:39:00.566015 3658 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.012454ms I0429 19:39:00.566451 3658 namespace.go:102] [openshift-marketplace] adding namespace took 6.034855ms I0429 19:39:00.566462 3658 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0429 19:39:00.566878 3658 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.015893ms I0429 19:39:00.567280 3658 namespace.go:102] [openshift-infra] adding namespace took 6.029122ms W0429 19:39:00.568180 3658 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-555bt: pod openshift-network-diagnostics/network-check-target-555bt: no pod IPs found W0429 19:39:00.568193 3658 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-r4ptz: pod openshift-network-diagnostics/network-check-target-r4ptz: no pod IPs found I0429 19:39:00.568209 3658 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.476859ms I0429 19:39:00.568701 3658 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.024573ms I0429 19:39:00.569101 3658 namespace.go:102] [kube-public] adding namespace took 4.768876ms I0429 19:39:00.569504 3658 namespace.go:102] [openshift-network-node-identity] adding namespace took 3.842856ms I0429 19:39:00.569913 3658 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 3.446758ms W0429 19:39:00.569993 3658 master.go:471] Unable to parse remote node ip-10-0-2-37.ec2.internal chassis-id annotation. Chassis may be removed during sync I0429 19:39:00.570087 3658 master.go:548] Adding or Updating local node "ip-10-0-0-121.ec2.internal" for network "default" I0429 19:39:00.570174 3658 obj_retry.go:578] Failed to create *v1.Node ip-10-0-2-37.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-37.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-37.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-37.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-37.ec2.internal I0429 19:39:00.571000 3658 zone_ic_handler.go:195] Time taken to create transit switch: 355.346µs I0429 19:39:00.571014 3658 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-196.ec2.internal for the network default I0429 19:39:00.571950 3658 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-196.ec2.internal" on network "default" took: 1.31702ms W0429 19:39:00.578060 3658 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-wsr6s: pod openshift-multus/network-metrics-daemon-wsr6s: no pod IPs found W0429 19:39:00.578080 3658 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-555bt: pod openshift-network-diagnostics/network-check-target-555bt: no pod IPs found I0429 19:39:00.579117 3658 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-121.ec2.internal for the network default I0429 19:39:00.579300 3658 zone_ic_handler.go:195] Time taken to create transit switch: 169.633µs I0429 19:39:00.580109 3658 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0429 19:39:00.580142 3658 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0429 19:39:00.580145 3658 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-121.ec2.internal in network "default" I0429 19:39:00.580163 3658 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0429 19:39:00.580193 3658 node_tracker.go:169] Node ip-10-0-0-121.ec2.internal switch + router changed, syncing services in network "default" I0429 19:39:00.580210 3658 services_controller.go:574] Full service sync requested for network=default I0429 19:39:00.580214 3658 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-196.ec2.internal in network "default" I0429 19:39:00.580238 3658 node_tracker.go:169] Node ip-10-0-1-196.ec2.internal switch + router changed, syncing services in network "default" I0429 19:39:00.580245 3658 services_controller.go:574] Full service sync requested for network=default I0429 19:39:00.580250 3658 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-37.ec2.internal in network "default" I0429 19:39:00.580259 3658 node_tracker.go:235] Node ip-10-0-2-37.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-37.ec2.internal" I0429 19:39:00.580268 3658 node_tracker.go:169] Node ip-10-0-2-37.ec2.internal switch + router changed, syncing services in network "default" I0429 19:39:00.580272 3658 services_controller.go:574] Full service sync requested for network=default I0429 19:39:00.680465 3658 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0429 19:39:00.680505 3658 services_controller.go:208] Setting up event handlers for services for network=default I0429 19:39:00.680599 3658 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0429 19:39:00.680643 3658 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0429 19:39:00.680660 3658 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0429 19:39:00.781718 3658 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0429 19:39:00.781753 3658 repair.go:60] Starting repairing loop for services I0429 19:39:00.781839 3658 repair.go:131] Deleted 0 stale service LBs I0429 19:39:00.781862 3658 repair.go:137] Deleted 0 stale Chassis Template Vars I0429 19:39:00.781877 3658 repair.go:62] Finished repairing loop for services: 126.212µs I0429 19:39:00.781902 3658 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0429 19:39:00.781910 3658 services_controller.go:254] Starting workers for network=default W0429 19:39:00.782072 3658 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 19:39:00.782236 3658 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-b7p9m, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-b7p9m: suppressed error logged: pod openshift-multus/network-metrics-daemon-b7p9m: no pod IPs found I0429 19:39:00.782244 3658 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-555bt] creating logical port openshift-network-diagnostics_network-check-target-555bt for pod on switch ip-10-0-0-121.ec2.internal I0429 19:39:00.782396 3658 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-555bt I0429 19:39:00.782403 3658 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-wsr6s] creating logical port openshift-multus_network-metrics-daemon-wsr6s for pod on switch ip-10-0-0-121.ec2.internal I0429 19:39:00.782422 3658 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-r4ptz, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-r4ptz: suppressed error logged: pod openshift-network-diagnostics/network-check-target-r4ptz: no pod IPs found I0429 19:39:00.782532 3658 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-wsr6s I0429 19:39:00.801667 3658 pod.go:62] [openshift-network-diagnostics/network-check-target-555bt] pod update took 19.317628ms I0429 19:39:00.801695 3658 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-555bt] addLogicalPort annotation time took 19.349943ms I0429 19:39:00.801740 3658 pod.go:62] [openshift-multus/network-metrics-daemon-wsr6s] pod update took 19.266629ms I0429 19:39:00.801762 3658 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-wsr6s] addLogicalPort annotation time took 19.290762ms I0429 19:39:00.802693 3658 pods.go:271] [openshift-multus/network-metrics-daemon-wsr6s] addLogicalPort took 20.300428ms, libovsdb time 709.986µs I0429 19:39:00.803025 3658 pods.go:271] [openshift-network-diagnostics/network-check-target-555bt] addLogicalPort took 20.787943ms, libovsdb time 1.037ms I0429 19:39:00.803507 3658 admin_network_policy_controller.go:227] Starting controller default-network-controller I0429 19:39:00.803530 3658 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0429 19:39:00.803543 3658 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0429 19:39:00.803547 3658 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0429 19:39:00.803733 3658 repair.go:29] Repairing admin network policies took 180.294µs I0429 19:39:00.803829 3658 repair.go:92] Repairing baseline admin network policies took 88.321µs I0429 19:39:00.804324 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0429 19:39:00.804343 3658 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0429 19:39:00.806063 3658 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0429 19:39:00.806083 3658 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0429 19:39:00.806093 3658 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0429 19:39:00.807763 3658 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0429 19:39:00.807784 3658 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0429 19:39:00.807794 3658 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0429 19:39:00.809686 3658 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0429 19:39:00.809706 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0429 19:39:00.809719 3658 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0429 19:39:00.811390 3658 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0429 19:39:00.811410 3658 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0429 19:39:00.811419 3658 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0429 19:39:00.813518 3658 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0429 19:39:00.813545 3658 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0429 19:39:00.813558 3658 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0429 19:39:00.816064 3658 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0429 19:39:00.816093 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0429 19:39:00.816102 3658 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0429 19:39:00.817885 3658 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0429 19:39:00.817909 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0429 19:39:00.817921 3658 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0429 19:39:00.819275 3658 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0429 19:39:00.819297 3658 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0429 19:39:00.819307 3658 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0429 19:39:00.819582 3658 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0429 19:39:00.819601 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0429 19:39:00.819610 3658 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0429 19:39:00.820111 3658 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0429 19:39:00.820132 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0429 19:39:00.820144 3658 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0429 19:39:00.820608 3658 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0429 19:39:00.820627 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0429 19:39:00.820634 3658 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0429 19:39:00.821168 3658 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0429 19:39:00.821191 3658 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0429 19:39:00.821206 3658 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0429 19:39:00.822596 3658 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0429 19:39:00.822617 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0429 19:39:00.822625 3658 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0429 19:39:00.823187 3658 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0429 19:39:00.823209 3658 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0429 19:39:00.823223 3658 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0429 19:39:00.824391 3658 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0429 19:39:00.824416 3658 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0429 19:39:00.824423 3658 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0429 19:39:00.824814 3658 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0429 19:39:00.824835 3658 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0429 19:39:00.824846 3658 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0429 19:39:00.826013 3658 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0429 19:39:00.826036 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0429 19:39:00.826050 3658 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0429 19:39:00.826797 3658 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0429 19:39:00.826815 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0429 19:39:00.826821 3658 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0429 19:39:00.827142 3658 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0429 19:39:00.827163 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0429 19:39:00.828020 3658 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0429 19:39:00.828041 3658 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0429 19:39:00.828827 3658 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0429 19:39:00.828860 3658 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0429 19:39:00.828868 3658 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0429 19:39:00.829955 3658 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0429 19:39:00.829977 3658 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0429 19:39:00.829986 3658 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0429 19:39:00.830922 3658 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0429 19:39:00.830941 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0429 19:39:00.830949 3658 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0429 19:39:00.831421 3658 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0429 19:39:00.831435 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0429 19:39:00.831455 3658 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0429 19:39:00.832183 3658 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0429 19:39:00.832204 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0429 19:39:00.832212 3658 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0429 19:39:00.832736 3658 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0429 19:39:00.832756 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0429 19:39:00.832762 3658 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0429 19:39:00.833414 3658 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0429 19:39:00.833434 3658 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0429 19:39:00.833441 3658 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0429 19:39:00.833953 3658 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0429 19:39:00.833969 3658 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0429 19:39:00.833978 3658 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0429 19:39:00.834979 3658 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0429 19:39:00.835005 3658 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0429 19:39:00.835030 3658 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0429 19:39:00.835568 3658 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0429 19:39:00.842527 3658 egressfirewall.go:374] Starting EgressFirewall controller I0429 19:39:00.842555 3658 controller.go:133] Adding controller egress-firewall-controller event handlers I0429 19:39:00.842579 3658 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0429 19:39:00.842599 3658 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0429 19:39:00.842602 3658 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0429 19:39:00.842649 3658 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0429 19:39:00.842661 3658 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0429 19:39:00.842961 3658 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0429 19:39:00.842974 3658 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0429 19:39:00.842979 3658 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0429 19:39:00.842988 3658 egressqos.go:193] Setting up event handlers for EgressQoS I0429 19:39:00.843008 3658 egressfirewall.go:723] Syncing node "ip-10-0-2-37.ec2.internal" for egress firewall I0429 19:39:00.843029 3658 egressfirewall.go:723] Syncing node "ip-10-0-0-121.ec2.internal" for egress firewall I0429 19:39:00.843037 3658 egressfirewall.go:723] Syncing node "ip-10-0-1-196.ec2.internal" for egress firewall I0429 19:39:00.843216 3658 egressqos.go:245] Starting EgressQoS Controller I0429 19:39:00.843230 3658 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0429 19:39:00.843239 3658 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0429 19:39:00.843245 3658 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0429 19:39:00.843250 3658 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0429 19:39:00.843257 3658 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0429 19:39:00.843264 3658 shared_informer.go:356] "Caches are synced" controller="egressqos" I0429 19:39:00.843271 3658 egressqos.go:259] Repairing EgressQoSes I0429 19:39:00.843276 3658 egressqos.go:399] Starting repairing loop for egressqos I0429 19:39:00.843360 3658 egressqos.go:401] Finished repairing loop for egressqos: 84.481µs I0429 19:39:00.843378 3658 egressservice_zone.go:132] Setting up event handlers for Egress Services I0429 19:39:00.843400 3658 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-121.ec2.internal I0429 19:39:00.843438 3658 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-121.ec2.internal : 38.703µs I0429 19:39:00.843544 3658 egressservice_zone.go:210] Starting Egress Services Controller I0429 19:39:00.843580 3658 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0429 19:39:00.843589 3658 shared_informer.go:356] "Caches are synced" controller="egressservices" I0429 19:39:00.843595 3658 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0429 19:39:00.843599 3658 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0429 19:39:00.843604 3658 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0429 19:39:00.843609 3658 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0429 19:39:00.843614 3658 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0429 19:39:00.843619 3658 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0429 19:39:00.843623 3658 egressservice_zone.go:228] Repairing Egress Services I0429 19:39:00.843914 3658 master_controller.go:88] Starting Admin Policy Based Route Controller I0429 19:39:00.843925 3658 external_controller.go:278] Starting Admin Policy Based Route Controller I0429 19:39:00.843986 3658 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-121.ec2.internal I0429 19:39:00.843996 3658 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-121.ec2.internal: 11.786µs I0429 19:39:00.844002 3658 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-196.ec2.internal I0429 19:39:00.844011 3658 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-196.ec2.internal: 9.566µs I0429 19:39:00.844018 3658 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-37.ec2.internal I0429 19:39:00.844022 3658 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-37.ec2.internal: 4.379µs I0429 19:39:00.844017 3658 default_network_controller.go:585] Completing all the Watchers took 310.510194ms I0429 19:39:00.844049 3658 default_network_controller.go:589] Starting unidling controllers I0429 19:39:00.844061 3658 unidle.go:46] Registering OVN SB ControllerEvent handler I0429 19:39:00.844066 3658 unidle.go:63] Populating Initial ContollerEvent events I0429 19:39:00.844095 3658 unidle.go:79] Setting up event handlers for services I0429 19:39:00.844211 3658 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0429 19:39:00.898384 3658 default_node_network_controller.go:519] Node connection status = connected I0429 19:39:01.398506 3658 default_node_network_controller.go:519] Node connection status = connected I0429 19:39:01.755478 3658 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0429 19:39:01.898689 3658 default_node_network_controller.go:519] Node connection status = connected I0429 19:39:01.916822 3658 bridgeconfig.go:537] Gateway is ready I0429 19:39:01.916842 3658 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0429 19:39:01.928970 3658 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.121. Updating OVN encap IP. I0429 19:39:01.934544 3658 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.121 - it is already configured I0429 19:39:01.934558 3658 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.121/24:{} 172.20.0.1/32:{}]. Updating annotations. I0429 19:39:01.934766 3658 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.121/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-121.ec2.internal","mac-address":"12:11:6d:78:9a:a1","ip-addresses":["10.0.0.121/24"],"ip-address":"10.0.0.121/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:3f771be6-3db9-4bf0-9376-8585c6b4eb85 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.121/24"}] on node ip-10-0-0-121.ec2.internal I0429 19:39:01.950908 3658 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-121.ec2.internal I0429 19:39:01.950922 3658 egressfirewall.go:723] Syncing node "ip-10-0-0-121.ec2.internal" for egress firewall I0429 19:39:01.950927 3658 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-121.ec2.internal: 24.789µs I0429 19:39:01.950978 3658 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-121.ec2.internal in network "default" I0429 19:39:01.951023 3658 master.go:548] Adding or Updating local node "ip-10-0-0-121.ec2.internal" for network "default" I0429 19:39:01.951037 3658 node_tracker.go:169] Node ip-10-0-0-121.ec2.internal switch + router changed, syncing services in network "default" I0429 19:39:01.951051 3658 services_controller.go:574] Full service sync requested for network=default W0429 19:39:01.951193 3658 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 19:39:01.951520 3658 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.0.121 I0429 19:39:01.951662 3658 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-121.ec2.internal I0429 19:39:01.956064 3658 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-121.ec2.internal. Updating no re-route policies I0429 19:39:01.956153 3658 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-555bt I0429 19:39:01.956169 3658 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-555bt I0429 19:39:01.956166 3658 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-65vpk I0429 19:39:01.956182 3658 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-65vpk I0429 19:39:01.956186 3658 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-d4vk5 I0429 19:39:01.956191 3658 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-dzgf8 I0429 19:39:01.956199 3658 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-65vpk after 0 failed attempt(s) I0429 19:39:01.956202 3658 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-d4vk5 I0429 19:39:01.956204 3658 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-c9d65 I0429 19:39:01.956200 3658 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-555bt] creating logical port openshift-network-diagnostics_network-check-target-555bt for pod on switch ip-10-0-0-121.ec2.internal I0429 19:39:01.956214 3658 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-121.ec2.internal I0429 19:39:01.956218 3658 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-c9d65 I0429 19:39:01.956222 3658 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-121.ec2.internal I0429 19:39:01.956214 3658 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-kg6zw I0429 19:39:01.956229 3658 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-d4vk5 after 0 failed attempt(s) I0429 19:39:01.956227 3658 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-lc64d I0429 19:39:01.956234 3658 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-121.ec2.internal after 0 failed attempt(s) I0429 19:39:01.956236 3658 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-wsr6s I0429 19:39:01.956243 3658 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-8c87f I0429 19:39:01.956245 3658 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-jwc2p I0429 19:39:01.956243 3658 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-lc64d I0429 19:39:01.956249 3658 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-8c87f I0429 19:39:01.956252 3658 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-jwc2p I0429 19:39:01.956252 3658 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-121.ec2.internal I0429 19:39:01.956251 3658 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-wsr6s I0429 19:39:01.956264 3658 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-8c87f after 0 failed attempt(s) I0429 19:39:01.956266 3658 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-121.ec2.internal I0429 19:39:01.956270 3658 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-jwc2p after 0 failed attempt(s) I0429 19:39:01.956288 3658 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-121.ec2.internal after 0 failed attempt(s) I0429 19:39:01.956300 3658 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-wsr6s] creating logical port openshift-multus_network-metrics-daemon-wsr6s for pod on switch ip-10-0-0-121.ec2.internal I0429 19:39:01.956235 3658 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-kg6zw I0429 19:39:01.956352 3658 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-kg6zw after 0 failed attempt(s) I0429 19:39:01.956193 3658 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-tjdqz I0429 19:39:01.956375 3658 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-tjdqz I0429 19:39:01.956380 3658 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-lc64d after 0 failed attempt(s) I0429 19:39:01.956387 3658 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-tjdqz after 0 failed attempt(s) I0429 19:39:01.956235 3658 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-c9d65 after 0 failed attempt(s) I0429 19:39:01.956213 3658 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-dzgf8 I0429 19:39:01.956409 3658 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-dzgf8 after 0 failed attempt(s) I0429 19:39:01.956927 3658 pods.go:271] [openshift-network-diagnostics/network-check-target-555bt] addLogicalPort took 732.097µs, libovsdb time 241.71µs I0429 19:39:01.956940 3658 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-555bt after 0 failed attempt(s) I0429 19:39:01.957008 3658 pods.go:271] [openshift-multus/network-metrics-daemon-wsr6s] addLogicalPort took 717.679µs, libovsdb time 273.973µs I0429 19:39:01.957021 3658 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-wsr6s after 0 failed attempt(s) I0429 19:39:01.968312 3658 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0429 19:39:01.981207 3658 iptables.go:108] Creating table: mangle chain: OUTPUT I0429 19:39:01.984991 3658 iptables.go:108] Creating table: nat chain: OUTPUT I0429 19:39:02.032635 3658 iptables.go:108] Creating table: nat chain: PREROUTING I0429 19:39:02.086948 3658 iptables.go:108] Creating table: filter chain: FORWARD I0429 19:39:02.105206 3658 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0429 19:39:02.105324 3658 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0429 19:39:02.105379 3658 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0429 19:39:02.105468 3658 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0429 19:39:02.105538 3658 gateway_init.go:325] Gateway and management port readiness took 1.712293226s I0429 19:39:02.105568 3658 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0429 19:39:02.105606 3658 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0429 19:39:02.105630 3658 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0429 19:39:02.105641 3658 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0429 19:39:02.105718 3658 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0429 19:39:02.105811 3658 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 64.217µs I0429 19:39:02.106076 3658 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0429 19:39:02.106105 3658 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 19.691µs I0429 19:39:02.106114 3658 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 38.014µs I0429 19:39:02.106119 3658 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0429 19:39:02.106129 3658 nad_controller.go:379] [node-nad-controller NAD controller]: started I0429 19:39:02.106137 3658 default_node_network_controller.go:899] Starting the default node network controller I0429 19:39:02.106141 3658 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0429 19:39:02.106217 3658 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 12.898µs I0429 19:39:02.106305 3658 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0429 19:39:02.106313 3658 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0429 19:39:02.106337 3658 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a93050 0xc000325b20 0xc000325b90 0xc000325c00 0xc000325c70 0xc000325ce0 0xc000325d50 0xc000aad400 0xc000325dc0 0xc000325e30 0xc000aac320 0xc0064793b0 0xc000325ea0 map[0x2ff9080:0xc0029a4540 0x2ff95c0:0xc0029a46c0 0x2ffa2e0:0xc0029a4480 0x2ffa580:0xc0029a44e0 0x2ffad60:0xc0029a45a0 0x2ffb000:0xc0029a4600 0x2ffb2a0:0xc0029a4660 0x2ffba80:0xc0029a4720 0x2ffbd20:0xc004d989c0 0x3026980:0xc001b28420 0x3026d20:0xc001b28480 0x3027800:0xc000af5680 0x3029160:0xc001b28000 0x30298a0:0xc00241e600 0x302a720:0xc000b3a480] 0xc000aae2a0 0} I0429 19:39:02.106393 3658 gateway.go:151] Starting gateway service sync I0429 19:39:02.106851 3658 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0429 19:39:02.109228 3658 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0429 19:39:02.141742 3658 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0429 19:39:02.143151 3658 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0429 19:39:02.144422 3658 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0429 19:39:02.179846 3658 gateway.go:168] Gateway service sync done. Time taken: 73.438736ms I0429 19:39:02.180316 3658 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30707 I0429 19:39:02.180662 3658 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a93050 0xc000325b20 0xc000325b90 0xc000325c00 0xc000325c70 0xc000325ce0 0xc000325d50 0xc000aad400 0xc000325dc0 0xc000325e30 0xc000aac320 0xc0064793b0 0xc000325ea0 map[0x2ff9080:0xc0029a4540 0x2ff95c0:0xc0029a46c0 0x2ffa2e0:0xc0029a4480 0x2ffa580:0xc0029a44e0 0x2ffad60:0xc0029a45a0 0x2ffb000:0xc0029a4600 0x2ffb2a0:0xc0029a4660 0x2ffba80:0xc0029a4720 0x2ffbd20:0xc004d989c0 0x3026980:0xc001b28420 0x3026d20:0xc001b28480 0x3027800:0xc000af5680 0x3029160:0xc001b28000 0x30298a0:0xc00241e600 0x302a720:0xc000b3a480] 0xc000aae2a0 0} I0429 19:39:02.181138 3658 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a93050 0xc000325b20 0xc000325b90 0xc000325c00 0xc000325c70 0xc000325ce0 0xc000325d50 0xc000aad400 0xc000325dc0 0xc000325e30 0xc000aac320 0xc0064793b0 0xc000325ea0 map[0x2ff9080:0xc0029a4540 0x2ff95c0:0xc0029a46c0 0x2ffa2e0:0xc0029a4480 0x2ffa580:0xc0029a44e0 0x2ffad60:0xc0029a45a0 0x2ffb000:0xc0029a4600 0x2ffb2a0:0xc0029a4660 0x2ffba80:0xc0029a4720 0x2ffbd20:0xc004d989c0 0x3026980:0xc001b28420 0x3026d20:0xc001b28480 0x3027800:0xc000af5680 0x3029160:0xc001b28000 0x30298a0:0xc00241e600 0x302a720:0xc000b3a480] 0xc000aae2a0 0} I0429 19:39:02.181179 3658 gateway.go:532] Reconciling gateway with updates I0429 19:39:02.187110 3658 gateway.go:339] Spawning Conntrack Rule Check Thread I0429 19:39:02.187172 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0429 19:39:02.187194 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0429 19:39:02.187190 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0429 19:39:02.187191 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0429 19:39:02.187209 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0429 19:39:02.187213 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0429 19:39:02.187216 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0429 19:39:02.187219 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0429 19:39:02.187228 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0429 19:39:02.187236 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0429 19:39:02.187251 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0429 19:39:02.187267 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0429 19:39:02.187276 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0429 19:39:02.187274 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0429 19:39:02.187251 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0429 19:39:02.187289 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0429 19:39:02.187291 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0429 19:39:02.187307 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0429 19:39:02.187315 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0429 19:39:02.187309 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0429 19:39:02.187326 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0429 19:39:02.187330 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0429 19:39:02.187331 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0429 19:39:02.187339 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0429 19:39:02.187343 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0429 19:39:02.187345 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0429 19:39:02.187347 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0429 19:39:02.187345 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0429 19:39:02.187355 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0429 19:39:02.187356 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0429 19:39:02.187361 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0429 19:39:02.187364 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0429 19:39:02.187366 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0429 19:39:02.187378 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0429 19:39:02.187382 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0429 19:39:02.187381 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0429 19:39:02.187171 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0429 19:39:02.187391 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0429 19:39:02.187388 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0429 19:39:02.187401 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0429 19:39:02.187397 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0429 19:39:02.187399 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0429 19:39:02.187405 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0429 19:39:02.187407 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0429 19:39:02.187408 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0429 19:39:02.187417 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0429 19:39:02.187414 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0429 19:39:02.187418 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0429 19:39:02.187421 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0429 19:39:02.187437 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0429 19:39:02.187200 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0429 19:39:02.187452 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0429 19:39:02.187438 3658 port_claim.go:71] Svc openshift-ingress/router-default: port 31755 is already open, no action needed I0429 19:39:02.187464 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 19:39:02.187464 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0429 19:39:02.187472 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0429 19:39:02.187473 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0429 19:39:02.187477 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0429 19:39:02.187499 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0429 19:39:02.187503 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0429 19:39:02.187507 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0429 19:39:02.187505 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0429 19:39:02.187508 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0429 19:39:02.187513 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0429 19:39:02.187511 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0429 19:39:02.187521 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0429 19:39:02.187523 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0429 19:39:02.187528 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0429 19:39:02.187533 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0429 19:39:02.187536 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0429 19:39:02.187543 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0429 19:39:02.187547 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0429 19:39:02.187554 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0429 19:39:02.187555 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0429 19:39:02.187519 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0429 19:39:02.187594 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0429 19:39:02.187610 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0429 19:39:02.187620 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0429 19:39:02.187349 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0429 19:39:02.187633 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0429 19:39:02.187640 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0429 19:39:02.187681 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0429 19:39:02.187368 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0429 19:39:02.187695 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0429 19:39:02.187712 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0429 19:39:02.187715 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0429 19:39:02.187730 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0429 19:39:02.187740 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0429 19:39:02.187772 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0429 19:39:02.187718 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0429 19:39:02.187368 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0429 19:39:02.187359 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0429 19:39:02.187839 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0429 19:39:02.187865 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0429 19:39:02.187373 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0429 19:39:02.187875 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0429 19:39:02.187894 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0429 19:39:02.187385 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0429 19:39:02.187386 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0429 19:39:02.187907 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0429 19:39:02.187925 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0429 19:39:02.187388 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 19:39:02.187938 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 19:39:02.187955 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0429 19:39:02.187404 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0429 19:39:02.187978 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0429 19:39:02.187983 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0429 19:39:02.187452 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0429 19:39:02.187994 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0429 19:39:02.187472 3658 port_claim.go:71] Svc openshift-ingress/router-default: port 32516 is already open, no action needed I0429 19:39:02.187474 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0429 19:39:02.188059 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0429 19:39:02.187475 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 19:39:02.187471 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0429 19:39:02.188111 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0429 19:39:02.187503 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0429 19:39:02.188135 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0429 19:39:02.187503 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0429 19:39:02.188193 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0429 19:39:02.188198 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0429 19:39:02.187507 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0429 19:39:02.188205 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0429 19:39:02.188208 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0429 19:39:02.187519 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0429 19:39:02.188215 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0429 19:39:02.187526 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0429 19:39:02.187283 3658 default_node_network_controller.go:944] Gateway and management port readiness took 80.960396ms I0429 19:39:02.188357 3658 node_ip_handler_linux.go:160] Node IP manager is running W0429 19:39:02.227148 3658 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0429 19:39:02.227364 3658 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0429 19:39:02.227587 3658 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0429 19:39:02.234860 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0429 19:39:02.255186 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0429 19:39:02.255248 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0429 19:39:02.255272 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0429 19:39:02.255279 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0429 19:39:02.255285 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0429 19:39:02.255295 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0429 19:39:02.255302 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0429 19:39:04.158012 3658 default_node_network_controller.go:1380] Starting node controller node sync W0429 19:39:04.158070 3658 egressip_healthcheck.go:76] Health checking using insecure connection I0429 19:39:04.158115 3658 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0429 19:39:04.170173 3658 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 12.140801ms I0429 19:39:04.170285 3658 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-196.ec2.internal", IP: 10.0.1.196 to PMTUD blocking rules I0429 19:39:04.170296 3658 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-196.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0429 19:39:04.176264 3658 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0429 19:39:04.176549 3658 egressservice_node.go:93] Setting up event handlers for Egress Services I0429 19:39:04.176667 3658 egressservice_node.go:183] Starting Egress Services Controller I0429 19:39:04.176687 3658 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0429 19:39:04.176700 3658 shared_informer.go:356] "Caches are synced" controller="egressservices" I0429 19:39:04.176710 3658 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0429 19:39:04.176719 3658 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0429 19:39:04.176733 3658 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0429 19:39:04.176744 3658 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0429 19:39:04.176751 3658 egressservice_node.go:197] Repairing Egress Services I0429 19:39:04.187036 3658 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0429 19:39:04.187052 3658 external_controller.go:278] Starting Admin Policy Based Route Controller I0429 19:39:04.187282 3658 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0429 19:39:04.187297 3658 link_network_manager.go:119] Link manager is running I0429 19:39:04.187306 3658 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0429 19:39:04.187479 3658 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0429 19:39:04.187687 3658 vrf_manager.go:128] VRF manager is running I0429 19:39:04.187866 3658 node_controller_manager.go:437] Removing flows to drop GARP I0429 19:39:04.187876 3658 gateway.go:532] Reconciling gateway with updates I0429 19:39:04.215506 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0429 19:39:04.215522 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0429 19:39:04.215528 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0429 19:39:04.215538 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0429 19:39:04.215546 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0429 19:39:04.215543 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0429 19:39:04.215506 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0429 19:39:04.215552 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0429 19:39:04.215558 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0429 19:39:04.215564 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0429 19:39:04.215562 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0429 19:39:04.215567 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0429 19:39:04.215583 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0429 19:39:04.215584 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0429 19:39:04.215589 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0429 19:39:04.215594 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0429 19:39:04.215587 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0429 19:39:04.215601 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0429 19:39:04.215542 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0429 19:39:04.215607 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0429 19:39:04.215608 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0429 19:39:04.215517 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0429 19:39:04.215607 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0429 19:39:04.215625 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0429 19:39:04.215626 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 19:39:04.215627 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0429 19:39:04.215633 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0429 19:39:04.215632 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0429 19:39:04.215636 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 19:39:04.215640 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0429 19:39:04.215507 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0429 19:39:04.215639 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0429 19:39:04.215647 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0429 19:39:04.215652 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0429 19:39:04.215652 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0429 19:39:04.215651 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0429 19:39:04.215657 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0429 19:39:04.215658 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0429 19:39:04.215657 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0429 19:39:04.215665 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0429 19:39:04.215667 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0429 19:39:04.215669 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0429 19:39:04.215674 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0429 19:39:04.215678 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0429 19:39:04.215678 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0429 19:39:04.215684 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0429 19:39:04.215689 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0429 19:39:04.215693 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0429 19:39:04.215697 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0429 19:39:04.215698 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0429 19:39:04.215700 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0429 19:39:04.215695 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0429 19:39:04.215702 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0429 19:39:04.215704 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0429 19:39:04.215715 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0429 19:39:04.215715 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0429 19:39:04.215717 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0429 19:39:04.215732 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0429 19:39:04.215701 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0429 19:39:04.215640 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0429 19:39:04.215788 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0429 19:39:04.215625 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0429 19:39:04.215811 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0429 19:39:04.215851 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0429 19:39:04.215859 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0429 19:39:04.215631 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0429 19:39:04.215554 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0429 19:39:04.215872 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0429 19:39:04.215566 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0429 19:39:04.215882 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0429 19:39:04.215634 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0429 19:39:04.215542 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0429 19:39:04.215945 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0429 19:39:04.215541 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0429 19:39:04.215969 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0429 19:39:04.215554 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0429 19:39:04.216051 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0429 19:39:04.215588 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0429 19:39:04.216065 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0429 19:39:04.216073 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0429 19:39:04.215597 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0429 19:39:04.216105 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0429 19:39:04.215610 3658 port_claim.go:71] Svc openshift-ingress/router-default: port 31755 is already open, no action needed I0429 19:39:04.216153 3658 port_claim.go:71] Svc openshift-ingress/router-default: port 32516 is already open, no action needed I0429 19:39:04.215612 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0429 19:39:04.215612 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0429 19:39:04.216216 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0429 19:39:04.215516 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0429 19:39:04.216240 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0429 19:39:04.215507 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0429 19:39:04.216271 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0429 19:39:04.215616 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0429 19:39:04.216298 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0429 19:39:04.215622 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0429 19:39:04.216311 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0429 19:39:04.215529 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 19:39:04.216327 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 19:39:04.215636 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0429 19:39:04.216374 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0429 19:39:04.215641 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0429 19:39:04.215643 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0429 19:39:04.215645 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0429 19:39:04.215644 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0429 19:39:04.215647 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0429 19:39:04.216419 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0429 19:39:04.215658 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0429 19:39:04.215660 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0429 19:39:04.216442 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0429 19:39:04.215657 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0429 19:39:04.216451 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0429 19:39:04.215679 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0429 19:39:04.215692 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0429 19:39:04.215532 3658 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0429 19:39:04.216477 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0429 19:39:04.215699 3658 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0429 19:39:04.215715 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0429 19:39:04.235938 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0429 19:39:04.235946 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0429 19:39:04.235958 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0429 19:39:04.235965 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0429 19:39:04.235970 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0429 19:39:04.235976 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0429 19:39:04.253447 3658 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-37.ec2.internal in network "default" I0429 19:39:04.253462 3658 default_network_controller.go:1025] Node "ip-10-0-2-37.ec2.internal" in remote zone "ip-10-0-2-37.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 19:39:04.253503 3658 node_tracker.go:235] Node ip-10-0-2-37.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-37.ec2.internal" I0429 19:39:04.253519 3658 node_tracker.go:169] Node ip-10-0-2-37.ec2.internal switch + router changed, syncing services in network "default" I0429 19:39:04.253534 3658 services_controller.go:574] Full service sync requested for network=default I0429 19:39:04.279211 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0429 19:39:04.279218 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0429 19:39:04.279236 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0429 19:39:04.320413 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0429 19:39:04.320420 3658 default_network_controller.go:1025] Node "ip-10-0-2-37.ec2.internal" in remote zone "ip-10-0-2-37.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 19:39:04.320437 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0429 19:39:04.320461 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0429 19:39:04.320498 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0429 19:39:04.320508 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0429 19:39:04.320519 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0429 19:39:04.320533 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0429 19:39:04.320543 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0429 19:39:04.320549 3658 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0429 19:39:04.379615 3658 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-37.ec2.internal in network "default" I0429 19:39:04.379650 3658 default_network_controller.go:1025] Node "ip-10-0-2-37.ec2.internal" in remote zone "ip-10-0-2-37.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 19:39:04.379672 3658 node_tracker.go:169] Node ip-10-0-2-37.ec2.internal switch + router changed, syncing services in network "default" I0429 19:39:04.379685 3658 services_controller.go:574] Full service sync requested for network=default I0429 19:39:04.380834 3658 zone_ic_handler.go:195] Time taken to create transit switch: 282.162µs I0429 19:39:04.380853 3658 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-37.ec2.internal for the network default I0429 19:39:04.381895 3658 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-37.ec2.internal" on network "default" took: 1.358081ms I0429 19:39:05.911369 3658 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-121.ec2.internal I0429 19:39:05.911399 3658 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-121.ec2.internal: 36.814µs I0429 19:39:05.939688 3658 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-9b94754f9-lqtbq] creating logical port openshift-image-registry_image-registry-9b94754f9-lqtbq for pod on switch ip-10-0-0-121.ec2.internal I0429 19:39:05.939805 3658 kube.go:256] Updating pod openshift-image-registry/image-registry-9b94754f9-lqtbq I0429 19:39:05.960356 3658 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-cx8dw] creating logical port openshift-dns_dns-default-cx8dw for pod on switch ip-10-0-0-121.ec2.internal I0429 19:39:05.960714 3658 kube.go:256] Updating pod openshift-dns/dns-default-cx8dw I0429 19:39:05.963105 3658 pod.go:62] [openshift-image-registry/image-registry-9b94754f9-lqtbq] pod update took 23.331249ms I0429 19:39:05.963140 3658 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-9b94754f9-lqtbq] addLogicalPort annotation time took 23.370083ms I0429 19:39:05.964399 3658 pods.go:271] [openshift-image-registry/image-registry-9b94754f9-lqtbq] addLogicalPort took 24.72202ms, libovsdb time 963.663µs I0429 19:39:05.977547 3658 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-kdkjz] creating logical port openshift-ingress-canary_ingress-canary-kdkjz for pod on switch ip-10-0-0-121.ec2.internal I0429 19:39:05.977638 3658 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-kdkjz I0429 19:39:05.980317 3658 pod.go:62] [openshift-dns/dns-default-cx8dw] pod update took 19.630545ms I0429 19:39:05.980337 3658 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-cx8dw] addLogicalPort annotation time took 19.656294ms I0429 19:39:05.981697 3658 pods.go:271] [openshift-dns/dns-default-cx8dw] addLogicalPort took 21.354919ms, libovsdb time 1.046339ms I0429 19:39:05.996227 3658 pod.go:62] [openshift-ingress-canary/ingress-canary-kdkjz] pod update took 18.616799ms I0429 19:39:05.996245 3658 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-kdkjz] addLogicalPort annotation time took 18.640598ms I0429 19:39:05.997249 3658 pods.go:271] [openshift-ingress-canary/ingress-canary-kdkjz] addLogicalPort took 19.720096ms, libovsdb time 729.676µs I0429 19:39:06.395442 3658 cni.go:452] [openshift-ingress-canary/ingress-canary-kdkjz c9837545d0a37df6406c1b1a2a9043a62b73f6a6455b96701f292612ebbfd3dc network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-kdkjz c9837545d0a37df6406c1b1a2a9043a62b73f6a6455b96701f292612ebbfd3dc network default NAD default NAD key ] I0429 19:39:06.444450 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-kdkjz, hostIfaceName: c9837545d0a37df, network: default, NAD default, SandboxID: "c9837545d0a37df6406c1b1a2a9043a62b73f6a6455b96701f292612ebbfd3dc", PCI device ID: , UID: "043af919-d915-4589-81b1-8d4d5e77b759", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0429 19:39:06.498012 3658 cni.go:473] [openshift-ingress-canary/ingress-canary-kdkjz c9837545d0a37df6406c1b1a2a9043a62b73f6a6455b96701f292612ebbfd3dc network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-kdkjz c9837545d0a37df6406c1b1a2a9043a62b73f6a6455b96701f292612ebbfd3dc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:80:67:d4:7e:04\",\"name\":\"c9837545d0a37df\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/629cf974-5fac-4ea0-bbe5-cfccad0bc9a4\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:39:06.934930 3658 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-37.ec2.internal in network "default" I0429 19:39:06.934963 3658 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-37.ec2.internal I0429 19:39:06.934961 3658 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-37.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0429 19:39:06.934978 3658 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-37.ec2.internal: 19.627µs I0429 19:39:06.934984 3658 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-37.ec2.internal", IP: 10.0.2.37 to PMTUD blocking rules I0429 19:39:06.934991 3658 egressfirewall.go:723] Syncing node "ip-10-0-2-37.ec2.internal" for egress firewall I0429 19:39:06.935002 3658 node_tracker.go:169] Node ip-10-0-2-37.ec2.internal switch + router changed, syncing services in network "default" I0429 19:39:06.935017 3658 services_controller.go:574] Full service sync requested for network=default I0429 19:39:06.941345 3658 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-37.ec2.internal. Updating no re-route policies I0429 19:39:06.950971 3658 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-37.ec2.internal", IP: 10.0.2.37 to PMTUD blocking rules I0429 19:39:06.950992 3658 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-37.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0429 19:39:09.701930 3658 cni.go:452] [openshift-network-diagnostics/network-check-target-555bt 0835fe917290a2aeaec30ad70a021e73b5b2b94768877c4e999be92d48e571b6 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-555bt 0835fe917290a2aeaec30ad70a021e73b5b2b94768877c4e999be92d48e571b6 network default NAD default NAD key ] I0429 19:39:09.714632 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-555bt, hostIfaceName: 0835fe917290a2a, network: default, NAD default, SandboxID: "0835fe917290a2aeaec30ad70a021e73b5b2b94768877c4e999be92d48e571b6", PCI device ID: , UID: "9e12638f-435f-41bd-b066-e066e893657c", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0429 19:39:09.767835 3658 cni.go:473] [openshift-network-diagnostics/network-check-target-555bt 0835fe917290a2aeaec30ad70a021e73b5b2b94768877c4e999be92d48e571b6 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-555bt 0835fe917290a2aeaec30ad70a021e73b5b2b94768877c4e999be92d48e571b6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:05:96:c4:b0:67\",\"name\":\"0835fe917290a2a\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3a0d784c-7d1e-474b-8422-c7f1f3d851ab\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:39:10.432764 3658 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-37.ec2.internal I0429 19:39:10.432796 3658 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-37.ec2.internal: 40.484µs W0429 19:39:15.814225 3658 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 19:39:17.739608 3658 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 19:39:23.580252 3658 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0429 19:39:29.709278 3658 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq2h8p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:39:29.709905 3658 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq2h8p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:39:29.709938 3658 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq2h8p of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:39:29.709956 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0429 19:39:29.709969 3658 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq2h8p of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:39:29.709974 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0429 19:39:30.635071 3658 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq2h8p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:39:30.635093 3658 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq2h8p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:39:30.635111 3658 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq2h8p of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:39:30.635120 3658 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dq2h8p of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:39:31.200627 3658 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0429 19:39:31.211564 3658 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0429 19:39:46.819782 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0429 19:40:24.127046 3658 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6m728] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-6m728 for pod on switch ip-10-0-0-121.ec2.internal I0429 19:40:24.127135 3658 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6m728 I0429 19:40:24.139157 3658 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-7b77f48c7f-4tcbg] creating logical port openshift-ingress_router-default-7b77f48c7f-4tcbg for pod on switch ip-10-0-0-121.ec2.internal I0429 19:40:24.139250 3658 kube.go:256] Updating pod openshift-ingress/router-default-7b77f48c7f-4tcbg I0429 19:40:24.142624 3658 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-557b65ff59-q4kpj] creating logical port openshift-ingress_router-default-557b65ff59-q4kpj for pod on switch ip-10-0-0-121.ec2.internal I0429 19:40:24.142738 3658 kube.go:256] Updating pod openshift-ingress/router-default-557b65ff59-q4kpj I0429 19:40:24.145295 3658 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6m728] pod update took 18.185651ms I0429 19:40:24.145315 3658 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6m728] addLogicalPort annotation time took 18.210869ms I0429 19:40:24.146505 3658 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6m728] addLogicalPort took 19.478207ms, libovsdb time 887.479µs I0429 19:40:24.161010 3658 pod.go:62] [openshift-ingress/router-default-7b77f48c7f-4tcbg] pod update took 21.790889ms I0429 19:40:24.161030 3658 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-7b77f48c7f-4tcbg] addLogicalPort annotation time took 21.81596ms I0429 19:40:24.162104 3658 pods.go:271] [openshift-ingress/router-default-7b77f48c7f-4tcbg] addLogicalPort took 22.964108ms, libovsdb time 826.926µs I0429 19:40:24.165727 3658 pod.go:62] [openshift-ingress/router-default-557b65ff59-q4kpj] pod update took 23.011225ms I0429 19:40:24.165747 3658 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-557b65ff59-q4kpj] addLogicalPort annotation time took 23.035789ms I0429 19:40:24.166687 3658 pods.go:271] [openshift-ingress/router-default-557b65ff59-q4kpj] addLogicalPort took 24.076972ms, libovsdb time 708.667µs I0429 19:40:24.236675 3658 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-4btzn] creating logical port openshift-network-diagnostics_network-check-source-84fd6d9d54-4btzn for pod on switch ip-10-0-0-121.ec2.internal I0429 19:40:24.236785 3658 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-84fd6d9d54-4btzn I0429 19:40:24.257239 3658 pod.go:62] [openshift-network-diagnostics/network-check-source-84fd6d9d54-4btzn] pod update took 20.48047ms I0429 19:40:24.257268 3658 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-4btzn] addLogicalPort annotation time took 20.515128ms I0429 19:40:24.258272 3658 pods.go:271] [openshift-network-diagnostics/network-check-source-84fd6d9d54-4btzn] addLogicalPort took 21.611035ms, libovsdb time 698.862µs I0429 19:40:24.341135 3658 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-zvtgf] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-zvtgf for pod on switch ip-10-0-0-121.ec2.internal I0429 19:40:24.341256 3658 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-zvtgf I0429 19:40:24.369003 3658 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-vqr72] creating logical port openshift-service-ca-operator_service-ca-operator-bc9564db4-vqr72 for pod on switch ip-10-0-0-121.ec2.internal I0429 19:40:24.369086 3658 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-bc9564db4-vqr72 I0429 19:40:24.378950 3658 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-zvtgf] pod update took 37.726833ms I0429 19:40:24.378971 3658 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-zvtgf] addLogicalPort annotation time took 37.75103ms I0429 19:40:24.379896 3658 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-zvtgf] addLogicalPort took 38.781671ms, libovsdb time 668.148µs I0429 19:40:24.401106 3658 pod.go:62] [openshift-service-ca-operator/service-ca-operator-bc9564db4-vqr72] pod update took 32.037304ms I0429 19:40:24.401130 3658 base_network_controller_pods.go:951] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-vqr72] addLogicalPort annotation time took 32.06839ms I0429 19:40:24.402298 3658 pods.go:271] [openshift-service-ca-operator/service-ca-operator-bc9564db4-vqr72] addLogicalPort took 33.312638ms, libovsdb time 857.701µs I0429 19:40:24.482113 3658 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6m728 0f8fe4e9e90116623679fd754562145e92db477c618703a224e4f7f146a30e4f network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6m728 0f8fe4e9e90116623679fd754562145e92db477c618703a224e4f7f146a30e4f network default NAD default NAD key ] I0429 19:40:24.492294 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-6m728, hostIfaceName: 0f8fe4e9e901166, network: default, NAD default, SandboxID: "0f8fe4e9e90116623679fd754562145e92db477c618703a224e4f7f146a30e4f", PCI device ID: , UID: "c0efef91-e759-42c2-b12d-92632f1e14f0", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0429 19:40:24.538938 3658 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6m728 0f8fe4e9e90116623679fd754562145e92db477c618703a224e4f7f146a30e4f network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6m728 0f8fe4e9e90116623679fd754562145e92db477c618703a224e4f7f146a30e4f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:ca:c2:28:72:22\",\"name\":\"0f8fe4e9e901166\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e5cbca01-a232-4093-af71-2da981ee2168\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:40:24.589479 3658 cni.go:452] [openshift-network-diagnostics/network-check-source-84fd6d9d54-4btzn 2b6b2baa26e9856a42cd5f65c063fcb41335fbe32d890897a012b63edc27e86a network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-4btzn 2b6b2baa26e9856a42cd5f65c063fcb41335fbe32d890897a012b63edc27e86a network default NAD default NAD key ] I0429 19:40:24.599616 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-84fd6d9d54-4btzn, hostIfaceName: 2b6b2baa26e9856, network: default, NAD default, SandboxID: "2b6b2baa26e9856a42cd5f65c063fcb41335fbe32d890897a012b63edc27e86a", PCI device ID: , UID: "ecfd0fa8-c3a5-4cb9-bb4e-b9d14bc16958", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0429 19:40:24.646892 3658 cni.go:473] [openshift-network-diagnostics/network-check-source-84fd6d9d54-4btzn 2b6b2baa26e9856a42cd5f65c063fcb41335fbe32d890897a012b63edc27e86a network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-4btzn 2b6b2baa26e9856a42cd5f65c063fcb41335fbe32d890897a012b63edc27e86a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:a4:e3:83:0a:0a\",\"name\":\"2b6b2baa26e9856\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6be1a402-e5dc-43af-9a11-fcf31ba64558\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:40:24.733610 3658 cni.go:452] [openshift-service-ca-operator/service-ca-operator-bc9564db4-vqr72 efa74a01a24ff7294ebd1feb9faea488f71ae662a1dde889ead0079383d4dcba network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-vqr72 efa74a01a24ff7294ebd1feb9faea488f71ae662a1dde889ead0079383d4dcba network default NAD default NAD key ] I0429 19:40:24.745574 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-bc9564db4-vqr72, hostIfaceName: efa74a01a24ff72, network: default, NAD default, SandboxID: "efa74a01a24ff7294ebd1feb9faea488f71ae662a1dde889ead0079383d4dcba", PCI device ID: , UID: "83adc342-d417-457d-a219-db27e8ccecb8", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0429 19:40:24.793285 3658 cni.go:473] [openshift-service-ca-operator/service-ca-operator-bc9564db4-vqr72 efa74a01a24ff7294ebd1feb9faea488f71ae662a1dde889ead0079383d4dcba network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-vqr72 efa74a01a24ff7294ebd1feb9faea488f71ae662a1dde889ead0079383d4dcba network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:e5:ee:79:fe:d0\",\"name\":\"efa74a01a24ff72\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/be7a7be0-8458-4614-8095-6e814b11573e\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:40:27.900290 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0429 19:40:29.203153 3658 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0429 19:40:29.204173 3658 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 998.509µs I0429 19:40:29.225517 3658 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0429 19:40:30.107248 3658 namespace.go:98] [openshift-service-ca] adding namespace I0429 19:40:30.108220 3658 namespace.go:102] [openshift-service-ca] adding namespace took 944.971µs I0429 19:40:30.126116 3658 namespace.go:142] [openshift-service-ca] updating namespace W0429 19:40:38.689349 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0429 19:40:45.701783 3658 cni.go:452] [openshift-multus/network-metrics-daemon-wsr6s b8810605f51f07b795cacd3a6366d85f3934c76f34c9a92fef16ae772305da55 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-wsr6s b8810605f51f07b795cacd3a6366d85f3934c76f34c9a92fef16ae772305da55 network default NAD default NAD key ] I0429 19:40:45.712511 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-wsr6s, hostIfaceName: b8810605f51f07b, network: default, NAD default, SandboxID: "b8810605f51f07b795cacd3a6366d85f3934c76f34c9a92fef16ae772305da55", PCI device ID: , UID: "b7f4d5d6-5f09-4f7f-8542-b45f2fd06ebb", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0429 19:40:45.759712 3658 cni.go:473] [openshift-multus/network-metrics-daemon-wsr6s b8810605f51f07b795cacd3a6366d85f3934c76f34c9a92fef16ae772305da55 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-wsr6s b8810605f51f07b795cacd3a6366d85f3934c76f34c9a92fef16ae772305da55 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:b2:4a:75:4c:57\",\"name\":\"b8810605f51f07b\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/35748dcc-d651-4b6f-acf0-f659e09e8c09\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:40:54.543605 3658 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-v4grd] creating logical port openshift-insights_insights-runtime-extractor-v4grd for pod on switch ip-10-0-0-121.ec2.internal I0429 19:40:54.543741 3658 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-v4grd I0429 19:40:54.570835 3658 pod.go:62] [openshift-insights/insights-runtime-extractor-v4grd] pod update took 27.125662ms I0429 19:40:54.570861 3658 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-v4grd] addLogicalPort annotation time took 27.154157ms I0429 19:40:54.571891 3658 pods.go:271] [openshift-insights/insights-runtime-extractor-v4grd] addLogicalPort took 28.304897ms, libovsdb time 750.79µs I0429 19:40:54.898556 3658 cni.go:452] [openshift-insights/insights-runtime-extractor-v4grd 5e9344ae1153ad94aff1c4116215f83230850966299a7c7ed83cbf5ee3ccb0c8 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-v4grd 5e9344ae1153ad94aff1c4116215f83230850966299a7c7ed83cbf5ee3ccb0c8 network default NAD default NAD key ] I0429 19:40:54.908038 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-v4grd, hostIfaceName: 5e9344ae1153ad9, network: default, NAD default, SandboxID: "5e9344ae1153ad94aff1c4116215f83230850966299a7c7ed83cbf5ee3ccb0c8", PCI device ID: , UID: "387e7dac-96f8-49b4-81d3-c04c0b0a0bb9", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0429 19:40:54.956623 3658 cni.go:473] [openshift-insights/insights-runtime-extractor-v4grd 5e9344ae1153ad94aff1c4116215f83230850966299a7c7ed83cbf5ee3ccb0c8 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-v4grd 5e9344ae1153ad94aff1c4116215f83230850966299a7c7ed83cbf5ee3ccb0c8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:a4:c5:47:3a:4e\",\"name\":\"5e9344ae1153ad9\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/88c1e560-7e57-4169-9855-413b172270dc\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:40:55.859260 3658 obj_retry.go:498] Detected object openshift-image-registry/image-registry-b6dd48dcb-7x5lr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:40:55.859943 3658 obj_retry.go:498] Detected object openshift-image-registry/image-registry-b6dd48dcb-7x5lr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:40:55.882241 3658 obj_retry.go:492] Detected object openshift-image-registry/image-registry-b6dd48dcb-7x5lr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:40:55.882267 3658 obj_retry.go:492] Detected object openshift-image-registry/image-registry-b6dd48dcb-7x5lr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:40:55.894286 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-b6dd48dcb-7x5lr I0429 19:40:55.894310 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-b6dd48dcb-7x5lr I0429 19:40:55.993970 3658 obj_retry.go:498] Detected object openshift-image-registry/image-registry-9b94754f9-52vmd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:40:55.994619 3658 obj_retry.go:498] Detected object openshift-image-registry/image-registry-9b94754f9-52vmd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:40:56.009251 3658 obj_retry.go:492] Detected object openshift-image-registry/image-registry-9b94754f9-52vmd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:40:56.009277 3658 obj_retry.go:492] Detected object openshift-image-registry/image-registry-9b94754f9-52vmd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:40:56.019102 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-9b94754f9-52vmd I0429 19:40:56.019128 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-9b94754f9-52vmd I0429 19:40:56.306045 3658 cni.go:452] [openshift-ingress/router-default-557b65ff59-q4kpj 3925aa3ad50038f2d69c5252f98d067027f3093885018ebad2da274a6574140a network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-557b65ff59-q4kpj 3925aa3ad50038f2d69c5252f98d067027f3093885018ebad2da274a6574140a network default NAD default NAD key ] I0429 19:40:56.306053 3658 cni.go:452] [openshift-ingress/router-default-7b77f48c7f-4tcbg f2ccb919cac5a2b1ee76f25b27ba92b6c9613db48b1ee7fb24bb6c49cb343340 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-7b77f48c7f-4tcbg f2ccb919cac5a2b1ee76f25b27ba92b6c9613db48b1ee7fb24bb6c49cb343340 network default NAD default NAD key ] I0429 19:40:56.316475 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-7b77f48c7f-4tcbg, hostIfaceName: f2ccb919cac5a2b, network: default, NAD default, SandboxID: "f2ccb919cac5a2b1ee76f25b27ba92b6c9613db48b1ee7fb24bb6c49cb343340", PCI device ID: , UID: "001a049a-9e6c-4aaf-8ea7-23dbdc73bbd2", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0429 19:40:56.316635 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-557b65ff59-q4kpj, hostIfaceName: 3925aa3ad50038f, network: default, NAD default, SandboxID: "3925aa3ad50038f2d69c5252f98d067027f3093885018ebad2da274a6574140a", PCI device ID: , UID: "ce12bf4e-6554-4c2a-95a2-7c8ad160b90a", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0429 19:40:56.364948 3658 cni.go:473] [openshift-ingress/router-default-7b77f48c7f-4tcbg f2ccb919cac5a2b1ee76f25b27ba92b6c9613db48b1ee7fb24bb6c49cb343340 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-7b77f48c7f-4tcbg f2ccb919cac5a2b1ee76f25b27ba92b6c9613db48b1ee7fb24bb6c49cb343340 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:8f:85:a6:c3:94\",\"name\":\"f2ccb919cac5a2b\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5e4ff56c-5cbe-4a09-8140-32a73665177b\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:40:56.381321 3658 cni.go:473] [openshift-ingress/router-default-557b65ff59-q4kpj 3925aa3ad50038f2d69c5252f98d067027f3093885018ebad2da274a6574140a network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-557b65ff59-q4kpj 3925aa3ad50038f2d69c5252f98d067027f3093885018ebad2da274a6574140a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:08:c2:dd:60:c1\",\"name\":\"3925aa3ad50038f\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4f1d4cde-a2b6-4675-b2c2-6d2ced625751\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:40:56.521742 3658 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-zvtgf 65511838450d170073593af8a2b1b299a00be34091c72fe9b76d9767011cffed network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-zvtgf 65511838450d170073593af8a2b1b299a00be34091c72fe9b76d9767011cffed network default NAD default NAD key ] I0429 19:40:56.532524 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-zvtgf, hostIfaceName: 65511838450d170, network: default, NAD default, SandboxID: "65511838450d170073593af8a2b1b299a00be34091c72fe9b76d9767011cffed", PCI device ID: , UID: "efeac74e-b5d8-417e-845a-4503c9f789a8", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0429 19:40:56.583557 3658 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-zvtgf 65511838450d170073593af8a2b1b299a00be34091c72fe9b76d9767011cffed network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-zvtgf 65511838450d170073593af8a2b1b299a00be34091c72fe9b76d9767011cffed network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:1d:19:69:62:59\",\"name\":\"65511838450d170\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1197ecd6-8f95-4c0f-96c9-5720257e1d6c\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:40:56.864450 3658 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0429 19:40:56.895364 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:56.895385 3658 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:56.895390 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:56.895446 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:56.895462 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:57.001322 3658 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0429 19:40:57.087537 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:57.087556 3658 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:57.087567 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:57.091794 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:57.091810 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:57.118668 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:57.118686 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:57.120230 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:57.120248 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:57.120254 3658 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:57.904983 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:57.905009 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:57.905007 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:57.905023 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:57.905034 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:57.905038 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:57.905136 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:57.905156 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:57.905164 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:57.905183 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:57.905192 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:57.905200 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:58.121501 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:58.121522 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:58.121532 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 19:40:58.123186 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0429 19:40:58.123209 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0429 19:40:58.245367 3658 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0429 19:40:58.359395 3658 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0429 19:41:01.886774 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0429 19:41:04.901201 3658 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0429 19:41:04.901220 3658 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0429 19:41:04.901225 3658 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0429 19:41:06.848002 3658 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-b8587444b-vzt4t] creating logical port openshift-monitoring_openshift-state-metrics-b8587444b-vzt4t for pod on switch ip-10-0-0-121.ec2.internal I0429 19:41:06.848107 3658 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-b8587444b-vzt4t I0429 19:41:06.869128 3658 pod.go:62] [openshift-monitoring/openshift-state-metrics-b8587444b-vzt4t] pod update took 21.048091ms I0429 19:41:06.869151 3658 base_network_controller_pods.go:951] [default/openshift-monitoring/openshift-state-metrics-b8587444b-vzt4t] addLogicalPort annotation time took 21.077357ms I0429 19:41:06.870583 3658 pods.go:271] [openshift-monitoring/openshift-state-metrics-b8587444b-vzt4t] addLogicalPort took 22.606188ms, libovsdb time 1.098206ms I0429 19:41:07.807723 3658 cni.go:452] [openshift-monitoring/openshift-state-metrics-b8587444b-vzt4t 1c0ff7a1d6dc14a3dec88ae94b866244dca2a6593210e6d0249794a0b0e92979 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-vzt4t 1c0ff7a1d6dc14a3dec88ae94b866244dca2a6593210e6d0249794a0b0e92979 network default NAD default NAD key ] I0429 19:41:07.817906 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-b8587444b-vzt4t, hostIfaceName: 1c0ff7a1d6dc14a, network: default, NAD default, SandboxID: "1c0ff7a1d6dc14a3dec88ae94b866244dca2a6593210e6d0249794a0b0e92979", PCI device ID: , UID: "1996e218-f014-4290-a36f-49cc010958b3", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0429 19:41:07.872386 3658 cni.go:473] [openshift-monitoring/openshift-state-metrics-b8587444b-vzt4t 1c0ff7a1d6dc14a3dec88ae94b866244dca2a6593210e6d0249794a0b0e92979 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-vzt4t 1c0ff7a1d6dc14a3dec88ae94b866244dca2a6593210e6d0249794a0b0e92979 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:2b:64:88:0b:67\",\"name\":\"1c0ff7a1d6dc14a\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c82d25ec-0618-4314-a174-596035452052\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:41:09.964258 3658 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-6f5fd6c589-lz88w] creating logical port openshift-monitoring_thanos-querier-6f5fd6c589-lz88w for pod on switch ip-10-0-0-121.ec2.internal I0429 19:41:09.964360 3658 kube.go:256] Updating pod openshift-monitoring/thanos-querier-6f5fd6c589-lz88w I0429 19:41:09.982004 3658 pod.go:62] [openshift-monitoring/thanos-querier-6f5fd6c589-lz88w] pod update took 17.680422ms I0429 19:41:09.982033 3658 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-6f5fd6c589-lz88w] addLogicalPort annotation time took 17.712718ms I0429 19:41:09.983126 3658 pods.go:271] [openshift-monitoring/thanos-querier-6f5fd6c589-lz88w] addLogicalPort took 18.884612ms, libovsdb time 788.861µs W0429 19:41:10.108211 3658 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 19:41:10.108234 3658 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 19:41:10.108240 3658 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0429 19:41:10.326786 3658 cni.go:452] [openshift-monitoring/thanos-querier-6f5fd6c589-lz88w 8cc377d88bd130213da5d190c81fea94dcbe4b7b8728bfd9f1642504dc16e8cd network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-6f5fd6c589-lz88w 8cc377d88bd130213da5d190c81fea94dcbe4b7b8728bfd9f1642504dc16e8cd network default NAD default NAD key ] I0429 19:41:10.337159 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-6f5fd6c589-lz88w, hostIfaceName: 8cc377d88bd1302, network: default, NAD default, SandboxID: "8cc377d88bd130213da5d190c81fea94dcbe4b7b8728bfd9f1642504dc16e8cd", PCI device ID: , UID: "26413196-f00e-497a-b8e3-acd7c0afc649", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0429 19:41:10.386632 3658 cni.go:473] [openshift-monitoring/thanos-querier-6f5fd6c589-lz88w 8cc377d88bd130213da5d190c81fea94dcbe4b7b8728bfd9f1642504dc16e8cd network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-6f5fd6c589-lz88w 8cc377d88bd130213da5d190c81fea94dcbe4b7b8728bfd9f1642504dc16e8cd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:a3:63:46:14:17\",\"name\":\"8cc377d88bd1302\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/22deb22c-1af4-4ec1-bddc-bf4190c8c7ad\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:41:10.947517 3658 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 19:41:10.947539 3658 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 19:41:10.947545 3658 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0429 19:41:11.623064 3658 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-72kw5] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-72kw5 for pod on switch ip-10-0-0-121.ec2.internal I0429 19:41:11.623160 3658 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-72kw5 I0429 19:41:11.662331 3658 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-72kw5 I0429 19:41:11.679293 3658 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-72kw5] pod update took 56.162115ms I0429 19:41:11.679322 3658 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-72kw5] addLogicalPort annotation time took 56.193671ms I0429 19:41:11.680632 3658 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-72kw5] addLogicalPort took 57.580625ms, libovsdb time 931.387µs I0429 19:41:13.033885 3658 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-72kw5 13ff98c6f376601ab5502e107af60825c56e9ca0fbe0c797560c035e8db81b73 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-72kw5 13ff98c6f376601ab5502e107af60825c56e9ca0fbe0c797560c035e8db81b73 network default NAD default NAD key ] I0429 19:41:13.044616 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-72kw5, hostIfaceName: 13ff98c6f376601, network: default, NAD default, SandboxID: "13ff98c6f376601ab5502e107af60825c56e9ca0fbe0c797560c035e8db81b73", PCI device ID: , UID: "ede1a604-c7c3-44ca-bee4-5cf56b201224", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0429 19:41:13.115664 3658 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-72kw5 13ff98c6f376601ab5502e107af60825c56e9ca0fbe0c797560c035e8db81b73 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-72kw5 13ff98c6f376601ab5502e107af60825c56e9ca0fbe0c797560c035e8db81b73 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:d7:1c:39:97:4a\",\"name\":\"13ff98c6f376601\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/654ff74e-59d8-4645-9b10-127cd26dc1d6\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:41:13.343258 3658 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-0-121.ec2.internal I0429 19:41:13.343431 3658 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0429 19:41:13.371825 3658 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 28.440381ms I0429 19:41:13.371860 3658 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 28.481922ms I0429 19:41:13.373559 3658 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 30.322199ms, libovsdb time 1.237271ms I0429 19:41:13.702054 3658 cni.go:452] [openshift-monitoring/prometheus-k8s-0 c4ce8c8ed870f8c3ef8b3c393466bcd1304fef7fa70476c5a306c3eb804b1626 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 c4ce8c8ed870f8c3ef8b3c393466bcd1304fef7fa70476c5a306c3eb804b1626 network default NAD default NAD key ] I0429 19:41:13.713860 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: c4ce8c8ed870f8c, network: default, NAD default, SandboxID: "c4ce8c8ed870f8c3ef8b3c393466bcd1304fef7fa70476c5a306c3eb804b1626", PCI device ID: , UID: "a30a30a1-290d-490d-b539-698cf90569f0", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0429 19:41:13.775113 3658 cni.go:473] [openshift-monitoring/prometheus-k8s-0 c4ce8c8ed870f8c3ef8b3c393466bcd1304fef7fa70476c5a306c3eb804b1626 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 c4ce8c8ed870f8c3ef8b3c393466bcd1304fef7fa70476c5a306c3eb804b1626 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:60:b0:0c:c4:9b\",\"name\":\"c4ce8c8ed870f8c\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8dcb3696-c68b-4ab7-9ab7-cf5ca6570ee6\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:41:13.949866 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0429 19:41:13.988920 3658 cni.go:452] [openshift-image-registry/image-registry-9b94754f9-lqtbq 6c84fcc064ea453440c70eadfcbd306b7bee44a441cbd2eb382cc3a5825f400b network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-9b94754f9-lqtbq 6c84fcc064ea453440c70eadfcbd306b7bee44a441cbd2eb382cc3a5825f400b network default NAD default NAD key ] I0429 19:41:13.988951 3658 cni.go:452] [openshift-dns/dns-default-cx8dw a8b6af4ce1f2514dc26fbffdd157f0420fce2b793106277a91457e4cd5f9aa0e network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-cx8dw a8b6af4ce1f2514dc26fbffdd157f0420fce2b793106277a91457e4cd5f9aa0e network default NAD default NAD key ] I0429 19:41:14.002737 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-9b94754f9-lqtbq, hostIfaceName: 6c84fcc064ea453, network: default, NAD default, SandboxID: "6c84fcc064ea453440c70eadfcbd306b7bee44a441cbd2eb382cc3a5825f400b", PCI device ID: , UID: "a44e3b27-92d5-4f6b-b21a-20152d689285", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0429 19:41:14.002980 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-cx8dw, hostIfaceName: a8b6af4ce1f2514, network: default, NAD default, SandboxID: "a8b6af4ce1f2514dc26fbffdd157f0420fce2b793106277a91457e4cd5f9aa0e", PCI device ID: , UID: "bad56874-109c-4691-a03f-ffdc966ed307", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0429 19:41:14.056548 3658 cni.go:473] [openshift-dns/dns-default-cx8dw a8b6af4ce1f2514dc26fbffdd157f0420fce2b793106277a91457e4cd5f9aa0e network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-cx8dw a8b6af4ce1f2514dc26fbffdd157f0420fce2b793106277a91457e4cd5f9aa0e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:72:11:0d:f8:60\",\"name\":\"a8b6af4ce1f2514\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4bf99e09-9f37-490f-88a8-ef8388c8dffc\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:41:14.075277 3658 cni.go:473] [openshift-image-registry/image-registry-9b94754f9-lqtbq 6c84fcc064ea453440c70eadfcbd306b7bee44a441cbd2eb382cc3a5825f400b network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-9b94754f9-lqtbq 6c84fcc064ea453440c70eadfcbd306b7bee44a441cbd2eb382cc3a5825f400b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:24:fa:f8:0d:3f\",\"name\":\"6c84fcc064ea453\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2c3ac91e-b355-4533-bd42-7f8286995bf9\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:41:14.090168 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0429 19:41:14.975506 3658 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 19:41:14.975530 3658 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 19:41:15.164848 3658 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 19:41:15.164868 3658 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 19:41:15.164873 3658 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 19:41:15.164879 3658 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 19:41:15.229096 3658 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0429 19:41:15.275200 3658 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 19:41:15.275224 3658 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 19:41:16.129413 3658 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 19:41:16.129439 3658 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 19:41:16.129448 3658 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 19:41:16.129454 3658 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 19:41:17.883048 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0429 19:41:17.883071 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0429 19:41:18.031712 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0429 19:41:18.031746 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0429 19:41:19.749953 3658 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-dvksw] creating logical port openshift-console_downloads-5c54f4944f-dvksw for pod on switch ip-10-0-0-121.ec2.internal I0429 19:41:19.750060 3658 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-dvksw I0429 19:41:19.769310 3658 pod.go:62] [openshift-console/downloads-5c54f4944f-dvksw] pod update took 19.279194ms I0429 19:41:19.769336 3658 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-dvksw] addLogicalPort annotation time took 19.311539ms I0429 19:41:19.770614 3658 pods.go:271] [openshift-console/downloads-5c54f4944f-dvksw] addLogicalPort took 20.681023ms, libovsdb time 892.617µs I0429 19:41:20.112034 3658 cni.go:452] [openshift-console/downloads-5c54f4944f-dvksw e31e269164adc779c9cf49e5ae4711693b707263cba9c0b03f8a03671b6d3ccf network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-dvksw e31e269164adc779c9cf49e5ae4711693b707263cba9c0b03f8a03671b6d3ccf network default NAD default NAD key ] I0429 19:41:20.121672 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-dvksw, hostIfaceName: e31e269164adc77, network: default, NAD default, SandboxID: "e31e269164adc779c9cf49e5ae4711693b707263cba9c0b03f8a03671b6d3ccf", PCI device ID: , UID: "e42d5727-0721-481a-a04a-f7d0d3e05536", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0429 19:41:20.175055 3658 cni.go:473] [openshift-console/downloads-5c54f4944f-dvksw e31e269164adc779c9cf49e5ae4711693b707263cba9c0b03f8a03671b6d3ccf network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-dvksw e31e269164adc779c9cf49e5ae4711693b707263cba9c0b03f8a03671b6d3ccf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:78:13:eb:5c:74\",\"name\":\"e31e269164adc77\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0704945e-054d-4a1f-81c8-9047e9cf4750\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:41:21.183570 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 19:41:21.183592 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 19:41:21.183598 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 19:41:21.183603 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0429 19:41:27.139439 3658 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0429 19:41:27.183382 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 19:41:27.183407 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0429 19:41:27.183416 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0429 19:41:31.408222 3658 base_network_controller_pods.go:487] [default/openshift-console/console-54bcd559d5-ttshh] creating logical port openshift-console_console-54bcd559d5-ttshh for pod on switch ip-10-0-0-121.ec2.internal I0429 19:41:31.408361 3658 kube.go:256] Updating pod openshift-console/console-54bcd559d5-ttshh I0429 19:41:31.426779 3658 pod.go:62] [openshift-console/console-54bcd559d5-ttshh] pod update took 18.449044ms I0429 19:41:31.426802 3658 base_network_controller_pods.go:951] [default/openshift-console/console-54bcd559d5-ttshh] addLogicalPort annotation time took 18.476696ms I0429 19:41:31.428150 3658 pods.go:271] [openshift-console/console-54bcd559d5-ttshh] addLogicalPort took 19.945297ms, libovsdb time 891.272µs I0429 19:41:31.776299 3658 cni.go:452] [openshift-console/console-54bcd559d5-ttshh b5dbb5f4996bc602d191b3052e9e3b91cc8789d4ed77ce15aae90a4b9f7920ca network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-54bcd559d5-ttshh b5dbb5f4996bc602d191b3052e9e3b91cc8789d4ed77ce15aae90a4b9f7920ca network default NAD default NAD key ] I0429 19:41:31.786024 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-54bcd559d5-ttshh, hostIfaceName: b5dbb5f4996bc60, network: default, NAD default, SandboxID: "b5dbb5f4996bc602d191b3052e9e3b91cc8789d4ed77ce15aae90a4b9f7920ca", PCI device ID: , UID: "64b211a7-fa8f-4fd7-b95b-32406eaafe2a", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0429 19:41:31.841865 3658 cni.go:473] [openshift-console/console-54bcd559d5-ttshh b5dbb5f4996bc602d191b3052e9e3b91cc8789d4ed77ce15aae90a4b9f7920ca network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-54bcd559d5-ttshh b5dbb5f4996bc602d191b3052e9e3b91cc8789d4ed77ce15aae90a4b9f7920ca network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:b9:f8:dc:85:40\",\"name\":\"b5dbb5f4996bc60\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ae62dc83-2d19-4687-974a-605ca280a53c\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:41:34.254662 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0429 19:41:34.254668 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0429 19:41:34.254719 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0429 19:41:35.258561 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0429 19:41:36.193873 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0429 19:41:36.283609 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0429 19:41:40.389078 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0429 19:41:42.290852 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0429 19:41:43.083068 3658 cni.go:452] [openshift-image-registry/image-registry-9b94754f9-lqtbq 6c84fcc064ea453440c70eadfcbd306b7bee44a441cbd2eb382cc3a5825f400b network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-9b94754f9-lqtbq 6c84fcc064ea453440c70eadfcbd306b7bee44a441cbd2eb382cc3a5825f400b network default NAD default NAD key ] I0429 19:41:43.172344 3658 cni.go:473] [openshift-image-registry/image-registry-9b94754f9-lqtbq 6c84fcc064ea453440c70eadfcbd306b7bee44a441cbd2eb382cc3a5825f400b network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-9b94754f9-lqtbq 6c84fcc064ea453440c70eadfcbd306b7bee44a441cbd2eb382cc3a5825f400b network default NAD default NAD key default], result "{}", err W0429 19:41:43.223438 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads I0429 19:41:43.230330 3658 obj_retry.go:498] Detected object openshift-image-registry/image-registry-9b94754f9-lqtbq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:41:43.230369 3658 pods.go:176] Deleting pod: openshift-image-registry/image-registry-9b94754f9-lqtbq I0429 19:41:43.231788 3658 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-9b94754f9-lqtbq, ips: 10.129.0.5 I0429 19:41:43.231824 3658 obj_retry.go:498] Detected object openshift-image-registry/image-registry-9b94754f9-lqtbq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0429 19:41:43.331510 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads I0429 19:41:43.588102 3658 obj_retry.go:492] Detected object openshift-image-registry/image-registry-9b94754f9-lqtbq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:41:43.588130 3658 obj_retry.go:492] Detected object openshift-image-registry/image-registry-9b94754f9-lqtbq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:41:43.594864 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-9b94754f9-lqtbq I0429 19:41:43.594929 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-9b94754f9-lqtbq W0429 19:41:49.105430 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0429 19:41:59.360226 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0429 19:41:59.360231 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0429 19:41:59.360383 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0429 19:42:00.365519 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0429 19:42:05.713126 3658 obj_retry.go:498] Detected object openshift-console/console-54bcd559d5-rdrk9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:42:05.713741 3658 obj_retry.go:498] Detected object openshift-console/console-54bcd559d5-rdrk9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:42:06.338114 3658 obj_retry.go:492] Detected object openshift-console/console-54bcd559d5-rdrk9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:42:06.338140 3658 obj_retry.go:492] Detected object openshift-console/console-54bcd559d5-rdrk9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:42:06.346354 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-54bcd559d5-rdrk9 I0429 19:42:06.346382 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-54bcd559d5-rdrk9 I0429 19:42:13.209987 3658 cni.go:452] [openshift-ingress/router-default-557b65ff59-q4kpj 3925aa3ad50038f2d69c5252f98d067027f3093885018ebad2da274a6574140a network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-557b65ff59-q4kpj 3925aa3ad50038f2d69c5252f98d067027f3093885018ebad2da274a6574140a network default NAD default NAD key ] I0429 19:42:13.296333 3658 cni.go:473] [openshift-ingress/router-default-557b65ff59-q4kpj 3925aa3ad50038f2d69c5252f98d067027f3093885018ebad2da274a6574140a network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-557b65ff59-q4kpj 3925aa3ad50038f2d69c5252f98d067027f3093885018ebad2da274a6574140a network default NAD default NAD key default], result "{}", err I0429 19:42:13.371343 3658 obj_retry.go:498] Detected object openshift-ingress/router-default-557b65ff59-q4kpj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:42:13.371384 3658 pods.go:176] Deleting pod: openshift-ingress/router-default-557b65ff59-q4kpj I0429 19:42:13.372669 3658 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-557b65ff59-q4kpj, ips: 10.129.0.10 I0429 19:42:13.372698 3658 obj_retry.go:498] Detected object openshift-ingress/router-default-557b65ff59-q4kpj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:42:13.731078 3658 obj_retry.go:492] Detected object openshift-ingress/router-default-557b65ff59-q4kpj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:42:13.731109 3658 obj_retry.go:492] Detected object openshift-ingress/router-default-557b65ff59-q4kpj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:42:13.738902 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-557b65ff59-q4kpj I0429 19:42:13.738940 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-557b65ff59-q4kpj W0429 19:42:14.239939 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 19:42:14.239959 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 19:42:14.239973 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 19:42:14.239978 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 19:42:14.473739 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 19:42:14.473759 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0429 19:42:16.352756 3658 cni.go:452] [openshift-console/console-54bcd559d5-ttshh b5dbb5f4996bc602d191b3052e9e3b91cc8789d4ed77ce15aae90a4b9f7920ca network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-54bcd559d5-ttshh b5dbb5f4996bc602d191b3052e9e3b91cc8789d4ed77ce15aae90a4b9f7920ca network default NAD default NAD key ] I0429 19:42:16.442337 3658 cni.go:473] [openshift-console/console-54bcd559d5-ttshh b5dbb5f4996bc602d191b3052e9e3b91cc8789d4ed77ce15aae90a4b9f7920ca network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-54bcd559d5-ttshh b5dbb5f4996bc602d191b3052e9e3b91cc8789d4ed77ce15aae90a4b9f7920ca network default NAD default NAD key default], result "{}", err I0429 19:42:16.522787 3658 obj_retry.go:498] Detected object openshift-console/console-54bcd559d5-ttshh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:42:16.522826 3658 pods.go:176] Deleting pod: openshift-console/console-54bcd559d5-ttshh I0429 19:42:16.524028 3658 pods.go:236] Attempting to release IPs for pod: openshift-console/console-54bcd559d5-ttshh, ips: 10.129.0.20 I0429 19:42:16.524056 3658 obj_retry.go:498] Detected object openshift-console/console-54bcd559d5-ttshh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:42:17.437528 3658 obj_retry.go:492] Detected object openshift-console/console-54bcd559d5-ttshh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:42:17.437554 3658 obj_retry.go:492] Detected object openshift-console/console-54bcd559d5-ttshh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:42:17.442959 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-54bcd559d5-ttshh I0429 19:42:17.442986 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-54bcd559d5-ttshh I0429 19:42:28.949258 3658 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:42:28.949851 3658 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:42:29.420979 3658 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:42:29.421016 3658 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:42:29.427655 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0429 19:42:29.427688 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0429 19:42:33.355864 3658 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:42:33.356444 3658 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:42:34.290346 3658 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:42:34.290375 3658 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:42:34.298197 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0429 19:42:34.298217 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0429 19:42:35.841035 3658 base_network_controller_pods.go:487] [default/openshift-console/console-799447877-fc6tj] creating logical port openshift-console_console-799447877-fc6tj for pod on switch ip-10-0-0-121.ec2.internal I0429 19:42:35.841135 3658 kube.go:256] Updating pod openshift-console/console-799447877-fc6tj I0429 19:42:35.869890 3658 pod.go:62] [openshift-console/console-799447877-fc6tj] pod update took 28.777974ms I0429 19:42:35.869918 3658 base_network_controller_pods.go:951] [default/openshift-console/console-799447877-fc6tj] addLogicalPort annotation time took 28.811738ms I0429 19:42:35.871100 3658 pods.go:271] [openshift-console/console-799447877-fc6tj] addLogicalPort took 30.085891ms, libovsdb time 862.224µs I0429 19:42:36.207348 3658 cni.go:452] [openshift-console/console-799447877-fc6tj d097a83f24b8dc509841a41a450cbb31122ebc4c478ec1adb4ab67384c24a4ab network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-799447877-fc6tj d097a83f24b8dc509841a41a450cbb31122ebc4c478ec1adb4ab67384c24a4ab network default NAD default NAD key ] I0429 19:42:36.216580 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-799447877-fc6tj, hostIfaceName: d097a83f24b8dc5, network: default, NAD default, SandboxID: "d097a83f24b8dc509841a41a450cbb31122ebc4c478ec1adb4ab67384c24a4ab", PCI device ID: , UID: "93ec3608-ef56-4d99-812e-4c9d97d84f04", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0429 19:42:36.268596 3658 cni.go:473] [openshift-console/console-799447877-fc6tj d097a83f24b8dc509841a41a450cbb31122ebc4c478ec1adb4ab67384c24a4ab network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-799447877-fc6tj d097a83f24b8dc509841a41a450cbb31122ebc4c478ec1adb4ab67384c24a4ab network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:32:c4:ee:bc:5a\",\"name\":\"d097a83f24b8dc5\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c89f011a-414c-41e6-b4f8-0ff4e6c51522\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:42:46.562100 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0429 19:43:01.161070 3658 obj_retry.go:498] Detected object openshift-console/console-7d5fd49fd8-6b7br of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:43:01.161827 3658 obj_retry.go:498] Detected object openshift-console/console-7d5fd49fd8-6b7br of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:43:01.208149 3658 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:43:01.208674 3658 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:43:01.405918 3658 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:43:01.405941 3658 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:43:01.418205 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0429 19:43:01.418234 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0429 19:43:01.569792 3658 obj_retry.go:492] Detected object openshift-console/console-7d5fd49fd8-6b7br of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:43:01.569812 3658 obj_retry.go:492] Detected object openshift-console/console-7d5fd49fd8-6b7br of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:43:01.577318 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d5fd49fd8-6b7br I0429 19:43:01.577345 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d5fd49fd8-6b7br I0429 19:43:01.580112 3658 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0429 19:43:11.884553 3658 obj_retry.go:498] Detected object openshift-console/console-7d5fd49fd8-vx5zg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:43:11.885089 3658 obj_retry.go:498] Detected object openshift-console/console-7d5fd49fd8-vx5zg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:43:12.452458 3658 obj_retry.go:492] Detected object openshift-console/console-7d5fd49fd8-vx5zg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:43:12.452481 3658 obj_retry.go:492] Detected object openshift-console/console-7d5fd49fd8-vx5zg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:43:12.459081 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d5fd49fd8-vx5zg I0429 19:43:12.459109 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d5fd49fd8-vx5zg I0429 19:43:12.749692 3658 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624860-c4pzr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:43:12.750215 3658 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624860-c4pzr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:43:13.633170 3658 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624860-c4pzr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:43:13.633195 3658 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624860-c4pzr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:43:36.749189 3658 cni.go:452] [openshift-monitoring/prometheus-k8s-0 c4ce8c8ed870f8c3ef8b3c393466bcd1304fef7fa70476c5a306c3eb804b1626 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 c4ce8c8ed870f8c3ef8b3c393466bcd1304fef7fa70476c5a306c3eb804b1626 network default NAD default NAD key ] I0429 19:43:36.823649 3658 cni.go:473] [openshift-monitoring/prometheus-k8s-0 c4ce8c8ed870f8c3ef8b3c393466bcd1304fef7fa70476c5a306c3eb804b1626 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 c4ce8c8ed870f8c3ef8b3c393466bcd1304fef7fa70476c5a306c3eb804b1626 network default NAD default NAD key default], result "{}", err I0429 19:43:36.890248 3658 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:43:36.890309 3658 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0429 19:43:36.892191 3658 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.129.0.18 I0429 19:43:36.892235 3658 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:43:37.749324 3658 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:43:37.749390 3658 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:43:37.756229 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0429 19:43:37.756260 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0429 19:43:37.797472 3658 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-0-121.ec2.internal I0429 19:43:37.797622 3658 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0429 19:43:37.821507 3658 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 23.902377ms I0429 19:43:37.821567 3658 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 23.989291ms I0429 19:43:37.822873 3658 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 25.416901ms, libovsdb time 964.181µs I0429 19:43:38.158789 3658 cni.go:452] [openshift-monitoring/prometheus-k8s-0 3df6ccf51372250b5151f88fdbfd23b725b8cc1aee68054744864b9a0cb51a09 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 3df6ccf51372250b5151f88fdbfd23b725b8cc1aee68054744864b9a0cb51a09 network default NAD default NAD key ] I0429 19:43:38.168559 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 3df6ccf51372250, network: default, NAD default, SandboxID: "3df6ccf51372250b5151f88fdbfd23b725b8cc1aee68054744864b9a0cb51a09", PCI device ID: , UID: "6a29212e-73c2-4e1b-8939-987155ec4132", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0429 19:43:38.219324 3658 cni.go:473] [openshift-monitoring/prometheus-k8s-0 3df6ccf51372250b5151f88fdbfd23b725b8cc1aee68054744864b9a0cb51a09 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 3df6ccf51372250b5151f88fdbfd23b725b8cc1aee68054744864b9a0cb51a09 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:17:4c:6f:84:94\",\"name\":\"3df6ccf51372250\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/abf4ce67-3e0a-470a-992a-da22a5102f06\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:44:14.959078 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0429 19:44:18.968648 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0429 19:44:29.862464 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0429 19:44:31.963903 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 W0429 19:44:35.954998 3658 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0429 19:44:39.029912 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 19:44:39.029933 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0429 19:44:46.994950 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0429 19:44:46.995021 3658 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-dpnrw IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-dpnrw: no pod IPs found I0429 19:45:08.952810 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0429 19:45:14.314231 3658 cni.go:452] [openshift-console/console-799447877-fc6tj d097a83f24b8dc509841a41a450cbb31122ebc4c478ec1adb4ab67384c24a4ab network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-799447877-fc6tj d097a83f24b8dc509841a41a450cbb31122ebc4c478ec1adb4ab67384c24a4ab network default NAD default NAD key ] I0429 19:45:14.396432 3658 cni.go:473] [openshift-console/console-799447877-fc6tj d097a83f24b8dc509841a41a450cbb31122ebc4c478ec1adb4ab67384c24a4ab network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-799447877-fc6tj d097a83f24b8dc509841a41a450cbb31122ebc4c478ec1adb4ab67384c24a4ab network default NAD default NAD key default], result "{}", err I0429 19:45:14.444637 3658 obj_retry.go:498] Detected object openshift-console/console-799447877-fc6tj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:45:14.444677 3658 pods.go:176] Deleting pod: openshift-console/console-799447877-fc6tj I0429 19:45:14.445890 3658 pods.go:236] Attempting to release IPs for pod: openshift-console/console-799447877-fc6tj, ips: 10.129.0.21 I0429 19:45:14.445923 3658 obj_retry.go:498] Detected object openshift-console/console-799447877-fc6tj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:45:14.453944 3658 base_network_controller_pods.go:487] [default/openshift-console/console-64d48c864d-rv5n5] creating logical port openshift-console_console-64d48c864d-rv5n5 for pod on switch ip-10-0-0-121.ec2.internal I0429 19:45:14.454065 3658 kube.go:256] Updating pod openshift-console/console-64d48c864d-rv5n5 I0429 19:45:14.470978 3658 pod.go:62] [openshift-console/console-64d48c864d-rv5n5] pod update took 16.933434ms I0429 19:45:14.471006 3658 base_network_controller_pods.go:951] [default/openshift-console/console-64d48c864d-rv5n5] addLogicalPort annotation time took 16.965881ms I0429 19:45:14.472408 3658 pods.go:271] [openshift-console/console-64d48c864d-rv5n5] addLogicalPort took 18.478868ms, libovsdb time 999.759µs I0429 19:45:14.815841 3658 cni.go:452] [openshift-console/console-64d48c864d-rv5n5 494b0153df581a4f257d3d008e5058213b9817ce960b3501611737500a5c4000 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-64d48c864d-rv5n5 494b0153df581a4f257d3d008e5058213b9817ce960b3501611737500a5c4000 network default NAD default NAD key ] I0429 19:45:14.825566 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-64d48c864d-rv5n5, hostIfaceName: 494b0153df581a4, network: default, NAD default, SandboxID: "494b0153df581a4f257d3d008e5058213b9817ce960b3501611737500a5c4000", PCI device ID: , UID: "f706f235-03bb-4c1a-a728-f1431fb1ce19", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0429 19:45:14.875993 3658 cni.go:473] [openshift-console/console-64d48c864d-rv5n5 494b0153df581a4f257d3d008e5058213b9817ce960b3501611737500a5c4000 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-64d48c864d-rv5n5 494b0153df581a4f257d3d008e5058213b9817ce960b3501611737500a5c4000 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:59:1b:38:5a:21\",\"name\":\"494b0153df581a4\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b1bb553b-6512-4613-aa38-fe53668cad96\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:45:15.140858 3658 obj_retry.go:492] Detected object openshift-console/console-799447877-fc6tj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:45:15.140898 3658 obj_retry.go:492] Detected object openshift-console/console-799447877-fc6tj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:45:15.146914 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-799447877-fc6tj I0429 19:45:15.146941 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-799447877-fc6tj I0429 19:45:21.984906 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 W0429 19:45:25.182827 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0429 19:45:25.209538 3658 obj_retry.go:498] Detected object openshift-console/console-799447877-8k2mz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:45:25.210068 3658 obj_retry.go:498] Detected object openshift-console/console-799447877-8k2mz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:45:26.149956 3658 obj_retry.go:492] Detected object openshift-console/console-799447877-8k2mz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:45:26.149980 3658 obj_retry.go:492] Detected object openshift-console/console-799447877-8k2mz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:45:26.157385 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-799447877-8k2mz I0429 19:45:26.157409 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-799447877-8k2mz I0429 19:45:31.865603 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=14 I0429 19:46:00.985639 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0429 19:46:05.863942 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=143 I0429 19:46:13.979658 3658 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0429 19:46:33.872279 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=585 I0429 19:47:02.973285 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0429 19:47:04.863227 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=72 I0429 19:47:06.948865 3658 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0429 19:48:00.948988 3658 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0429 19:48:32.863103 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=73 I0429 19:50:04.368226 3658 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624870-2lqgc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:50:04.368819 3658 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624870-2lqgc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:50:05.258430 3658 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624870-2lqgc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:50:05.258507 3658 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624870-2lqgc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:50:52.869195 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=7 I0429 19:51:03.989463 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=5 I0429 19:51:24.739793 3658 namespace.go:98] [tsf] adding namespace I0429 19:51:24.740754 3658 namespace.go:102] [tsf] adding namespace took 932.472µs I0429 19:51:24.760303 3658 namespace.go:142] [tsf] updating namespace I0429 19:51:24.771833 3658 namespace.go:142] [tsf] updating namespace I0429 19:51:28.656089 3658 namespace.go:98] [tsf-keycloak] adding namespace I0429 19:51:28.656944 3658 namespace.go:102] [tsf-keycloak] adding namespace took 834.213µs I0429 19:51:28.657677 3658 namespace.go:98] [rhtpa-operator] adding namespace I0429 19:51:28.658344 3658 namespace.go:102] [rhtpa-operator] adding namespace took 647.673µs I0429 19:51:28.664678 3658 namespace.go:98] [tsf-tpa] adding namespace I0429 19:51:28.664708 3658 namespace.go:98] [cert-manager-operator] adding namespace I0429 19:51:28.665362 3658 namespace.go:102] [tsf-tpa] adding namespace took 668.3µs I0429 19:51:28.665725 3658 namespace.go:98] [konflux-ui] adding namespace I0429 19:51:28.665799 3658 namespace.go:98] [konflux-operator] adding namespace I0429 19:51:28.665990 3658 namespace.go:102] [cert-manager-operator] adding namespace took 1.269539ms I0429 19:51:28.666565 3658 namespace.go:102] [konflux-ui] adding namespace took 827.147µs I0429 19:51:28.667060 3658 namespace.go:102] [konflux-operator] adding namespace took 1.244868ms I0429 19:51:28.668988 3658 namespace.go:98] [rhbk-operator] adding namespace I0429 19:51:28.669538 3658 namespace.go:102] [rhbk-operator] adding namespace took 534.078µs I0429 19:51:28.674715 3658 namespace.go:142] [tsf-keycloak] updating namespace I0429 19:51:28.680504 3658 namespace.go:98] [tsf-tas] adding namespace I0429 19:51:28.681102 3658 namespace.go:102] [tsf-tas] adding namespace took 585.243µs I0429 19:51:28.685529 3658 namespace.go:142] [tsf-keycloak] updating namespace I0429 19:51:28.696715 3658 namespace.go:142] [rhtpa-operator] updating namespace I0429 19:51:28.711133 3658 namespace.go:142] [rhtpa-operator] updating namespace I0429 19:51:28.716042 3658 namespace.go:142] [tsf-tpa] updating namespace I0429 19:51:28.735758 3658 namespace.go:142] [cert-manager-operator] updating namespace I0429 19:51:28.735843 3658 namespace.go:142] [konflux-ui] updating namespace I0429 19:51:28.739927 3658 namespace.go:142] [tsf-tpa] updating namespace I0429 19:51:28.742394 3658 namespace.go:142] [rhbk-operator] updating namespace I0429 19:51:28.742413 3658 namespace.go:142] [konflux-operator] updating namespace I0429 19:51:28.745370 3658 namespace.go:142] [tsf-tas] updating namespace I0429 19:51:28.748690 3658 namespace.go:142] [tsf-keycloak] updating namespace I0429 19:51:28.757328 3658 namespace.go:142] [rhtpa-operator] updating namespace I0429 19:51:28.783158 3658 namespace.go:142] [konflux-ui] updating namespace I0429 19:51:28.796675 3658 namespace.go:142] [konflux-ui] updating namespace I0429 19:51:28.804057 3658 namespace.go:142] [konflux-operator] updating namespace I0429 19:51:28.819480 3658 namespace.go:142] [konflux-operator] updating namespace I0429 19:51:28.826181 3658 namespace.go:142] [rhbk-operator] updating namespace I0429 19:51:28.841088 3658 namespace.go:142] [rhbk-operator] updating namespace I0429 19:51:28.848768 3658 namespace.go:142] [tsf-tas] updating namespace I0429 19:51:28.862822 3658 namespace.go:142] [tsf-tas] updating namespace I0429 19:51:28.867802 3658 namespace.go:142] [cert-manager-operator] updating namespace I0429 19:51:28.878010 3658 namespace.go:142] [cert-manager-operator] updating namespace I0429 19:51:28.890034 3658 namespace.go:142] [tsf-tpa] updating namespace I0429 19:51:29.943621 3658 namespace.go:142] [tsf-keycloak] updating namespace I0429 19:51:29.947303 3658 namespace.go:142] [tsf-tpa] updating namespace I0429 19:51:35.567997 3658 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lgl54 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:51:35.568594 3658 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lgl54 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:51:35.568618 3658 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lgl54 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:51:35.568626 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 19:51:35.568641 3658 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lgl54 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:51:35.568646 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 19:51:35.967253 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0429 19:51:36.575547 3658 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b4fk9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:51:36.576040 3658 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b4fk9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:51:36.576068 3658 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b4fk9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:51:36.576076 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 19:51:36.576087 3658 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b4fk9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:51:36.576091 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 19:51:36.593738 3658 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lgl54 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:36.593755 3658 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lgl54 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:36.593765 3658 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lgl54 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:36.593775 3658 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lgl54 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:37.603499 3658 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b4fk9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:37.603529 3658 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b4fk9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:37.603541 3658 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b4fk9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:37.603554 3658 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b4fk9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:40.619653 3658 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dp48cj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:51:40.620225 3658 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dp48cj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:51:40.620254 3658 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dp48cj of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:51:40.620262 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 19:51:40.620273 3658 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dp48cj of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:51:40.620277 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 19:51:40.774463 3658 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b516krqc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:51:40.775017 3658 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b516krqc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:51:40.775045 3658 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b516krqc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:51:40.775053 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 19:51:40.775063 3658 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b516krqc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:51:40.775068 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 19:51:41.654089 3658 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dp48cj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:41.654121 3658 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dp48cj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:41.654139 3658 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dp48cj of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:41.654149 3658 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dp48cj of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:41.802235 3658 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b516krqc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:41.802263 3658 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b516krqc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:41.802277 3658 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b516krqc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:41.802294 3658 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b516krqc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:42.628451 3658 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hghf7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:51:42.629038 3658 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hghf7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:51:42.629064 3658 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hghf7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:51:42.629072 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 19:51:42.629088 3658 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hghf7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:51:42.629092 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 19:51:43.382380 3658 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-fb86cd8d9-5nxkg] creating logical port rhbk-operator_rhbk-operator-fb86cd8d9-5nxkg for pod on switch ip-10-0-0-121.ec2.internal I0429 19:51:43.382512 3658 kube.go:256] Updating pod rhbk-operator/rhbk-operator-fb86cd8d9-5nxkg I0429 19:51:43.397063 3658 pod.go:62] [rhbk-operator/rhbk-operator-fb86cd8d9-5nxkg] pod update took 14.59755ms I0429 19:51:43.397090 3658 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-fb86cd8d9-5nxkg] addLogicalPort annotation time took 14.631042ms I0429 19:51:43.398275 3658 pods.go:271] [rhbk-operator/rhbk-operator-fb86cd8d9-5nxkg] addLogicalPort took 15.916164ms, libovsdb time 824.284µs I0429 19:51:43.641917 3658 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136kn5p7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:51:43.642532 3658 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136kn5p7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:51:43.642562 3658 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136kn5p7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:51:43.642573 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 19:51:43.642584 3658 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136kn5p7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 19:51:43.642588 3658 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 19:51:43.679874 3658 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hghf7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:43.679900 3658 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hghf7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:43.679915 3658 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hghf7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:43.679929 3658 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hghf7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:43.743026 3658 cni.go:452] [rhbk-operator/rhbk-operator-fb86cd8d9-5nxkg 54b2dbfe658a36fcac86ffd63c575be79ea8cc8a5cc565d9b0f8f83583f36dcc network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-5nxkg 54b2dbfe658a36fcac86ffd63c575be79ea8cc8a5cc565d9b0f8f83583f36dcc network default NAD default NAD key ] I0429 19:51:43.752606 3658 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-fb86cd8d9-5nxkg, hostIfaceName: 54b2dbfe658a36f, network: default, NAD default, SandboxID: "54b2dbfe658a36fcac86ffd63c575be79ea8cc8a5cc565d9b0f8f83583f36dcc", PCI device ID: , UID: "a4d1f251-74e8-4aa1-905a-8759fdd25c1d", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0429 19:51:43.804852 3658 cni.go:473] [rhbk-operator/rhbk-operator-fb86cd8d9-5nxkg 54b2dbfe658a36fcac86ffd63c575be79ea8cc8a5cc565d9b0f8f83583f36dcc network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-5nxkg 54b2dbfe658a36fcac86ffd63c575be79ea8cc8a5cc565d9b0f8f83583f36dcc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:76:0d:be:28:fd\",\"name\":\"54b2dbfe658a36f\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/33548c1e-984d-4ac7-8a9f-81ef8178d49a\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:51:44.683096 3658 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136kn5p7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:44.683123 3658 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136kn5p7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:44.683136 3658 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136kn5p7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:44.683145 3658 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136kn5p7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:51:54.348935 3658 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-66b879f574-7h5rk] creating logical port konflux-operator_konflux-operator-controller-manager-66b879f574-7h5rk for pod on switch ip-10-0-0-121.ec2.internal I0429 19:51:54.349085 3658 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-66b879f574-7h5rk I0429 19:51:54.372867 3658 pod.go:62] [konflux-operator/konflux-operator-controller-manager-66b879f574-7h5rk] pod update took 23.808632ms I0429 19:51:54.372892 3658 base_network_controller_pods.go:951] [default/konflux-operator/konflux-operator-controller-manager-66b879f574-7h5rk] addLogicalPort annotation time took 23.840806ms I0429 19:51:54.374231 3658 pods.go:271] [konflux-operator/konflux-operator-controller-manager-66b879f574-7h5rk] addLogicalPort took 25.315269ms, libovsdb time 941.052µs I0429 19:51:55.031076 3658 cni.go:452] [konflux-operator/konflux-operator-controller-manager-66b879f574-7h5rk b06482bb0eb662fd2d7c873f7b48a089591d590863023d7b657afba6a4782a3a network default NAD default NAD key ] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-66b879f574-7h5rk b06482bb0eb662fd2d7c873f7b48a089591d590863023d7b657afba6a4782a3a network default NAD default NAD key ] I0429 19:51:55.040837 3658 helper_linux.go:556] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-66b879f574-7h5rk, hostIfaceName: b06482bb0eb662f, network: default, NAD default, SandboxID: "b06482bb0eb662fd2d7c873f7b48a089591d590863023d7b657afba6a4782a3a", PCI device ID: , UID: "b643fd10-176f-4a7c-875d-4c9923eb931a", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0429 19:51:55.096349 3658 cni.go:473] [konflux-operator/konflux-operator-controller-manager-66b879f574-7h5rk b06482bb0eb662fd2d7c873f7b48a089591d590863023d7b657afba6a4782a3a network default NAD default NAD key default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-66b879f574-7h5rk b06482bb0eb662fd2d7c873f7b48a089591d590863023d7b657afba6a4782a3a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:ef:f5:d9:2a:aa\",\"name\":\"b06482bb0eb662f\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ac46311-415e-43ed-8582-6700c0d3d8be\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:51:55.589277 3658 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-2g5mz] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-5b66478c58-2g5mz for pod on switch ip-10-0-0-121.ec2.internal I0429 19:51:55.589405 3658 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-2g5mz I0429 19:51:55.607804 3658 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-2g5mz] pod update took 18.428489ms I0429 19:51:55.607831 3658 base_network_controller_pods.go:951] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-2g5mz] addLogicalPort annotation time took 18.460516ms I0429 19:51:55.609241 3658 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-2g5mz] addLogicalPort took 19.984832ms, libovsdb time 1.058948ms I0429 19:51:55.984204 3658 cni.go:452] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-2g5mz 0237e7f689a710e260b5cf51b9f9534613683f8e668c567205a2983179a583ee network default NAD default NAD key ] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-2g5mz 0237e7f689a710e260b5cf51b9f9534613683f8e668c567205a2983179a583ee network default NAD default NAD key ] I0429 19:51:55.996021 3658 helper_linux.go:556] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-5b66478c58-2g5mz, hostIfaceName: 0237e7f689a710e, network: default, NAD default, SandboxID: "0237e7f689a710e260b5cf51b9f9534613683f8e668c567205a2983179a583ee", PCI device ID: , UID: "528a5d8f-1094-47be-b5c1-deec0d0781d4", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0429 19:51:56.053257 3658 cni.go:473] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-2g5mz 0237e7f689a710e260b5cf51b9f9534613683f8e668c567205a2983179a583ee network default NAD default NAD key default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-2g5mz 0237e7f689a710e260b5cf51b9f9534613683f8e668c567205a2983179a583ee network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:eb:1f:ed:19:7a\",\"name\":\"0237e7f689a710e\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d376d02b-c822-4589-a905-e76972cb8da4\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:51:56.240865 3658 namespace.go:142] [openshift-operators] updating namespace I0429 19:51:56.256443 3658 namespace.go:142] [openshift-operators] updating namespace W0429 19:51:57.674118 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0429 19:52:00.294301 3658 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-btsgc] creating logical port tsf-tpa_rhtpa-operator-controller-manager-586cbb496b-btsgc for pod on switch ip-10-0-0-121.ec2.internal I0429 19:52:00.294446 3658 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-btsgc I0429 19:52:00.380248 3658 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-btsgc I0429 19:52:00.442502 3658 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-btsgc] pod update took 148.06725ms I0429 19:52:00.442525 3658 base_network_controller_pods.go:951] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-btsgc] addLogicalPort annotation time took 148.11029ms I0429 19:52:00.443706 3658 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-btsgc] addLogicalPort took 149.421669ms, libovsdb time 863.71µs I0429 19:52:00.646243 3658 cni.go:452] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-btsgc 993034d46fec21a5c219ce4e2a21e50e8b109f13e464e59acebcc54929182028 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-btsgc 993034d46fec21a5c219ce4e2a21e50e8b109f13e464e59acebcc54929182028 network default NAD default NAD key ] I0429 19:52:00.656090 3658 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-586cbb496b-btsgc, hostIfaceName: 993034d46fec21a, network: default, NAD default, SandboxID: "993034d46fec21a5c219ce4e2a21e50e8b109f13e464e59acebcc54929182028", PCI device ID: , UID: "3a834f55-76db-4f2b-89ca-227f4274f527", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0429 19:52:00.708344 3658 namespace.go:98] [cert-manager] adding namespace I0429 19:52:00.709146 3658 namespace.go:102] [cert-manager] adding namespace took 780.703µs I0429 19:52:00.709995 3658 cni.go:473] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-btsgc 993034d46fec21a5c219ce4e2a21e50e8b109f13e464e59acebcc54929182028 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-btsgc 993034d46fec21a5c219ce4e2a21e50e8b109f13e464e59acebcc54929182028 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:05:b3:35:91:db\",\"name\":\"993034d46fec21a\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b17152c5-5766-4628-97aa-fa15a1ec7fad\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:52:00.881596 3658 namespace.go:142] [cert-manager] updating namespace I0429 19:52:00.926407 3658 namespace.go:142] [cert-manager] updating namespace W0429 19:52:01.953533 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0429 19:52:01.966591 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0429 19:52:02.167731 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0429 19:52:02.173866 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0429 19:52:02.918270 3658 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-sw9vj] creating logical port openshift-operators_rhtas-operator-controller-manager-68888cb75-sw9vj for pod on switch ip-10-0-0-121.ec2.internal I0429 19:52:02.918417 3658 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-68888cb75-sw9vj I0429 19:52:02.965084 3658 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-68888cb75-sw9vj] pod update took 46.694292ms I0429 19:52:02.965108 3658 base_network_controller_pods.go:951] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-sw9vj] addLogicalPort annotation time took 46.725692ms I0429 19:52:02.966464 3658 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-68888cb75-sw9vj] addLogicalPort took 48.21225ms, libovsdb time 997.15µs I0429 19:52:03.537444 3658 cni.go:452] [openshift-operators/rhtas-operator-controller-manager-68888cb75-sw9vj 19a56dba82a948ef1dfb70a977fd28617ad363be05805576dff5427d69bf17ba network default NAD default NAD key ] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-sw9vj 19a56dba82a948ef1dfb70a977fd28617ad363be05805576dff5427d69bf17ba network default NAD default NAD key ] I0429 19:52:03.548306 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-68888cb75-sw9vj, hostIfaceName: 19a56dba82a948e, network: default, NAD default, SandboxID: "19a56dba82a948ef1dfb70a977fd28617ad363be05805576dff5427d69bf17ba", PCI device ID: , UID: "b7082977-82d5-45c1-be10-9d2a0e4d6da7", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0429 19:52:03.603183 3658 cni.go:473] [openshift-operators/rhtas-operator-controller-manager-68888cb75-sw9vj 19a56dba82a948ef1dfb70a977fd28617ad363be05805576dff5427d69bf17ba network default NAD default NAD key default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-sw9vj 19a56dba82a948ef1dfb70a977fd28617ad363be05805576dff5427d69bf17ba network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:b1:a4:c6:d9:0b\",\"name\":\"19a56dba82a948e\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4dd1e877-4291-406c-b956-7632a5c576cc\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:52:03.719530 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0429 19:52:03.983829 3658 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=5 I0429 19:52:06.125851 3658 namespace.go:98] [trusted-artifact-signer] adding namespace I0429 19:52:06.126753 3658 namespace.go:102] [trusted-artifact-signer] adding namespace took 878.856µs I0429 19:52:06.217617 3658 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 19:52:06.256968 3658 namespace.go:142] [trusted-artifact-signer] updating namespace W0429 19:52:06.650697 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0429 19:52:06.650717 3658 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0429 19:52:06.717971 3658 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0429 19:52:08.322934 3658 namespace.go:142] [openshift-operators] updating namespace W0429 19:52:08.689794 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0429 19:52:09.017249 3658 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-t952s] creating logical port openshift-operators_openshift-pipelines-operator-6bdf6c8794-t952s for pod on switch ip-10-0-0-121.ec2.internal I0429 19:52:09.017398 3658 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6bdf6c8794-t952s I0429 19:52:09.058772 3658 pod.go:62] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-t952s] pod update took 41.413381ms I0429 19:52:09.058798 3658 base_network_controller_pods.go:951] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-t952s] addLogicalPort annotation time took 41.443944ms I0429 19:52:09.059951 3658 pods.go:271] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-t952s] addLogicalPort took 42.726526ms, libovsdb time 816.142µs I0429 19:52:09.400999 3658 cni.go:452] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-t952s 95c10001922df87b375a0aebcd4b745a2086486a306d4aec3f1434e8a70a3855 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-t952s 95c10001922df87b375a0aebcd4b745a2086486a306d4aec3f1434e8a70a3855 network default NAD default NAD key ] I0429 19:52:09.411276 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6bdf6c8794-t952s, hostIfaceName: 95c10001922df87, network: default, NAD default, SandboxID: "95c10001922df87b375a0aebcd4b745a2086486a306d4aec3f1434e8a70a3855", PCI device ID: , UID: "22e2bf8d-65c4-499a-81ad-71017b72a663", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0429 19:52:09.465405 3658 cni.go:473] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-t952s 95c10001922df87b375a0aebcd4b745a2086486a306d4aec3f1434e8a70a3855 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-t952s 95c10001922df87b375a0aebcd4b745a2086486a306d4aec3f1434e8a70a3855 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:ef:da:79:f7:d4\",\"name\":\"95c10001922df87\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c2ba987c-b750-4442-a257-da310b95cfe5\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:52:09.962264 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0429 19:52:12.870373 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 W0429 19:52:14.707004 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0429 19:52:17.750553 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0429 19:52:23.961351 3658 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:52:23.961929 3658 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:52:23.971833 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0429 19:52:23.977151 3658 namespace.go:98] [openshift-pipelines] adding namespace I0429 19:52:23.978065 3658 namespace.go:102] [openshift-pipelines] adding namespace took 896.043µs I0429 19:52:23.987326 3658 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:52:23.987348 3658 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:52:23.996040 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0429 19:52:23.996067 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0429 19:52:24.007129 3658 namespace.go:142] [openshift-pipelines] updating namespace I0429 19:52:24.720791 3658 namespace.go:142] [cert-manager] updating namespace I0429 19:52:24.920424 3658 namespace.go:142] [cert-manager-operator] updating namespace I0429 19:52:25.323140 3658 namespace.go:142] [konflux-operator] updating namespace I0429 19:52:25.514546 3658 namespace.go:142] [konflux-ui] updating namespace I0429 19:52:25.723909 3658 namespace.go:142] [open-cluster-management-2q00ovcq3dp22etaglm2t8tqigpmb591] updating namespace I0429 19:52:26.115567 3658 namespace.go:142] [rhbk-operator] updating namespace I0429 19:52:26.318701 3658 namespace.go:142] [rhtpa-operator] updating namespace I0429 19:52:26.520711 3658 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 19:52:26.718773 3658 namespace.go:142] [tsf] updating namespace I0429 19:52:26.922131 3658 namespace.go:142] [tsf-keycloak] updating namespace I0429 19:52:27.120291 3658 namespace.go:142] [tsf-tas] updating namespace I0429 19:52:27.320603 3658 namespace.go:142] [tsf-tpa] updating namespace I0429 19:52:27.453102 3658 namespace.go:142] [cert-manager] updating namespace I0429 19:52:27.464691 3658 namespace.go:142] [cert-manager-operator] updating namespace I0429 19:52:27.476885 3658 namespace.go:142] [default] updating namespace I0429 19:52:27.488590 3658 namespace.go:142] [konflux-operator] updating namespace I0429 19:52:27.501729 3658 namespace.go:142] [konflux-ui] updating namespace I0429 19:52:27.512389 3658 namespace.go:142] [open-cluster-management-2q00ovcq3dp22etaglm2t8tqigpmb591] updating namespace I0429 19:52:27.524793 3658 namespace.go:142] [openshift] updating namespace I0429 19:52:27.538513 3658 namespace.go:142] [rhbk-operator] updating namespace I0429 19:52:27.551589 3658 namespace.go:142] [rhtpa-operator] updating namespace I0429 19:52:27.565285 3658 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 19:52:27.575421 3658 namespace.go:142] [tsf] updating namespace I0429 19:52:27.586291 3658 namespace.go:142] [tsf-keycloak] updating namespace I0429 19:52:27.598583 3658 namespace.go:142] [tsf-tas] updating namespace I0429 19:52:27.613228 3658 namespace.go:142] [tsf-tpa] updating namespace I0429 19:52:27.704460 3658 namespace.go:142] [cert-manager] updating namespace I0429 19:52:27.808272 3658 namespace.go:142] [cert-manager-operator] updating namespace I0429 19:52:27.912777 3658 namespace.go:142] [default] updating namespace I0429 19:52:28.008209 3658 namespace.go:142] [konflux-operator] updating namespace I0429 19:52:28.103000 3658 namespace.go:142] [konflux-ui] updating namespace I0429 19:52:28.202478 3658 namespace.go:142] [open-cluster-management-2q00ovcq3dp22etaglm2t8tqigpmb591] updating namespace I0429 19:52:28.300538 3658 namespace.go:142] [openshift] updating namespace I0429 19:52:28.403472 3658 namespace.go:142] [rhbk-operator] updating namespace I0429 19:52:28.502584 3658 namespace.go:142] [rhtpa-operator] updating namespace I0429 19:52:28.602626 3658 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 19:52:28.706258 3658 namespace.go:142] [tsf] updating namespace I0429 19:52:28.805465 3658 namespace.go:142] [tsf-keycloak] updating namespace I0429 19:52:28.903256 3658 namespace.go:142] [tsf-tas] updating namespace I0429 19:52:29.005173 3658 namespace.go:142] [tsf-tpa] updating namespace I0429 19:52:29.300016 3658 namespace.go:142] [openshift-pipelines] updating namespace I0429 19:52:36.891985 3658 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-szrnl] creating logical port openshift-pipelines_tekton-pipelines-controller-86bd8d9bfc-szrnl for pod on switch ip-10-0-0-121.ec2.internal I0429 19:52:36.892109 3658 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-szrnl I0429 19:52:36.910850 3658 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-szrnl] pod update took 18.765399ms I0429 19:52:36.910879 3658 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-szrnl] addLogicalPort annotation time took 18.79905ms I0429 19:52:36.912248 3658 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-szrnl] addLogicalPort took 20.293161ms, libovsdb time 932.865µs I0429 19:52:37.269704 3658 cni.go:452] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-szrnl 7ebc53b97fe381456d91c69a81841c5751706cfe9a0b066d47b52747233ae092 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-szrnl 7ebc53b97fe381456d91c69a81841c5751706cfe9a0b066d47b52747233ae092 network default NAD default NAD key ] I0429 19:52:37.280598 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-86bd8d9bfc-szrnl, hostIfaceName: 7ebc53b97fe3814, network: default, NAD default, SandboxID: "7ebc53b97fe381456d91c69a81841c5751706cfe9a0b066d47b52747233ae092", PCI device ID: , UID: "a6fde2e1-fb4c-4d58-909c-5cf8ebebaa5a", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0429 19:52:37.334822 3658 cni.go:473] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-szrnl 7ebc53b97fe381456d91c69a81841c5751706cfe9a0b066d47b52747233ae092 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-szrnl 7ebc53b97fe381456d91c69a81841c5751706cfe9a0b066d47b52747233ae092 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:c6:66:6e:56:c2\",\"name\":\"7ebc53b97fe3814\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8fa70e4e-917f-494b-afac-065a3d01d2cb\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:52:40.811543 3658 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0429 19:52:40.811561 3658 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0429 19:52:40.811565 3658 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0429 19:52:40.881101 3658 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0429 19:52:40.881117 3658 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0429 19:52:40.881124 3658 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0429 19:52:42.936156 3658 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0429 19:52:42.993161 3658 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0429 19:52:43.027632 3658 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 19:52:43.027651 3658 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 19:52:43.027656 3658 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 19:52:43.027661 3658 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0429 19:52:43.956673 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 W0429 19:52:51.883950 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0429 19:52:51.883971 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0429 19:52:51.883977 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0429 19:52:57.952284 3658 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0429 19:52:59.976989 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0429 19:53:00.806570 3658 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-dzpgm] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-dzpgm for pod on switch ip-10-0-0-121.ec2.internal I0429 19:53:00.806667 3658 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-dzpgm I0429 19:53:00.846369 3658 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-dzpgm] pod update took 39.721862ms I0429 19:53:00.846398 3658 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-dzpgm] addLogicalPort annotation time took 39.755195ms I0429 19:53:00.847623 3658 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-dzpgm] addLogicalPort took 41.072502ms, libovsdb time 846.903µs I0429 19:53:00.916721 3658 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-wd69j] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-76f587ddc9-wd69j for pod on switch ip-10-0-0-121.ec2.internal I0429 19:53:00.916818 3658 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-wd69j I0429 19:53:00.963618 3658 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-wd69j I0429 19:53:00.987713 3658 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-wd69j] pod update took 70.913771ms I0429 19:53:00.987741 3658 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-wd69j] addLogicalPort annotation time took 70.952613ms I0429 19:53:00.989144 3658 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-wd69j] addLogicalPort took 72.439919ms, libovsdb time 1.001695ms I0429 19:53:01.165339 3658 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-dzpgm 5cd98330930944a1ec0740855d49a8db06e1dc7020a8beaf0dfa5342a0f96031 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-dzpgm 5cd98330930944a1ec0740855d49a8db06e1dc7020a8beaf0dfa5342a0f96031 network default NAD default NAD key ] I0429 19:53:01.175706 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-dzpgm, hostIfaceName: 5cd98330930944a, network: default, NAD default, SandboxID: "5cd98330930944a1ec0740855d49a8db06e1dc7020a8beaf0dfa5342a0f96031", PCI device ID: , UID: "781e423b-4ec5-4e97-8204-0f2d8ff0934a", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0429 19:53:01.230559 3658 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-dzpgm 5cd98330930944a1ec0740855d49a8db06e1dc7020a8beaf0dfa5342a0f96031 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-dzpgm 5cd98330930944a1ec0740855d49a8db06e1dc7020a8beaf0dfa5342a0f96031 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:6c:03:6f:b5:2d\",\"name\":\"5cd98330930944a\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/39da9635-9161-42ad-94cd-519558fc8928\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:53:01.268264 3658 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-wd69j d721b619253feb0a94b88f98f82c40ee140e36e8169c64d942fa9d482f947400 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-wd69j d721b619253feb0a94b88f98f82c40ee140e36e8169c64d942fa9d482f947400 network default NAD default NAD key ] I0429 19:53:01.294196 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-76f587ddc9-wd69j, hostIfaceName: d721b619253feb0, network: default, NAD default, SandboxID: "d721b619253feb0a94b88f98f82c40ee140e36e8169c64d942fa9d482f947400", PCI device ID: , UID: "04b5b3a2-4b51-4861-86e8-b24eec797118", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0429 19:53:01.346239 3658 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-wd69j d721b619253feb0a94b88f98f82c40ee140e36e8169c64d942fa9d482f947400 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-wd69j d721b619253feb0a94b88f98f82c40ee140e36e8169c64d942fa9d482f947400 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:bf:99:cf:da:31\",\"name\":\"d721b619253feb0\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a4b9e85f-c0d8-4ec9-9118-e750e05dada2\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:53:04.990855 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0429 19:53:05.875648 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=234 I0429 19:53:06.187423 3658 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:53:06.188008 3658 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:53:06.208817 3658 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:53:06.208839 3658 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:53:06.213179 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0429 19:53:06.213198 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0429 19:53:08.265246 3658 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0429 19:53:08.265929 3658 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0429 19:53:08.265952 3658 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0429 19:53:08.268059 3658 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel W0429 19:53:16.006381 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0429 19:53:16.126220 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0429 19:53:16.126240 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0429 19:53:16.130329 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0429 19:53:16.130343 3658 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service I0429 19:53:17.918263 3658 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-55fb698565-rxscr] creating logical port openshift-pipelines_tekton-chains-controller-55fb698565-rxscr for pod on switch ip-10-0-0-121.ec2.internal I0429 19:53:17.918370 3658 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-55fb698565-rxscr I0429 19:53:17.932922 3658 pod.go:62] [openshift-pipelines/tekton-chains-controller-55fb698565-rxscr] pod update took 14.573543ms I0429 19:53:17.932953 3658 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-chains-controller-55fb698565-rxscr] addLogicalPort annotation time took 14.612583ms I0429 19:53:17.934292 3658 pods.go:271] [openshift-pipelines/tekton-chains-controller-55fb698565-rxscr] addLogicalPort took 16.052948ms, libovsdb time 953.176µs I0429 19:53:18.582449 3658 cni.go:452] [openshift-pipelines/tekton-chains-controller-55fb698565-rxscr 2b1e46c66fb2e71d9f5ac3b93711ab374f1c9b2f72a37bfa7431d4d95247bdee network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-rxscr 2b1e46c66fb2e71d9f5ac3b93711ab374f1c9b2f72a37bfa7431d4d95247bdee network default NAD default NAD key ] I0429 19:53:18.592003 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-55fb698565-rxscr, hostIfaceName: 2b1e46c66fb2e71, network: default, NAD default, SandboxID: "2b1e46c66fb2e71d9f5ac3b93711ab374f1c9b2f72a37bfa7431d4d95247bdee", PCI device ID: , UID: "a5cbc672-f831-4661-99e7-04d5544a35b2", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0429 19:53:18.645269 3658 cni.go:473] [openshift-pipelines/tekton-chains-controller-55fb698565-rxscr 2b1e46c66fb2e71d9f5ac3b93711ab374f1c9b2f72a37bfa7431d4d95247bdee network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-rxscr 2b1e46c66fb2e71d9f5ac3b93711ab374f1c9b2f72a37bfa7431d4d95247bdee network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:41:56:d6:c6:af\",\"name\":\"2b1e46c66fb2e71\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/65b9ed0a-1a40-4f30-bc37-43761a7009e7\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:53:24.187034 3658 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0429 19:53:30.952315 3658 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0429 19:53:38.868382 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=41 I0429 19:53:43.720400 3658 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-brvsz] creating logical port openshift-pipelines_tekton-results-watcher-7777c5dbd5-brvsz for pod on switch ip-10-0-0-121.ec2.internal I0429 19:53:43.720543 3658 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-7777c5dbd5-brvsz I0429 19:53:43.738445 3658 pod.go:62] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-brvsz] pod update took 17.956852ms I0429 19:53:43.738468 3658 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-brvsz] addLogicalPort annotation time took 17.987803ms I0429 19:53:43.739893 3658 pods.go:271] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-brvsz] addLogicalPort took 19.514144ms, libovsdb time 979.081µs I0429 19:53:44.085230 3658 cni.go:452] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-brvsz 6ddf3abb924207c715b5370c3630900625438a39b62ba88a91e775c008d0be85 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-brvsz 6ddf3abb924207c715b5370c3630900625438a39b62ba88a91e775c008d0be85 network default NAD default NAD key ] I0429 19:53:44.094728 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-7777c5dbd5-brvsz, hostIfaceName: 6ddf3abb924207c, network: default, NAD default, SandboxID: "6ddf3abb924207c715b5370c3630900625438a39b62ba88a91e775c008d0be85", PCI device ID: , UID: "8e9de7fc-1110-425d-ae71-8b2808622b53", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0429 19:53:44.149473 3658 cni.go:473] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-brvsz 6ddf3abb924207c715b5370c3630900625438a39b62ba88a91e775c008d0be85 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-brvsz 6ddf3abb924207c715b5370c3630900625438a39b62ba88a91e775c008d0be85 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:4b:12:54:19:7b\",\"name\":\"6ddf3abb924207c\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/03d0416c-84f4-4472-aab4-8ca7f8c5b87d\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:53:48.348673 3658 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-fckz4] creating logical port tsf-keycloak_tsf-iam-fckz4 for pod on switch ip-10-0-0-121.ec2.internal I0429 19:53:48.348817 3658 kube.go:256] Updating pod tsf-keycloak/tsf-iam-fckz4 I0429 19:53:48.367463 3658 pod.go:62] [tsf-keycloak/tsf-iam-fckz4] pod update took 18.683141ms I0429 19:53:48.367510 3658 base_network_controller_pods.go:951] [default/tsf-keycloak/tsf-iam-fckz4] addLogicalPort annotation time took 18.733386ms I0429 19:53:48.368759 3658 pods.go:271] [tsf-keycloak/tsf-iam-fckz4] addLogicalPort took 20.103648ms, libovsdb time 890.291µs I0429 19:53:48.709567 3658 cni.go:452] [tsf-keycloak/tsf-iam-fckz4 ec9077441bae3f967e0fb004b7e9213a82c1e3d9fb57263a49ef14daf2971c47 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/tsf-iam-fckz4 ec9077441bae3f967e0fb004b7e9213a82c1e3d9fb57263a49ef14daf2971c47 network default NAD default NAD key ] I0429 19:53:48.719159 3658 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-fckz4, hostIfaceName: ec9077441bae3f9, network: default, NAD default, SandboxID: "ec9077441bae3f967e0fb004b7e9213a82c1e3d9fb57263a49ef14daf2971c47", PCI device ID: , UID: "3bf17f79-8708-4e1c-bc6c-40b806dc1792", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0429 19:53:48.772587 3658 cni.go:473] [tsf-keycloak/tsf-iam-fckz4 ec9077441bae3f967e0fb004b7e9213a82c1e3d9fb57263a49ef14daf2971c47 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/tsf-iam-fckz4 ec9077441bae3f967e0fb004b7e9213a82c1e3d9fb57263a49ef14daf2971c47 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:9f:88:41:98:8b\",\"name\":\"ec9077441bae3f9\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/492654b5-3094-4da7-a0ad-4c3e00f2b7a3\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:53:53.199966 3658 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0429 19:53:53.199992 3658 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0429 19:53:53.199998 3658 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0429 19:53:56.793650 3658 namespace.go:142] [openshift-pipelines] updating namespace I0429 19:53:58.486404 3658 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:53:58.486989 3658 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:53:58.534695 3658 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:53:58.534749 3658 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:53:58.540422 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0429 19:53:58.540456 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam W0429 19:54:03.257233 3658 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0429 19:54:03.257255 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server I0429 19:54:09.831722 3658 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-62gdc] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-62gdc for pod on switch ip-10-0-0-121.ec2.internal I0429 19:54:09.831842 3658 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-62gdc I0429 19:54:09.844001 3658 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-62gdc] pod update took 12.196051ms I0429 19:54:09.844023 3658 base_network_controller_pods.go:951] [default/openshift-pipelines/tkn-cli-serve-784b566968-62gdc] addLogicalPort annotation time took 12.223525ms I0429 19:54:09.845366 3658 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-62gdc] addLogicalPort took 13.671627ms, libovsdb time 895.413µs I0429 19:54:10.196058 3658 cni.go:452] [openshift-pipelines/tkn-cli-serve-784b566968-62gdc 47219d581d1e017099a331535b749e0395fd4d4ae39805efeab2e650f5b3c82c network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-62gdc 47219d581d1e017099a331535b749e0395fd4d4ae39805efeab2e650f5b3c82c network default NAD default NAD key ] I0429 19:54:10.208234 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-62gdc, hostIfaceName: 47219d581d1e017, network: default, NAD default, SandboxID: "47219d581d1e017099a331535b749e0395fd4d4ae39805efeab2e650f5b3c82c", PCI device ID: , UID: "da6a7e61-8c1d-4803-8831-7ff1aa86636e", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0429 19:54:10.265973 3658 cni.go:473] [openshift-pipelines/tkn-cli-serve-784b566968-62gdc 47219d581d1e017099a331535b749e0395fd4d4ae39805efeab2e650f5b3c82c network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-62gdc 47219d581d1e017099a331535b749e0395fd4d4ae39805efeab2e650f5b3c82c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:86:52:a0:a3:74\",\"name\":\"47219d581d1e017\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8ab1d2b9-4a3c-4b98-997a-73acb2733551\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:54:10.317611 3658 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0429 19:54:10.351645 3658 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0429 19:54:10.867803 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=85 I0429 19:54:12.361725 3658 cni.go:452] [tsf-keycloak/tsf-iam-fckz4 ec9077441bae3f967e0fb004b7e9213a82c1e3d9fb57263a49ef14daf2971c47 network default NAD default NAD key ] DEL starting CNI request [tsf-keycloak/tsf-iam-fckz4 ec9077441bae3f967e0fb004b7e9213a82c1e3d9fb57263a49ef14daf2971c47 network default NAD default NAD key ] I0429 19:54:12.455160 3658 cni.go:473] [tsf-keycloak/tsf-iam-fckz4 ec9077441bae3f967e0fb004b7e9213a82c1e3d9fb57263a49ef14daf2971c47 network default NAD default NAD key default] DEL finished CNI request [tsf-keycloak/tsf-iam-fckz4 ec9077441bae3f967e0fb004b7e9213a82c1e3d9fb57263a49ef14daf2971c47 network default NAD default NAD key default], result "{}", err I0429 19:54:13.732386 3658 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-6557z-nddqz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:54:13.733041 3658 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-6557z-nddqz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:54:14.359641 3658 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-fckz4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:54:14.359682 3658 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-fckz4 I0429 19:54:14.361101 3658 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-fckz4, ips: 10.129.0.35 I0429 19:54:14.361131 3658 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-fckz4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0429 19:54:14.454351 3658 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0429 19:54:14.757651 3658 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-6557z-nddqz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:54:14.757675 3658 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-6557z-nddqz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:54:15.386844 3658 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-fckz4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:54:15.386869 3658 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-fckz4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:54:25.216240 3658 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-f25vq] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-f25vq for pod on switch ip-10-0-0-121.ec2.internal I0429 19:54:25.216333 3658 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-f25vq I0429 19:54:25.233460 3658 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-f25vq] pod update took 17.146498ms I0429 19:54:25.233509 3658 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-f25vq] addLogicalPort annotation time took 17.182568ms I0429 19:54:25.234825 3658 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-f25vq] addLogicalPort took 18.601381ms, libovsdb time 873.401µs I0429 19:54:25.321714 3658 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-bjbfb] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-bjbfb for pod on switch ip-10-0-0-121.ec2.internal I0429 19:54:25.321805 3658 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-bjbfb I0429 19:54:25.339221 3658 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-bjbfb] pod update took 17.435377ms I0429 19:54:25.339246 3658 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-bjbfb] addLogicalPort annotation time took 17.47015ms I0429 19:54:25.340624 3658 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-bjbfb] addLogicalPort took 18.925158ms, libovsdb time 913.141µs I0429 19:54:25.579565 3658 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-f25vq 19c2ed80c99ee6547ddcc117dbe0c745a901557640f6d4242d09ec91d05a51b8 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-f25vq 19c2ed80c99ee6547ddcc117dbe0c745a901557640f6d4242d09ec91d05a51b8 network default NAD default NAD key ] I0429 19:54:25.589207 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-f25vq, hostIfaceName: 19c2ed80c99ee65, network: default, NAD default, SandboxID: "19c2ed80c99ee6547ddcc117dbe0c745a901557640f6d4242d09ec91d05a51b8", PCI device ID: , UID: "426a5fe0-8d95-41b8-9fdb-54b698499f92", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0429 19:54:25.646106 3658 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-f25vq 19c2ed80c99ee6547ddcc117dbe0c745a901557640f6d4242d09ec91d05a51b8 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-f25vq 19c2ed80c99ee6547ddcc117dbe0c745a901557640f6d4242d09ec91d05a51b8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:fa:ca:37:61:c3\",\"name\":\"19c2ed80c99ee65\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2aa1f9f0-9ac6-4b6a-b9fd-3bfbd5b7c775\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:54:25.685626 3658 cni.go:452] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-bjbfb f72bd75670186214f24b5ff2821adc8d106f6bd395fa564e9212cbb18955a1eb network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-bjbfb f72bd75670186214f24b5ff2821adc8d106f6bd395fa564e9212cbb18955a1eb network default NAD default NAD key ] I0429 19:54:25.695529 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-bjbfb, hostIfaceName: f72bd7567018621, network: default, NAD default, SandboxID: "f72bd75670186214f24b5ff2821adc8d106f6bd395fa564e9212cbb18955a1eb", PCI device ID: , UID: "4f818e44-0962-4efc-a738-2165d6bb46ee", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0429 19:54:25.749935 3658 cni.go:473] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-bjbfb f72bd75670186214f24b5ff2821adc8d106f6bd395fa564e9212cbb18955a1eb network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-bjbfb f72bd75670186214f24b5ff2821adc8d106f6bd395fa564e9212cbb18955a1eb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:e5:bd:b8:bd:1f\",\"name\":\"f72bd7567018621\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/aaeb4782-de1f-45b3-b04f-e2ae81ddcf51\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:54:28.446114 3658 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0429 19:54:29.527578 3658 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0429 19:54:29.527601 3658 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0429 19:54:39.458301 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0429 19:54:40.538146 3658 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-pnjcp] creating logical port openshift-pipelines_pipelines-console-plugin-65f788bc74-pnjcp for pod on switch ip-10-0-0-121.ec2.internal I0429 19:54:40.538239 3658 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-65f788bc74-pnjcp I0429 19:54:40.551977 3658 pod.go:62] [openshift-pipelines/pipelines-console-plugin-65f788bc74-pnjcp] pod update took 13.75858ms I0429 19:54:40.552004 3658 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-pnjcp] addLogicalPort annotation time took 13.792856ms I0429 19:54:40.553363 3658 pods.go:271] [openshift-pipelines/pipelines-console-plugin-65f788bc74-pnjcp] addLogicalPort took 15.237843ms, libovsdb time 900.3µs I0429 19:54:40.899711 3658 cni.go:452] [openshift-pipelines/pipelines-console-plugin-65f788bc74-pnjcp bb3a5d3a63e41a226de04fe94c33d04c1debacc2f3f6305e38ecd47446eb6077 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-pnjcp bb3a5d3a63e41a226de04fe94c33d04c1debacc2f3f6305e38ecd47446eb6077 network default NAD default NAD key ] I0429 19:54:40.928593 3658 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-65f788bc74-pnjcp, hostIfaceName: bb3a5d3a63e41a2, network: default, NAD default, SandboxID: "bb3a5d3a63e41a226de04fe94c33d04c1debacc2f3f6305e38ecd47446eb6077", PCI device ID: , UID: "9c7cffc9-abd1-453c-b1ee-38fdc823ef31", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0429 19:54:40.982718 3658 cni.go:473] [openshift-pipelines/pipelines-console-plugin-65f788bc74-pnjcp bb3a5d3a63e41a226de04fe94c33d04c1debacc2f3f6305e38ecd47446eb6077 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-pnjcp bb3a5d3a63e41a226de04fe94c33d04c1debacc2f3f6305e38ecd47446eb6077 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:f3:00:8d:bb:eb\",\"name\":\"bb3a5d3a63e41a2\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cfcf7519-801c-400f-b3a7-0816a077beba\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:54:48.759840 3658 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:54:48.760448 3658 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:54:48.782241 3658 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:54:48.782264 3658 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:54:48.788542 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0429 19:54:48.788569 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0429 19:54:54.988683 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0429 19:54:58.764795 3658 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-sb9vf-xl2rt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:54:58.765335 3658 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-sb9vf-xl2rt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:54:58.794844 3658 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-54q99-lkm9k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:54:58.795354 3658 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-54q99-lkm9k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:54:58.822845 3658 obj_retry.go:498] Detected object tsf/patch-tekton-config-bhnrw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:54:58.823401 3658 obj_retry.go:498] Detected object tsf/patch-tekton-config-bhnrw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:54:59.600916 3658 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-54q99-lkm9k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:54:59.600943 3658 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-54q99-lkm9k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:54:59.624920 3658 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-sb9vf-xl2rt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:54:59.624944 3658 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-sb9vf-xl2rt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:54:59.705237 3658 obj_retry.go:492] Detected object tsf/patch-tekton-config-bhnrw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:54:59.705262 3658 obj_retry.go:492] Detected object tsf/patch-tekton-config-bhnrw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0429 19:55:04.623223 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0429 19:55:05.727437 3658 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0429 19:55:05.831908 3658 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-xjv75 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:55:05.832477 3658 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-xjv75 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:55:06.853393 3658 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-xjv75 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:06.853415 3658 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-xjv75 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:06.894607 3658 obj_retry.go:492] Detected object tsf/patch-tekton-config-bhnrw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:06.894635 3658 obj_retry.go:492] Detected object tsf/patch-tekton-config-bhnrw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:06.900719 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-bhnrw I0429 19:55:06.900748 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-bhnrw W0429 19:55:07.639641 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0429 19:55:10.898190 3658 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:55:10.898798 3658 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:55:10.918812 3658 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:10.918835 3658 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:10.924604 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0429 19:55:10.924627 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa W0429 19:55:16.808819 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0429 19:55:17.966010 3658 obj_retry.go:498] Detected object tsf-tpa/create-db-rtkfj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:55:17.966585 3658 obj_retry.go:498] Detected object tsf-tpa/create-db-rtkfj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:55:18.990843 3658 obj_retry.go:492] Detected object tsf-tpa/create-db-rtkfj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:18.990870 3658 obj_retry.go:492] Detected object tsf-tpa/create-db-rtkfj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:19.106626 3658 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-6z4cf] creating logical port tsf-tpa_migrate-db-6z4cf for pod on switch ip-10-0-0-121.ec2.internal I0429 19:55:19.106743 3658 kube.go:256] Updating pod tsf-tpa/migrate-db-6z4cf I0429 19:55:19.119766 3658 pod.go:62] [tsf-tpa/migrate-db-6z4cf] pod update took 13.042446ms I0429 19:55:19.119791 3658 base_network_controller_pods.go:951] [default/tsf-tpa/migrate-db-6z4cf] addLogicalPort annotation time took 13.071878ms I0429 19:55:19.121194 3658 pods.go:271] [tsf-tpa/migrate-db-6z4cf] addLogicalPort took 14.58487ms, libovsdb time 875.667µs I0429 19:55:19.468841 3658 cni.go:452] [tsf-tpa/migrate-db-6z4cf c287dc8117af12c2d66b34b959a6dc738f01084b22d139cbc3b766c289c81861 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/migrate-db-6z4cf c287dc8117af12c2d66b34b959a6dc738f01084b22d139cbc3b766c289c81861 network default NAD default NAD key ] I0429 19:55:19.478215 3658 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-6z4cf, hostIfaceName: c287dc8117af12c, network: default, NAD default, SandboxID: "c287dc8117af12c2d66b34b959a6dc738f01084b22d139cbc3b766c289c81861", PCI device ID: , UID: "16676d8f-b8f7-43d0-975a-ef11f5e1bccd", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0429 19:55:19.534944 3658 cni.go:473] [tsf-tpa/migrate-db-6z4cf c287dc8117af12c2d66b34b959a6dc738f01084b22d139cbc3b766c289c81861 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/migrate-db-6z4cf c287dc8117af12c2d66b34b959a6dc738f01084b22d139cbc3b766c289c81861 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:a1:a7:23:64:30\",\"name\":\"c287dc8117af12c\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b1a73ade-f2d8-4558-bed1-369042385c08\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:55:25.756467 3658 cni.go:452] [tsf-tpa/migrate-db-6z4cf c287dc8117af12c2d66b34b959a6dc738f01084b22d139cbc3b766c289c81861 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/migrate-db-6z4cf c287dc8117af12c2d66b34b959a6dc738f01084b22d139cbc3b766c289c81861 network default NAD default NAD key ] I0429 19:55:25.845969 3658 cni.go:473] [tsf-tpa/migrate-db-6z4cf c287dc8117af12c2d66b34b959a6dc738f01084b22d139cbc3b766c289c81861 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/migrate-db-6z4cf c287dc8117af12c2d66b34b959a6dc738f01084b22d139cbc3b766c289c81861 network default NAD default NAD key default], result "{}", err I0429 19:55:25.901751 3658 obj_retry.go:498] Detected object tsf-tpa/migrate-db-6z4cf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:55:25.901785 3658 pods.go:176] Deleting pod: tsf-tpa/migrate-db-6z4cf I0429 19:55:25.903249 3658 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-6z4cf, ips: 10.129.0.40 I0429 19:55:25.903280 3658 obj_retry.go:498] Detected object tsf-tpa/migrate-db-6z4cf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:55:26.773496 3658 obj_retry.go:492] Detected object tsf-tpa/migrate-db-6z4cf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:26.773519 3658 obj_retry.go:492] Detected object tsf-tpa/migrate-db-6z4cf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:29.012868 3658 obj_retry.go:498] Detected object tsf-tpa/create-importers-xmjtv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:55:29.013408 3658 obj_retry.go:498] Detected object tsf-tpa/create-importers-xmjtv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:55:29.923361 3658 base_network_controller_pods.go:487] [default/tsf-tpa/server-67749b6d87-sr6st] creating logical port tsf-tpa_server-67749b6d87-sr6st for pod on switch ip-10-0-0-121.ec2.internal I0429 19:55:29.923467 3658 kube.go:256] Updating pod tsf-tpa/server-67749b6d87-sr6st I0429 19:55:29.939767 3658 pod.go:62] [tsf-tpa/server-67749b6d87-sr6st] pod update took 16.328004ms I0429 19:55:29.939791 3658 base_network_controller_pods.go:951] [default/tsf-tpa/server-67749b6d87-sr6st] addLogicalPort annotation time took 16.356856ms I0429 19:55:29.941064 3658 pods.go:271] [tsf-tpa/server-67749b6d87-sr6st] addLogicalPort took 17.727217ms, libovsdb time 849.58µs I0429 19:55:30.041325 3658 obj_retry.go:492] Detected object tsf-tpa/create-importers-xmjtv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:30.041351 3658 obj_retry.go:492] Detected object tsf-tpa/create-importers-xmjtv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:31.025178 3658 base_network_controller_pods.go:487] [default/tsf-tpa/importer-86bcdf477b-dxlgm] creating logical port tsf-tpa_importer-86bcdf477b-dxlgm for pod on switch ip-10-0-0-121.ec2.internal I0429 19:55:31.025297 3658 kube.go:256] Updating pod tsf-tpa/importer-86bcdf477b-dxlgm I0429 19:55:31.040397 3658 pod.go:62] [tsf-tpa/importer-86bcdf477b-dxlgm] pod update took 15.13486ms I0429 19:55:31.040420 3658 base_network_controller_pods.go:951] [default/tsf-tpa/importer-86bcdf477b-dxlgm] addLogicalPort annotation time took 15.164892ms I0429 19:55:31.041711 3658 pods.go:271] [tsf-tpa/importer-86bcdf477b-dxlgm] addLogicalPort took 16.549886ms, libovsdb time 850.58µs I0429 19:55:38.995643 3658 cni.go:452] [tsf-tpa/server-67749b6d87-sr6st dcfc82114b82b54d411df7b27de3df9d7b05e8485c1a3abc5a3714332500fd75 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-67749b6d87-sr6st dcfc82114b82b54d411df7b27de3df9d7b05e8485c1a3abc5a3714332500fd75 network default NAD default NAD key ] I0429 19:55:39.005853 3658 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: server-67749b6d87-sr6st, hostIfaceName: dcfc82114b82b54, network: default, NAD default, SandboxID: "dcfc82114b82b54d411df7b27de3df9d7b05e8485c1a3abc5a3714332500fd75", PCI device ID: , UID: "1962959a-0663-4744-8926-3970d1471f08", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0429 19:55:39.062555 3658 cni.go:473] [tsf-tpa/server-67749b6d87-sr6st dcfc82114b82b54d411df7b27de3df9d7b05e8485c1a3abc5a3714332500fd75 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-67749b6d87-sr6st dcfc82114b82b54d411df7b27de3df9d7b05e8485c1a3abc5a3714332500fd75 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:ca:bb:eb:79:3e\",\"name\":\"dcfc82114b82b54\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ebda6ef9-fe86-4ff4-9df8-10fdb4a49f50\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 19:55:39.187774 3658 cni.go:452] [tsf-tpa/importer-86bcdf477b-dxlgm 214f8a8aaa5e2918e412a9959c6fd6910da399ab6230d920e6a34410d6bf2397 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/importer-86bcdf477b-dxlgm 214f8a8aaa5e2918e412a9959c6fd6910da399ab6230d920e6a34410d6bf2397 network default NAD default NAD key ] I0429 19:55:39.197902 3658 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: importer-86bcdf477b-dxlgm, hostIfaceName: 214f8a8aaa5e291, network: default, NAD default, SandboxID: "214f8a8aaa5e2918e412a9959c6fd6910da399ab6230d920e6a34410d6bf2397", PCI device ID: , UID: "31cab357-a298-4d9e-9b3e-b53b2737d67a", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0429 19:55:39.254121 3658 cni.go:473] [tsf-tpa/importer-86bcdf477b-dxlgm 214f8a8aaa5e2918e412a9959c6fd6910da399ab6230d920e6a34410d6bf2397 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/importer-86bcdf477b-dxlgm 214f8a8aaa5e2918e412a9959c6fd6910da399ab6230d920e6a34410d6bf2397 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:e6:bd:d4:84:f9\",\"name\":\"214f8a8aaa5e291\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ee48255-34c6-4751-ac8b-feea0a074edf\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 19:55:39.832263 3658 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0429 19:55:50.139058 3658 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-64rlv-9dc68 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:55:50.139628 3658 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-64rlv-9dc68 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0429 19:55:50.882660 3658 util.go:959] Failed to build node endpoints for node ip-10-0-0-121.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0429 19:55:51.017194 3658 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-64rlv-9dc68 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:55:51.017218 3658 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-64rlv-9dc68 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0429 19:55:55.027111 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0429 19:56:05.222596 3658 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:56:05.223198 3658 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:56:05.269186 3658 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:56:05.269212 3658 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:56:05.275648 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0429 19:56:05.275675 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0429 19:56:06.034594 3658 namespace.go:98] [integration-service] adding namespace I0429 19:56:06.034628 3658 namespace.go:98] [default-tenant] adding namespace I0429 19:56:06.034811 3658 namespace.go:98] [image-controller] adding namespace I0429 19:56:06.035477 3658 namespace.go:102] [integration-service] adding namespace took 842.332µs I0429 19:56:06.035537 3658 namespace.go:98] [build-service] adding namespace I0429 19:56:06.036073 3658 namespace.go:102] [default-tenant] adding namespace took 1.432323ms I0429 19:56:06.036715 3658 namespace.go:102] [image-controller] adding namespace took 1.88878ms I0429 19:56:06.037187 3658 namespace.go:102] [build-service] adding namespace took 1.641815ms I0429 19:56:06.199555 3658 namespace.go:142] [default-tenant] updating namespace I0429 19:56:06.235581 3658 namespace.go:142] [default-tenant] updating namespace I0429 19:56:06.243871 3658 namespace.go:142] [integration-service] updating namespace I0429 19:56:06.271745 3658 namespace.go:142] [integration-service] updating namespace I0429 19:56:06.276295 3658 namespace.go:142] [build-service] updating namespace I0429 19:56:06.311624 3658 namespace.go:142] [image-controller] updating namespace I0429 19:56:06.314479 3658 namespace.go:142] [build-service] updating namespace I0429 19:56:06.339527 3658 namespace.go:142] [image-controller] updating namespace I0429 19:56:06.394802 3658 namespace.go:142] [build-service] updating namespace I0429 19:56:06.434248 3658 namespace.go:142] [integration-service] updating namespace I0429 19:56:06.482766 3658 namespace.go:98] [release-service] adding namespace I0429 19:56:06.483605 3658 namespace.go:102] [release-service] adding namespace took 819.299µs I0429 19:56:06.516698 3658 namespace.go:142] [release-service] updating namespace I0429 19:56:06.522208 3658 namespace.go:142] [konflux-ui] updating namespace I0429 19:56:06.547629 3658 namespace.go:142] [release-service] updating namespace I0429 19:56:06.634763 3658 namespace.go:98] [konflux-info] adding namespace I0429 19:56:06.635633 3658 namespace.go:102] [konflux-info] adding namespace took 848.748µs I0429 19:56:06.688153 3658 namespace.go:142] [konflux-info] updating namespace I0429 19:56:06.746567 3658 namespace.go:142] [konflux-info] updating namespace I0429 19:56:06.746610 3658 namespace.go:98] [namespace-lister] adding namespace I0429 19:56:06.747455 3658 namespace.go:102] [namespace-lister] adding namespace took 834.563µs I0429 19:56:06.794578 3658 namespace.go:142] [namespace-lister] updating namespace I0429 19:56:06.897818 3658 namespace.go:98] [enterprise-contract-service] adding namespace I0429 19:56:06.898694 3658 namespace.go:102] [enterprise-contract-service] adding namespace took 852.391µs I0429 19:56:06.994576 3658 namespace.go:142] [image-controller] updating namespace I0429 19:56:07.031762 3658 namespace.go:142] [enterprise-contract-service] updating namespace I0429 19:56:07.086739 3658 namespace.go:142] [namespace-lister] updating namespace I0429 19:56:07.172413 3658 namespace.go:142] [enterprise-contract-service] updating namespace I0429 19:56:07.320844 3658 namespace.go:142] [default-tenant] updating namespace I0429 19:56:07.527580 3658 namespace.go:98] [segment-bridge] adding namespace I0429 19:56:07.528457 3658 namespace.go:102] [segment-bridge] adding namespace took 856.934µs I0429 19:56:07.642592 3658 namespace.go:142] [segment-bridge] updating namespace I0429 19:56:07.859469 3658 namespace.go:142] [segment-bridge] updating namespace I0429 19:56:08.013922 3658 namespace.go:98] [konflux-cli] adding namespace I0429 19:56:08.014858 3658 namespace.go:102] [konflux-cli] adding namespace took 913.567µs I0429 19:56:08.076536 3658 namespace.go:142] [konflux-cli] updating namespace I0429 19:56:08.215693 3658 namespace.go:142] [konflux-cli] updating namespace I0429 19:56:08.492887 3658 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0429 19:56:08.493620 3658 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0429 19:56:08.493638 3658 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0429 19:56:08.496411 3658 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0429 19:56:08.688615 3658 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0429 19:56:08.688642 3658 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0429 19:56:08.689500 3658 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0429 19:56:09.438667 3658 namespace.go:142] [build-service] updating namespace I0429 19:56:09.864068 3658 namespace.go:142] [default-tenant] updating namespace I0429 19:56:10.154757 3658 namespace.go:142] [enterprise-contract-service] updating namespace I0429 19:56:10.336663 3658 namespace.go:142] [image-controller] updating namespace I0429 19:56:10.567183 3658 namespace.go:142] [integration-service] updating namespace I0429 19:56:10.750308 3658 namespace.go:142] [konflux-cli] updating namespace I0429 19:56:10.950171 3658 namespace.go:142] [konflux-info] updating namespace I0429 19:56:11.150142 3658 namespace.go:142] [namespace-lister] updating namespace I0429 19:56:11.351382 3658 namespace.go:142] [release-service] updating namespace I0429 19:56:11.567026 3658 namespace.go:142] [segment-bridge] updating namespace I0429 19:56:11.752169 3658 namespace.go:142] [build-service] updating namespace I0429 19:56:11.786585 3658 namespace.go:142] [default-tenant] updating namespace I0429 19:56:11.822702 3658 namespace.go:142] [enterprise-contract-service] updating namespace I0429 19:56:11.851125 3658 namespace.go:142] [image-controller] updating namespace I0429 19:56:11.887639 3658 namespace.go:142] [integration-service] updating namespace I0429 19:56:11.922413 3658 namespace.go:142] [konflux-cli] updating namespace I0429 19:56:11.964249 3658 namespace.go:142] [konflux-info] updating namespace I0429 19:56:11.998259 3658 namespace.go:142] [namespace-lister] updating namespace I0429 19:56:12.023606 3658 namespace.go:142] [release-service] updating namespace I0429 19:56:12.060918 3658 namespace.go:142] [segment-bridge] updating namespace I0429 19:56:12.179410 3658 namespace.go:142] [build-service] updating namespace I0429 19:56:12.322788 3658 namespace.go:142] [default-tenant] updating namespace I0429 19:56:12.494786 3658 namespace.go:142] [enterprise-contract-service] updating namespace I0429 19:56:12.653282 3658 namespace.go:142] [image-controller] updating namespace I0429 19:56:12.848892 3658 namespace.go:142] [integration-service] updating namespace I0429 19:56:12.962859 3658 namespace.go:142] [konflux-cli] updating namespace I0429 19:56:13.092663 3658 namespace.go:142] [konflux-info] updating namespace I0429 19:56:13.254188 3658 namespace.go:142] [namespace-lister] updating namespace I0429 19:56:13.330084 3658 namespace.go:142] [release-service] updating namespace I0429 19:56:13.398253 3658 namespace.go:142] [segment-bridge] updating namespace W0429 19:56:17.171685 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0429 19:56:17.223782 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0429 19:56:17.254087 3658 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0429 19:56:19.198061 3658 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0429 19:56:22.866667 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=78 W0429 19:56:23.238000 3658 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0429 19:56:23.238019 3658 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0429 19:56:23.268827 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0429 19:56:23.276012 3658 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0429 19:56:23.310046 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0429 19:56:23.311858 3658 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0429 19:56:23.334279 3658 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0429 19:56:23.339393 3658 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0429 19:56:24.076650 3658 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0429 19:56:24.076670 3658 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0429 19:56:32.873196 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=162 I0429 19:56:34.628453 3658 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-k5h72 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:56:34.629084 3658 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-k5h72 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:56:35.327523 3658 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-k5h72 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:56:35.327548 3658 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-k5h72 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:56:35.334287 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-k5h72 I0429 19:56:35.334315 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-k5h72 I0429 19:57:32.479969 3658 obj_retry.go:498] Detected object konflux-ui/proxy-fcf9db6b-5w76q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:57:32.480580 3658 obj_retry.go:498] Detected object konflux-ui/proxy-fcf9db6b-5w76q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:57:32.656232 3658 obj_retry.go:492] Detected object konflux-ui/proxy-fcf9db6b-5w76q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:57:32.656260 3658 obj_retry.go:492] Detected object konflux-ui/proxy-fcf9db6b-5w76q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:57:32.665432 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-fcf9db6b-5w76q I0429 19:57:32.665458 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-fcf9db6b-5w76q I0429 19:57:38.847338 3658 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 19:57:38.848007 3658 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 19:57:38.872071 3658 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:57:38.872092 3658 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 19:57:38.876551 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0429 19:57:38.876576 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0429 19:58:13.725744 3658 namespace.go:98] [managed-tenant-i8t4y] adding namespace I0429 19:58:13.726709 3658 namespace.go:102] [managed-tenant-i8t4y] adding namespace took 936.995µs I0429 19:58:13.751642 3658 namespace.go:142] [managed-tenant-i8t4y] updating namespace I0429 19:58:13.773146 3658 namespace.go:142] [managed-tenant-i8t4y] updating namespace I0429 19:58:14.925681 3658 namespace.go:142] [managed-tenant-i8t4y] updating namespace I0429 19:58:15.060561 3658 namespace.go:142] [managed-tenant-i8t4y] updating namespace I0429 19:58:15.157740 3658 namespace.go:142] [managed-tenant-i8t4y] updating namespace I0429 19:58:25.980704 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0429 19:58:59.976305 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0429 19:59:00.966610 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0429 20:00:04.658570 3658 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624880-6l27v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 20:00:04.659185 3658 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624880-6l27v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 20:00:05.533822 3658 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624880-6l27v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 20:00:05.533849 3658 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624880-6l27v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 20:00:07.971284 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0429 20:00:10.992778 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0429 20:00:34.879685 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=12 I0429 20:00:47.991870 3658 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0429 20:00:53.992244 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=5 I0429 20:01:35.872746 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=46 I0429 20:01:37.960025 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0429 20:01:41.956385 3658 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0429 20:01:50.871206 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=31 I0429 20:02:29.879785 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=417 I0429 20:02:32.871606 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=82 I0429 20:02:40.955658 3658 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 W0429 20:03:15.522649 3658 duration.go:461] Config duration recorder: measurement expired for pod/openshift-backplane/osd-delete-backplane-serviceaccounts-29624860-c4pzr I0429 20:05:16.980168 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0429 20:05:18.051095 3658 obj_retry.go:492] Detected object tsf-tpa/create-db-rtkfj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 20:05:18.051136 3658 obj_retry.go:492] Detected object tsf-tpa/create-db-rtkfj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 20:05:18.055830 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-rtkfj I0429 20:05:18.055861 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-rtkfj I0429 20:05:26.039690 3658 obj_retry.go:492] Detected object tsf-tpa/migrate-db-6z4cf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 20:05:26.039746 3658 obj_retry.go:492] Detected object tsf-tpa/migrate-db-6z4cf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 20:05:26.043524 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-6z4cf I0429 20:05:26.043545 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-6z4cf I0429 20:05:30.049985 3658 obj_retry.go:492] Detected object tsf-tpa/create-importers-xmjtv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 20:05:30.050043 3658 obj_retry.go:492] Detected object tsf-tpa/create-importers-xmjtv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 20:05:30.055692 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-xmjtv I0429 20:05:30.055718 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-xmjtv I0429 20:06:27.876958 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=16 I0429 20:06:29.883592 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=5 I0429 20:06:45.974964 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0429 20:07:00.984617 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0429 20:07:21.995260 3658 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0429 20:07:42.960335 3658 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=5 I0429 20:07:57.884156 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=16 I0429 20:08:16.875481 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=6 I0429 20:08:23.970291 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0429 20:09:06.874901 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=36 I0429 20:09:11.963927 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0429 20:09:17.959371 3658 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0429 20:09:28.996936 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0429 20:10:03.856363 3658 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624890-zdfsz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 20:10:03.856972 3658 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624890-zdfsz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 20:10:04.719422 3658 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624890-zdfsz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 20:10:04.719474 3658 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624890-zdfsz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 20:10:04.760746 3658 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624860-c4pzr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 20:10:04.760775 3658 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624860-c4pzr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 20:10:04.764796 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29624860-c4pzr I0429 20:10:04.764834 3658 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29624860-c4pzr I0429 20:10:11.995703 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0429 20:10:49.876628 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=10 I0429 20:11:38.983721 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0429 20:12:32.999669 3658 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0429 20:12:56.881096 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=8 I0429 20:13:00.888070 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=18 I0429 20:13:27.974030 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0429 20:14:11.988977 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0429 20:14:31.001272 3658 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=5 I0429 20:14:35.883572 3658 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=8