+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-78.ec2.internal ++ [[ -n ip-10-0-0-78.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-78.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0514 23:14:37.554027521 - disable conntrack on geneve port' I0514 23:14:37.554027521 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0514 23:14:37.570437262 - starting ovnkube-node' I0514 23:14:37.570437262 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-78.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-78.ec2.internal --init-node ip-10-0-0-78.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-78.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0514 23:14:37.612806 3663 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0514 23:14:37.612873 3663 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-bdaf609cfa.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0514 23:14:37.614229 3663 kube.go:431] Waiting for certificate I0514 23:14:37.614292 3663 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0514 23:14:37.614353 3663 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0514 23:14:37.615371 3663 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0514 23:14:37.645451 3663 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0514 23:14:37.645470 3663 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0514 23:14:37.650089 3663 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-f5tkg" I0514 23:14:37.650375 3663 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0514 23:14:37.656975 3663 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-f5tkg" I0514 23:14:37.657041 3663 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0514 23:14:38.615238 3663 kube.go:438] Certificate found I0514 23:14:38.615811 3663 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0514 23:14:38.616882 3663 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0514 23:14:38.660773 3663 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-05-15 23:09:37 +0000 UTC" deadline="2026-05-15 20:36:02.469658471 +0000 UTC" I0514 23:14:38.660925 3663 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="21h21m23.808749278s" I0514 23:14:38.688903 3663 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0514 23:14:38.701855 3663 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0514 23:14:38.701865 3663 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0514 23:14:38.701918 3663 server.go:101] MetricServer registers OVS metrics I0514 23:14:38.701920 3663 factory.go:561] Starting watch factory I0514 23:14:38.702007 3663 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.702031 3663 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.702186 3663 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.702235 3663 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.702236 3663 server.go:105] MetricServer registers OVN DB metrics I0514 23:14:38.702324 3663 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.702351 3663 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.702605 3663 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.702634 3663 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.702644 3663 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.702656 3663 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.703395 3663 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.703417 3663 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.706207 3663 controller_manager.go:370] Starting the ovnkube controller I0514 23:14:38.706268 3663 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-78.ec2.internal I0514 23:14:38.706332 3663 controller_manager.go:395] NBDB zone sync took: 49.346µs I0514 23:14:38.706344 3663 factory.go:561] Starting watch factory I0514 23:14:38.707441 3663 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0514 23:14:38.707461 3663 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0514 23:14:38.719062 3663 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.719368 3663 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.719968 3663 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.720387 3663 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.721603 3663 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0514 23:14:38.721707 3663 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0514 23:14:38.721796 3663 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.726386 3663 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0514 23:14:38.726426 3663 server.go:109] MetricServer registers OVN Controller metrics I0514 23:14:38.730028 3663 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:38.731261 3663 server.go:113] MetricServer registers OVN Northd metrics I0514 23:14:38.735615 3663 metrics.go:519] OVN Metrics Server starts to run ... I0514 23:14:38.807580 3663 factory.go:1977] *v1.Service informer cache synced successfully I0514 23:14:38.807600 3663 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0514 23:14:38.807605 3663 factory.go:1977] *v1.Pod informer cache synced successfully I0514 23:14:38.807608 3663 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0514 23:14:38.807611 3663 factory.go:1977] *v1.Namespace informer cache synced successfully I0514 23:14:38.807614 3663 factory.go:1977] *v1.Node informer cache synced successfully I0514 23:14:38.807743 3663 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:38.807773 3663 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:38.807770 3663 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:38.807792 3663 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:38.810337 3663 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:38.810340 3663 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:38.813361 3663 factory.go:1977] *v1.Pod informer cache synced successfully I0514 23:14:38.813372 3663 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0514 23:14:38.813381 3663 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0514 23:14:38.813376 3663 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0514 23:14:38.813425 3663 factory.go:1977] *v1.Namespace informer cache synced successfully I0514 23:14:38.813432 3663 factory.go:1977] *v1.Node informer cache synced successfully I0514 23:14:38.813435 3663 factory.go:1977] *v1.Service informer cache synced successfully I0514 23:14:38.813438 3663 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0514 23:14:38.813438 3663 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.813449 3663 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.815699 3663 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.819409 3663 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0514 23:14:38.819420 3663 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0514 23:14:38.819480 3663 factory.go:1977] *v1.EgressIP informer cache synced successfully I0514 23:14:38.819532 3663 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.819542 3663 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.822549 3663 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.824452 3663 factory.go:1977] *v1.EgressIP informer cache synced successfully I0514 23:14:38.824540 3663 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0514 23:14:38.824601 3663 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.824613 3663 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.827927 3663 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.829848 3663 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0514 23:14:38.829916 3663 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.829935 3663 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.829934 3663 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0514 23:14:38.833509 3663 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.835531 3663 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0514 23:14:38.835541 3663 factory.go:1977] *v1.EgressService informer cache synced successfully I0514 23:14:38.835604 3663 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.835641 3663 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.837760 3663 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.841543 3663 factory.go:1977] *v1.EgressService informer cache synced successfully I0514 23:14:38.841565 3663 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0514 23:14:38.841850 3663 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0514 23:14:38.841865 3663 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0514 23:14:38.843902 3663 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0514 23:14:38.846852 3663 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0514 23:14:38.846908 3663 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.846917 3663 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.846986 3663 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0514 23:14:38.846990 3663 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.847013 3663 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.851338 3663 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.851436 3663 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:38.852389 3663 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0514 23:14:38.852399 3663 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0514 23:14:38.852401 3663 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0514 23:14:38.852406 3663 factory.go:680] Watch Factory start up complete, took: 150.489118ms I0514 23:14:38.852466 3663 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0514 23:14:38.857847 3663 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0514 23:14:38.857860 3663 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0514 23:14:38.857868 3663 factory.go:680] Watch Factory start up complete, took: 151.523081ms I0514 23:14:38.857874 3663 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-0-78.ec2.internal" zone I0514 23:14:38.884966 3663 default_node_network_controller.go:710] Initializing the default node network controller I0514 23:14:38.941358 3663 udn_isolation.go:101] Starting UDN host isolation manager I0514 23:14:38.947508 3663 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0514 23:14:38.992437 3663 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0514 23:14:38.992568 3663 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0514 23:14:38.992584 3663 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0514 23:14:39.023423 3663 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0514 23:14:39.023479 3663 default_node_network_controller.go:809] Node ip-10-0-0-78.ec2.internal ready for ovn initialization with subnet 10.130.0.0/23 I0514 23:14:39.028417 3663 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.78"] k8s.ovn.org/zone-name:ip-10-0-0-78.ec2.internal] on node ip-10-0-0-78.ec2.internal I0514 23:14:39.049748 3663 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0514 23:14:39.055569 3663 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0514 23:14:39.055758 3663 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0514 23:14:39.055919 3663 gateway_init.go:263] Preparing Gateway I0514 23:14:39.055931 3663 gateway_shared_intf.go:1687] Creating new gateway I0514 23:14:39.108316 3663 controller_manager.go:429] Waiting for node in zone sync took: 250.430002ms I0514 23:14:39.114745 3663 controller_manager.go:611] Switching to transit router for layer2 networks I0514 23:14:39.114759 3663 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-78.ec2.internal I0514 23:14:39.131344 3663 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0514 23:14:39.135236 3663 iptables.go:108] Creating table: filter chain: FORWARD I0514 23:14:39.137254 3663 controller_manager.go:290] SCTP support detected in OVN I0514 23:14:39.137952 3663 services_controller.go:69] Creating services controller for network=default I0514 23:14:39.138070 3663 udn_enabled_svc.go:80] Waiting for service informer to sync I0514 23:14:39.138204 3663 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0514 23:14:39.138247 3663 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0514 23:14:39.138263 3663 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0514 23:14:39.138273 3663 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0514 23:14:39.138486 3663 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0514 23:14:39.138549 3663 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0514 23:14:39.138593 3663 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 65.661µs I0514 23:14:39.138609 3663 udn_enabled_svc.go:95] Performing full resync I0514 23:14:39.138614 3663 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 55.758µs I0514 23:14:39.138620 3663 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 70.932µs I0514 23:14:39.138639 3663 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0514 23:14:39.138648 3663 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0514 23:14:39.138656 3663 default_network_controller.go:325] Starting the default network controller I0514 23:14:39.138677 3663 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 10.644µs I0514 23:14:39.138945 3663 udn_enabled_svc.go:99] Waiting for handler to sync I0514 23:14:39.138959 3663 udn_enabled_svc.go:104] Starting worker I0514 23:14:39.141277 3663 iptables.go:108] Creating table: filter chain: OUTPUT I0514 23:14:39.147337 3663 gateway_shared_intf.go:1816] Gateway Creation Complete I0514 23:14:39.147350 3663 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-78.ec2.internal","mac-address":"12:2d:60:fc:83:f5","ip-addresses":["10.0.0.78/24"],"ip-address":"10.0.0.78/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:ab3d03df-ed52-4e10-a0dc-eafe819049f7 k8s.ovn.org/node-encap-ips:["10.0.0.78"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.78/24"} k8s.ovn.org/zone-name:ip-10-0-0-78.ec2.internal] on node ip-10-0-0-78.ec2.internal I0514 23:14:39.148418 3663 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0514 23:14:39.148476 3663 repair.go:33] Syncing exgw routes took 45.821µs I0514 23:14:39.148498 3663 default_network_controller.go:414] Starting all the Watchers... I0514 23:14:39.148645 3663 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0514 23:14:39.148653 3663 namespace.go:98] [openshift-route-controller-manager] adding namespace I0514 23:14:39.148655 3663 namespace.go:98] [openshift-network-diagnostics] adding namespace I0514 23:14:39.148667 3663 namespace.go:98] [openshift-kube-scheduler] adding namespace I0514 23:14:39.148672 3663 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0514 23:14:39.148676 3663 namespace.go:98] [openshift-multus] adding namespace I0514 23:14:39.148684 3663 namespace.go:98] [openshift-network-console] adding namespace I0514 23:14:39.148684 3663 namespace.go:98] [openshift-config-managed] adding namespace I0514 23:14:39.148695 3663 namespace.go:98] [openshift-backplane-tam] adding namespace I0514 23:14:39.148695 3663 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0514 23:14:39.148704 3663 namespace.go:98] [openshift-must-gather-operator] adding namespace I0514 23:14:39.148700 3663 namespace.go:98] [openshift-authentication-operator] adding namespace I0514 23:14:39.148712 3663 namespace.go:98] [dedicated-admin] adding namespace I0514 23:14:39.148706 3663 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0514 23:14:39.148728 3663 namespace.go:98] [openshift] adding namespace I0514 23:14:39.149246 3663 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 588µs I0514 23:14:39.149261 3663 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0514 23:14:39.149660 3663 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 394.469µs I0514 23:14:39.149673 3663 namespace.go:98] [openshift-node] adding namespace I0514 23:14:39.150150 3663 namespace.go:102] [openshift-node] adding namespace took 469.944µs I0514 23:14:39.150164 3663 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace W0514 23:14:39.150577 3663 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-kjlnq: pod openshift-network-diagnostics/network-check-target-kjlnq: no pod IPs found I0514 23:14:39.150610 3663 namespace.go:102] [openshift-route-controller-manager] adding namespace took 1.94842ms I0514 23:14:39.150628 3663 namespace.go:98] [openshift-service-ca-operator] adding namespace I0514 23:14:39.151216 3663 namespace.go:102] [openshift-network-diagnostics] adding namespace took 2.548106ms I0514 23:14:39.151230 3663 namespace.go:98] [openshift-package-operator] adding namespace I0514 23:14:39.151657 3663 namespace.go:102] [openshift-kube-scheduler] adding namespace took 2.977926ms I0514 23:14:39.151671 3663 namespace.go:98] [openshift-service-ca] adding namespace W0514 23:14:39.152158 3663 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-5t9hc: pod openshift-multus/network-metrics-daemon-5t9hc: no pod IPs found I0514 23:14:39.152198 3663 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 3.51353ms I0514 23:14:39.152218 3663 namespace.go:98] [openshift-backplane-ai-agent] adding namespace I0514 23:14:39.152752 3663 namespace.go:102] [openshift-multus] adding namespace took 4.066333ms I0514 23:14:39.152765 3663 namespace.go:98] [openshift-monitoring] adding namespace I0514 23:14:39.153285 3663 namespace.go:102] [openshift-network-console] adding namespace took 4.595822ms I0514 23:14:39.153303 3663 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0514 23:14:39.153720 3663 namespace.go:102] [openshift-config-managed] adding namespace took 5.02743ms I0514 23:14:39.153732 3663 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0514 23:14:39.154259 3663 namespace.go:102] [openshift-backplane-tam] adding namespace took 5.549008ms I0514 23:14:39.154281 3663 namespace.go:98] [default] adding namespace I0514 23:14:39.154768 3663 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.055588ms I0514 23:14:39.154783 3663 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0514 23:14:39.155231 3663 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.518974ms I0514 23:14:39.155240 3663 namespace.go:98] [openshift-image-registry] adding namespace I0514 23:14:39.155714 3663 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.997848ms I0514 23:14:39.155729 3663 namespace.go:98] [openshift-kube-apiserver] adding namespace I0514 23:14:39.156246 3663 namespace.go:102] [dedicated-admin] adding namespace took 7.528003ms I0514 23:14:39.156262 3663 namespace.go:98] [openshift-host-network] adding namespace I0514 23:14:39.156630 3663 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.911109ms I0514 23:14:39.156642 3663 namespace.go:98] [openshift-apiserver-operator] adding namespace I0514 23:14:39.157077 3663 namespace.go:102] [openshift] adding namespace took 8.342088ms I0514 23:14:39.157088 3663 namespace.go:98] [openshift-backplane-cee] adding namespace I0514 23:14:39.157513 3663 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 7.342688ms I0514 23:14:39.157528 3663 namespace.go:98] [openshift-operators-redhat] adding namespace I0514 23:14:39.157911 3663 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.277067ms I0514 23:14:39.157926 3663 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0514 23:14:39.158394 3663 namespace.go:102] [openshift-package-operator] adding namespace took 7.159885ms I0514 23:14:39.158405 3663 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0514 23:14:39.158774 3663 namespace.go:102] [openshift-service-ca] adding namespace took 7.099139ms I0514 23:14:39.158785 3663 namespace.go:98] [openshift-machine-config-operator] adding namespace I0514 23:14:39.159120 3663 namespace.go:102] [openshift-backplane-ai-agent] adding namespace took 6.894008ms I0514 23:14:39.159129 3663 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0514 23:14:39.159622 3663 namespace.go:102] [openshift-monitoring] adding namespace took 6.84851ms I0514 23:14:39.159636 3663 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0514 23:14:39.160058 3663 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.750378ms I0514 23:14:39.160070 3663 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0514 23:14:39.160492 3663 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.754195ms I0514 23:14:39.160506 3663 namespace.go:98] [openshift-backplane] adding namespace I0514 23:14:39.160937 3663 namespace.go:102] [default] adding namespace took 6.650535ms I0514 23:14:39.160947 3663 namespace.go:98] [openshift-apiserver] adding namespace I0514 23:14:39.161370 3663 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.583499ms I0514 23:14:39.161378 3663 namespace.go:98] [openshift-operators] adding namespace I0514 23:14:39.161832 3663 namespace.go:102] [openshift-image-registry] adding namespace took 6.586179ms I0514 23:14:39.161847 3663 namespace.go:98] [openshift-infra] adding namespace I0514 23:14:39.162216 3663 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.483578ms I0514 23:14:39.162227 3663 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0514 23:14:39.162661 3663 namespace.go:102] [openshift-host-network] adding namespace took 6.394665ms I0514 23:14:39.162672 3663 namespace.go:98] [openshift-dns] adding namespace I0514 23:14:39.163471 3663 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.824779ms I0514 23:14:39.163481 3663 namespace.go:98] [openshift-config-operator] adding namespace I0514 23:14:39.163808 3663 namespace.go:102] [openshift-backplane-cee] adding namespace took 6.71656ms I0514 23:14:39.163829 3663 namespace.go:98] [openshift-ingress-canary] adding namespace I0514 23:14:39.164335 3663 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.801812ms I0514 23:14:39.164723 3663 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.793136ms I0514 23:14:39.164733 3663 namespace.go:98] [openshift-config] adding namespace I0514 23:14:39.165179 3663 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.769058ms I0514 23:14:39.165193 3663 namespace.go:98] [openshift-ingress] adding namespace I0514 23:14:39.165557 3663 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.768153ms I0514 23:14:39.165568 3663 namespace.go:98] [openshift-console-user-settings] adding namespace I0514 23:14:39.166013 3663 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.879453ms I0514 23:14:39.166026 3663 namespace.go:98] [openshift-controller-manager] adding namespace I0514 23:14:39.166355 3663 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.712785ms I0514 23:14:39.166370 3663 namespace.go:98] [openshift-backplane-cse] adding namespace I0514 23:14:39.166841 3663 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.765611ms I0514 23:14:39.166857 3663 namespace.go:98] [openshift-network-operator] adding namespace I0514 23:14:39.167239 3663 namespace.go:102] [openshift-backplane] adding namespace took 6.727125ms I0514 23:14:39.167252 3663 namespace.go:98] [kube-node-lease] adding namespace I0514 23:14:39.167550 3663 namespace.go:102] [openshift-apiserver] adding namespace took 6.599014ms I0514 23:14:39.167558 3663 namespace.go:98] [openshift-etcd] adding namespace I0514 23:14:39.168040 3663 namespace.go:102] [openshift-operators] adding namespace took 6.656725ms I0514 23:14:39.168053 3663 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0514 23:14:39.168375 3663 namespace.go:102] [openshift-infra] adding namespace took 6.524658ms I0514 23:14:39.168386 3663 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0514 23:14:39.168849 3663 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.614028ms I0514 23:14:39.168871 3663 namespace.go:98] [kube-public] adding namespace I0514 23:14:39.169362 3663 namespace.go:102] [openshift-dns] adding namespace took 6.683938ms I0514 23:14:39.169378 3663 namespace.go:98] [openshift-backplane-csm] adding namespace I0514 23:14:39.169858 3663 namespace.go:102] [openshift-config-operator] adding namespace took 6.371033ms I0514 23:14:39.169872 3663 namespace.go:98] [kube-system] adding namespace I0514 23:14:39.170369 3663 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.534102ms I0514 23:14:39.170385 3663 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0514 23:14:39.170838 3663 namespace.go:102] [openshift-config] adding namespace took 6.098017ms I0514 23:14:39.170854 3663 namespace.go:98] [openshift-machine-api] adding namespace I0514 23:14:39.171373 3663 namespace.go:102] [openshift-ingress] adding namespace took 6.173686ms I0514 23:14:39.171389 3663 namespace.go:98] [openshift-console-operator] adding namespace I0514 23:14:39.171830 3663 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.240787ms I0514 23:14:39.171842 3663 namespace.go:98] [openshift-insights] adding namespace I0514 23:14:39.172346 3663 namespace.go:102] [openshift-controller-manager] adding namespace took 6.314096ms I0514 23:14:39.172362 3663 namespace.go:98] [openshift-network-node-identity] adding namespace I0514 23:14:39.172835 3663 default_node_network_controller.go:519] Node connection status = connected I0514 23:14:39.172968 3663 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.592527ms I0514 23:14:39.172983 3663 namespace.go:98] [openshift-customer-monitoring] adding namespace I0514 23:14:39.173450 3663 namespace.go:102] [openshift-network-operator] adding namespace took 6.589538ms I0514 23:14:39.173459 3663 namespace.go:98] [openshift-backplane-mobb] adding namespace I0514 23:14:39.173949 3663 namespace.go:102] [kube-node-lease] adding namespace took 6.691232ms I0514 23:14:39.173966 3663 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0514 23:14:39.174449 3663 namespace.go:102] [openshift-etcd] adding namespace took 6.885413ms I0514 23:14:39.174461 3663 namespace.go:98] [openshift-dns-operator] adding namespace I0514 23:14:39.175025 3663 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.966863ms I0514 23:14:39.175038 3663 namespace.go:98] [openshift-console] adding namespace I0514 23:14:39.175551 3663 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.15897ms I0514 23:14:39.175565 3663 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0514 23:14:39.176034 3663 namespace.go:102] [kube-public] adding namespace took 7.158968ms I0514 23:14:39.176044 3663 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0514 23:14:39.176459 3663 namespace.go:102] [openshift-backplane-csm] adding namespace took 7.077204ms I0514 23:14:39.176469 3663 namespace.go:98] [openshift-backplane-srep] adding namespace W0514 23:14:39.176494 3663 base_network_controller_namespace.go:410] Failed to get IPs for pod kube-system/global-pull-secret-syncer-gjtjf: pod kube-system/global-pull-secret-syncer-gjtjf: no pod IPs found I0514 23:14:39.177080 3663 namespace.go:102] [kube-system] adding namespace took 7.200998ms I0514 23:14:39.177099 3663 namespace.go:98] [openshift-marketplace] adding namespace I0514 23:14:39.177524 3663 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.133274ms I0514 23:14:39.177541 3663 namespace.go:98] [openshift-ingress-operator] adding namespace I0514 23:14:39.178171 3663 namespace.go:102] [openshift-machine-api] adding namespace took 7.308826ms I0514 23:14:39.178191 3663 namespace.go:98] [openshift-authentication] adding namespace I0514 23:14:39.178763 3663 namespace.go:102] [openshift-console-operator] adding namespace took 7.367478ms I0514 23:14:39.179298 3663 namespace.go:102] [openshift-insights] adding namespace took 7.450281ms I0514 23:14:39.179311 3663 namespace.go:98] [openshift-cluster-version] adding namespace I0514 23:14:39.179837 3663 namespace.go:102] [openshift-network-node-identity] adding namespace took 7.468933ms I0514 23:14:39.180309 3663 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.318ms I0514 23:14:39.180721 3663 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.256636ms I0514 23:14:39.181409 3663 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 7.438117ms I0514 23:14:39.181424 3663 namespace.go:98] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] adding namespace I0514 23:14:39.181984 3663 namespace.go:102] [openshift-dns-operator] adding namespace took 7.519281ms I0514 23:14:39.181993 3663 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0514 23:14:39.182388 3663 namespace.go:102] [openshift-console] adding namespace took 7.345471ms I0514 23:14:39.182396 3663 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0514 23:14:39.182983 3663 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.410646ms I0514 23:14:39.183457 3663 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 7.408213ms I0514 23:14:39.183845 3663 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.372965ms I0514 23:14:39.183856 3663 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0514 23:14:39.184375 3663 namespace.go:102] [openshift-marketplace] adding namespace took 7.269523ms I0514 23:14:39.184390 3663 namespace.go:98] [openshift-logging] adding namespace I0514 23:14:39.184839 3663 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.290709ms I0514 23:14:39.185212 3663 namespace.go:102] [openshift-authentication] adding namespace took 7.0145ms I0514 23:14:39.185716 3663 namespace.go:102] [openshift-cluster-version] adding namespace took 6.398396ms I0514 23:14:39.186167 3663 namespace.go:102] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] adding namespace took 4.738494ms I0514 23:14:39.186572 3663 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 4.573474ms I0514 23:14:39.187005 3663 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 4.60537ms I0514 23:14:39.187446 3663 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 3.58525ms I0514 23:14:39.187886 3663 namespace.go:102] [openshift-logging] adding namespace took 3.490763ms I0514 23:14:39.188084 3663 master.go:548] Adding or Updating local node "ip-10-0-0-78.ec2.internal" for network "default" I0514 23:14:39.189180 3663 zone_ic_handler.go:195] Time taken to create transit switch: 510.644µs I0514 23:14:39.189194 3663 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-98.ec2.internal for the network default E0514 23:14:39.189252 3663 obj_retry.go:575] Failed to create *v1.Node ip-10-0-2-152.ec2.internal, error: adding or updating remote node IC resources ip-10-0-2-152.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-2-152.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc009af8028 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u2286130236}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0514 23:14:39.189305 3663 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-2-152.ec2.internal", UID:"2f583bbd-05df-4db9-8f77-b4bb90e73ad6", APIVersion:"v1", ResourceVersion:"9619", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' adding or updating remote node IC resources ip-10-0-2-152.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-2-152.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc009af8028 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u2286130236}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0514 23:14:39.190479 3663 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-98.ec2.internal" on network "default" took: 1.822115ms W0514 23:14:39.196295 3663 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-5t9hc: pod openshift-multus/network-metrics-daemon-5t9hc: no pod IPs found W0514 23:14:39.196308 3663 gateway.go:207] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-gjtjf: pod kube-system/global-pull-secret-syncer-gjtjf: no pod IPs found W0514 23:14:39.196324 3663 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-kjlnq: pod openshift-network-diagnostics/network-check-target-kjlnq: no pod IPs found I0514 23:14:39.197334 3663 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-78.ec2.internal for the network default I0514 23:14:39.197510 3663 zone_ic_handler.go:195] Time taken to create transit switch: 163.721µs I0514 23:14:39.198217 3663 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0514 23:14:39.198240 3663 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0514 23:14:39.198253 3663 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0514 23:14:39.198257 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-152.ec2.internal in network "default" I0514 23:14:39.198305 3663 node_tracker.go:169] Node ip-10-0-2-152.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:39.198322 3663 services_controller.go:574] Full service sync requested for network=default I0514 23:14:39.198330 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-78.ec2.internal in network "default" I0514 23:14:39.198352 3663 node_tracker.go:169] Node ip-10-0-0-78.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:39.198358 3663 services_controller.go:574] Full service sync requested for network=default I0514 23:14:39.198364 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-98.ec2.internal in network "default" I0514 23:14:39.198386 3663 node_tracker.go:169] Node ip-10-0-1-98.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:39.198393 3663 services_controller.go:574] Full service sync requested for network=default I0514 23:14:39.298585 3663 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0514 23:14:39.298605 3663 services_controller.go:208] Setting up event handlers for services for network=default I0514 23:14:39.298688 3663 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0514 23:14:39.298747 3663 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0514 23:14:39.298757 3663 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0514 23:14:39.399486 3663 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0514 23:14:39.399509 3663 repair.go:60] Starting repairing loop for services I0514 23:14:39.399616 3663 repair.go:131] Deleted 0 stale service LBs I0514 23:14:39.399644 3663 repair.go:137] Deleted 0 stale Chassis Template Vars I0514 23:14:39.399671 3663 repair.go:62] Finished repairing loop for services: 163.902µs I0514 23:14:39.399707 3663 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0514 23:14:39.399721 3663 services_controller.go:254] Starting workers for network=default I0514 23:14:39.400095 3663 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-5t9hc] creating logical port openshift-multus_network-metrics-daemon-5t9hc for pod on switch ip-10-0-0-78.ec2.internal I0514 23:14:39.400138 3663 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-gjtjf] creating logical port kube-system_global-pull-secret-syncer-gjtjf for pod on switch ip-10-0-0-78.ec2.internal I0514 23:14:39.400231 3663 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-5t9hc I0514 23:14:39.400244 3663 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-gjtjf W0514 23:14:39.404418 3663 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0514 23:14:39.404997 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0514 23:14:39.415221 3663 pod.go:62] [kube-system/global-pull-secret-syncer-gjtjf] pod update took 15.011811ms I0514 23:14:39.415239 3663 base_network_controller_pods.go:951] [default/kube-system/global-pull-secret-syncer-gjtjf] addLogicalPort annotation time took 15.034332ms I0514 23:14:39.415994 3663 pods.go:271] [kube-system/global-pull-secret-syncer-gjtjf] addLogicalPort took 15.865435ms, libovsdb time 585.46µs I0514 23:14:39.416042 3663 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-kjlnq] creating logical port openshift-network-diagnostics_network-check-target-kjlnq for pod on switch ip-10-0-0-78.ec2.internal I0514 23:14:39.416100 3663 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-kjlnq I0514 23:14:39.418456 3663 pod.go:62] [openshift-multus/network-metrics-daemon-5t9hc] pod update took 18.280276ms I0514 23:14:39.418471 3663 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-5t9hc] addLogicalPort annotation time took 18.299631ms I0514 23:14:39.419227 3663 pods.go:271] [openshift-multus/network-metrics-daemon-5t9hc] addLogicalPort took 19.143138ms, libovsdb time 584.967µs I0514 23:14:39.433665 3663 pod.go:62] [openshift-network-diagnostics/network-check-target-kjlnq] pod update took 17.583498ms I0514 23:14:39.433682 3663 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-kjlnq] addLogicalPort annotation time took 17.602489ms I0514 23:14:39.434584 3663 pods.go:271] [openshift-network-diagnostics/network-check-target-kjlnq] addLogicalPort took 18.545863ms, libovsdb time 606.05µs I0514 23:14:39.435245 3663 admin_network_policy_controller.go:227] Starting controller default-network-controller I0514 23:14:39.435263 3663 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0514 23:14:39.435275 3663 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0514 23:14:39.435280 3663 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0514 23:14:39.435475 3663 repair.go:29] Repairing admin network policies took 189.957µs I0514 23:14:39.435587 3663 repair.go:92] Repairing baseline admin network policies took 102.783µs I0514 23:14:39.435991 3663 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0514 23:14:39.436008 3663 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0514 23:14:39.437463 3663 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0514 23:14:39.437482 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0514 23:14:39.438386 3663 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0514 23:14:39.438401 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0514 23:14:39.440197 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0514 23:14:39.440215 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0514 23:14:39.440222 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0514 23:14:39.441771 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0514 23:14:39.441789 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0514 23:14:39.441832 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0514 23:14:39.442686 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0514 23:14:39.442704 3663 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0514 23:14:39.442724 3663 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0514 23:14:39.445233 3663 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0514 23:14:39.445249 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0514 23:14:39.445257 3663 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0514 23:14:39.446889 3663 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0514 23:14:39.446906 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0514 23:14:39.446914 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0514 23:14:39.447406 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0514 23:14:39.447420 3663 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0514 23:14:39.447429 3663 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0514 23:14:39.448305 3663 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0514 23:14:39.448324 3663 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0514 23:14:39.448334 3663 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0514 23:14:39.449501 3663 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0514 23:14:39.449519 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0514 23:14:39.449526 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0514 23:14:39.450030 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0514 23:14:39.450044 3663 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0514 23:14:39.450053 3663 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0514 23:14:39.451125 3663 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0514 23:14:39.451142 3663 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0514 23:14:39.451150 3663 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0514 23:14:39.452951 3663 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0514 23:14:39.452970 3663 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0514 23:14:39.452978 3663 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0514 23:14:39.453971 3663 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0514 23:14:39.453990 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0514 23:14:39.453998 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0514 23:14:39.454460 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0514 23:14:39.454478 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0514 23:14:39.454487 3663 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0514 23:14:39.454952 3663 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0514 23:14:39.454970 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0514 23:14:39.454977 3663 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0514 23:14:39.455434 3663 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0514 23:14:39.455449 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0514 23:14:39.455455 3663 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0514 23:14:39.455749 3663 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0514 23:14:39.455767 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0514 23:14:39.455775 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0514 23:14:39.456265 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0514 23:14:39.456282 3663 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0514 23:14:39.456290 3663 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0514 23:14:39.458317 3663 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0514 23:14:39.458333 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0514 23:14:39.458342 3663 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0514 23:14:39.459079 3663 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0514 23:14:39.459096 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0514 23:14:39.459104 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0514 23:14:39.459614 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0514 23:14:39.459632 3663 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0514 23:14:39.459640 3663 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0514 23:14:39.460153 3663 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0514 23:14:39.460169 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0514 23:14:39.460176 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0514 23:14:39.460696 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0514 23:14:39.460714 3663 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0514 23:14:39.460720 3663 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0514 23:14:39.460995 3663 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0514 23:14:39.461011 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0514 23:14:39.461018 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0514 23:14:39.461493 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0514 23:14:39.461511 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0514 23:14:39.461518 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0514 23:14:39.463238 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0514 23:14:39.463257 3663 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0514 23:14:39.463264 3663 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0514 23:14:39.463768 3663 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0514 23:14:39.463784 3663 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0514 23:14:39.463792 3663 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0514 23:14:39.464756 3663 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0514 23:14:39.464780 3663 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0514 23:14:39.464829 3663 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0514 23:14:39.466060 3663 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0514 23:14:39.471745 3663 egressfirewall.go:374] Starting EgressFirewall controller I0514 23:14:39.471769 3663 controller.go:133] Adding controller egress-firewall-controller event handlers I0514 23:14:39.471770 3663 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0514 23:14:39.471795 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0514 23:14:39.471801 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0514 23:14:39.471808 3663 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0514 23:14:39.471828 3663 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0514 23:14:39.472100 3663 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0514 23:14:39.472113 3663 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0514 23:14:39.472118 3663 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0514 23:14:39.472126 3663 egressqos.go:193] Setting up event handlers for EgressQoS I0514 23:14:39.472132 3663 egressfirewall.go:723] Syncing node "ip-10-0-0-78.ec2.internal" for egress firewall I0514 23:14:39.472144 3663 egressfirewall.go:723] Syncing node "ip-10-0-1-98.ec2.internal" for egress firewall I0514 23:14:39.472148 3663 egressfirewall.go:723] Syncing node "ip-10-0-2-152.ec2.internal" for egress firewall I0514 23:14:39.472265 3663 egressqos.go:245] Starting EgressQoS Controller I0514 23:14:39.472279 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0514 23:14:39.472289 3663 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0514 23:14:39.472297 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0514 23:14:39.472303 3663 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0514 23:14:39.472308 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0514 23:14:39.472312 3663 shared_informer.go:356] "Caches are synced" controller="egressqos" I0514 23:14:39.472316 3663 egressqos.go:259] Repairing EgressQoSes I0514 23:14:39.472320 3663 egressqos.go:399] Starting repairing loop for egressqos I0514 23:14:39.472402 3663 egressqos.go:401] Finished repairing loop for egressqos: 81.935µs I0514 23:14:39.472418 3663 egressservice_zone.go:132] Setting up event handlers for Egress Services I0514 23:14:39.472458 3663 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-78.ec2.internal I0514 23:14:39.472499 3663 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-78.ec2.internal : 41.884µs I0514 23:14:39.472593 3663 egressservice_zone.go:210] Starting Egress Services Controller I0514 23:14:39.472606 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0514 23:14:39.472613 3663 shared_informer.go:356] "Caches are synced" controller="egressservices" I0514 23:14:39.472618 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0514 23:14:39.472623 3663 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0514 23:14:39.472629 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0514 23:14:39.472634 3663 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0514 23:14:39.472640 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0514 23:14:39.472644 3663 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0514 23:14:39.472648 3663 egressservice_zone.go:228] Repairing Egress Services I0514 23:14:39.472940 3663 master_controller.go:88] Starting Admin Policy Based Route Controller I0514 23:14:39.472953 3663 external_controller.go:278] Starting Admin Policy Based Route Controller I0514 23:14:39.472964 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal I0514 23:14:39.472972 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 8.983µs I0514 23:14:39.472978 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-98.ec2.internal I0514 23:14:39.472986 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-98.ec2.internal: 8.557µs I0514 23:14:39.472991 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-152.ec2.internal I0514 23:14:39.472996 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-152.ec2.internal: 5.228µs I0514 23:14:39.473080 3663 default_network_controller.go:585] Completing all the Watchers took 324.571759ms I0514 23:14:39.473096 3663 default_network_controller.go:589] Starting unidling controllers I0514 23:14:39.473112 3663 unidle.go:46] Registering OVN SB ControllerEvent handler I0514 23:14:39.473126 3663 unidle.go:63] Populating Initial ContollerEvent events I0514 23:14:39.473145 3663 unidle.go:79] Setting up event handlers for services I0514 23:14:39.473211 3663 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0514 23:14:39.674219 3663 default_node_network_controller.go:519] Node connection status = connected I0514 23:14:40.173684 3663 default_node_network_controller.go:519] Node connection status = connected W0514 23:14:40.464780 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0514 23:14:40.679443 3663 default_node_network_controller.go:519] Node connection status = connected I0514 23:14:40.748079 3663 bridgeconfig.go:537] Gateway is ready I0514 23:14:40.748099 3663 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0514 23:14:40.761313 3663 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.78. Updating OVN encap IP. I0514 23:14:40.763491 3663 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0514 23:14:40.767524 3663 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.78 - it is already configured I0514 23:14:40.767544 3663 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.78/24:{} 172.20.0.1/32:{}]. Updating annotations. I0514 23:14:40.767873 3663 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.78/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-78.ec2.internal","mac-address":"12:2d:60:fc:83:f5","ip-addresses":["10.0.0.78/24"],"ip-address":"10.0.0.78/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:ab3d03df-ed52-4e10-a0dc-eafe819049f7 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.78/24"}] on node ip-10-0-0-78.ec2.internal I0514 23:14:40.783987 3663 egressfirewall.go:723] Syncing node "ip-10-0-0-78.ec2.internal" for egress firewall I0514 23:14:40.784035 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal I0514 23:14:40.784036 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-78.ec2.internal in network "default" I0514 23:14:40.784047 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 15.157µs I0514 23:14:40.784096 3663 node_tracker.go:169] Node ip-10-0-0-78.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:40.784114 3663 services_controller.go:574] Full service sync requested for network=default I0514 23:14:40.784131 3663 master.go:548] Adding or Updating local node "ip-10-0-0-78.ec2.internal" for network "default" I0514 23:14:40.784328 3663 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.0.78 I0514 23:14:40.784489 3663 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-78.ec2.internal W0514 23:14:40.784660 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0514 23:14:40.784742 3663 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0514 23:14:40.788951 3663 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-78.ec2.internal. Updating no re-route policies I0514 23:14:40.789035 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-lq8sq I0514 23:14:40.789045 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-gfj5k I0514 23:14:40.789051 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-dff4x I0514 23:14:40.789058 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-gfj5k I0514 23:14:40.789058 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-kjlnq I0514 23:14:40.789069 3663 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-fck2p I0514 23:14:40.789074 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-kjlnq I0514 23:14:40.789089 3663 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-fck2p I0514 23:14:40.789097 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-xb7qv I0514 23:14:40.789100 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-l2p66 I0514 23:14:40.789066 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-78.ec2.internal I0514 23:14:40.789110 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-xb7qv I0514 23:14:40.789036 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-qtm65 I0514 23:14:40.789116 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-78.ec2.internal I0514 23:14:40.789120 3663 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-kjlnq] creating logical port openshift-network-diagnostics_network-check-target-kjlnq for pod on switch ip-10-0-0-78.ec2.internal I0514 23:14:40.789129 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-qtm65 I0514 23:14:40.789079 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-gfj5k after 0 failed attempt(s) I0514 23:14:40.789147 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-78.ec2.internal after 0 failed attempt(s) I0514 23:14:40.789102 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-dff4x I0514 23:14:40.789157 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-qtm65 after 0 failed attempt(s) I0514 23:14:40.789181 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-dff4x after 0 failed attempt(s) I0514 23:14:40.789090 3663 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-78.ec2.internal I0514 23:14:40.789218 3663 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-78.ec2.internal I0514 23:14:40.789221 3663 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-fck2p after 0 failed attempt(s) I0514 23:14:40.789233 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-xb7qv after 0 failed attempt(s) I0514 23:14:40.789237 3663 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-78.ec2.internal after 0 failed attempt(s) I0514 23:14:40.789116 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-l2p66 I0514 23:14:40.789053 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-lq8sq I0514 23:14:40.789259 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-l2p66 after 0 failed attempt(s) I0514 23:14:40.789110 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-vgrkf I0514 23:14:40.789274 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-lq8sq after 0 failed attempt(s) I0514 23:14:40.789114 3663 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-gjtjf I0514 23:14:40.789277 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-vgrkf I0514 23:14:40.789115 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-9brhd I0514 23:14:40.789319 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-9brhd I0514 23:14:40.789092 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-5t9hc I0514 23:14:40.789336 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-5t9hc I0514 23:14:40.789340 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-9brhd after 0 failed attempt(s) I0514 23:14:40.789300 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-vgrkf after 0 failed attempt(s) I0514 23:14:40.789285 3663 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-gjtjf I0514 23:14:40.789429 3663 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-5t9hc] creating logical port openshift-multus_network-metrics-daemon-5t9hc for pod on switch ip-10-0-0-78.ec2.internal I0514 23:14:40.789441 3663 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-gjtjf] creating logical port kube-system_global-pull-secret-syncer-gjtjf for pod on switch ip-10-0-0-78.ec2.internal I0514 23:14:40.790361 3663 pods.go:271] [openshift-network-diagnostics/network-check-target-kjlnq] addLogicalPort took 1.237219ms, libovsdb time 678.478µs I0514 23:14:40.790373 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-kjlnq after 0 failed attempt(s) I0514 23:14:40.790517 3663 pods.go:271] [openshift-multus/network-metrics-daemon-5t9hc] addLogicalPort took 1.096044ms, libovsdb time 458.011µs I0514 23:14:40.790529 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-5t9hc after 0 failed attempt(s) I0514 23:14:40.790588 3663 pods.go:271] [kube-system/global-pull-secret-syncer-gjtjf] addLogicalPort took 1.156019ms, libovsdb time 481.333µs I0514 23:14:40.790595 3663 obj_retry.go:418] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-gjtjf after 0 failed attempt(s) I0514 23:14:40.820578 3663 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0514 23:14:40.833300 3663 iptables.go:108] Creating table: mangle chain: OUTPUT I0514 23:14:40.836508 3663 iptables.go:108] Creating table: nat chain: OUTPUT I0514 23:14:40.859384 3663 iptables.go:108] Creating table: nat chain: PREROUTING I0514 23:14:40.903399 3663 iptables.go:108] Creating table: filter chain: FORWARD I0514 23:14:40.920011 3663 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0514 23:14:40.920145 3663 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0514 23:14:40.920203 3663 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0514 23:14:40.920294 3663 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0514 23:14:40.920357 3663 gateway_init.go:325] Gateway and management port readiness took 1.751414085s I0514 23:14:40.920391 3663 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0514 23:14:40.920436 3663 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0514 23:14:40.920451 3663 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0514 23:14:40.920466 3663 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0514 23:14:40.920539 3663 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0514 23:14:40.920624 3663 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 54.791µs I0514 23:14:40.920913 3663 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0514 23:14:40.920952 3663 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 27.384µs I0514 23:14:40.920965 3663 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 51.263µs I0514 23:14:40.920973 3663 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0514 23:14:40.920988 3663 nad_controller.go:379] [node-nad-controller NAD controller]: started I0514 23:14:40.920997 3663 default_node_network_controller.go:899] Starting the default node network controller I0514 23:14:40.921004 3663 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0514 23:14:40.921067 3663 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 11.92µs I0514 23:14:40.921196 3663 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0514 23:14:40.921207 3663 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0514 23:14:40.921236 3663 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b0b150 0xc000325ce0 0xc000325d50 0xc000325dc0 0xc000325e30 0xc000325ea0 0xc000325f10 0xc000b275e0 0xc00033e000 0xc00033e070 0xc000cdc370 0xc0064f7500 0xc00033e0e0 map[0x2ff9080:0xc0024966c0 0x2ff95c0:0xc0024969c0 0x2ffa2e0:0xc002496600 0x2ffa580:0xc002496660 0x2ffad60:0xc002496720 0x2ffb000:0xc002496780 0x2ffb2a0:0xc0024967e0 0x2ffba80:0xc002496a20 0x2ffbd20:0xc002496a80 0x3026980:0xc00092a720 0x3026d20:0xc002984540 0x3027800:0xc000c81980 0x3029160:0xc000e958c0 0x30298a0:0xc0029845a0 0x302a720:0xc000e95920] 0xc000b285b0 0} I0514 23:14:40.921300 3663 gateway.go:151] Starting gateway service sync I0514 23:14:40.921747 3663 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0514 23:14:40.923931 3663 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0514 23:14:40.933326 3663 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0514 23:14:40.934455 3663 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0514 23:14:40.935555 3663 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0514 23:14:40.970692 3663 gateway.go:168] Gateway service sync done. Time taken: 49.377468ms I0514 23:14:40.971142 3663 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32423 I0514 23:14:40.971532 3663 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b0b150 0xc000325ce0 0xc000325d50 0xc000325dc0 0xc000325e30 0xc000325ea0 0xc000325f10 0xc000b275e0 0xc00033e000 0xc00033e070 0xc000cdc370 0xc0064f7500 0xc00033e0e0 map[0x2ff9080:0xc0024966c0 0x2ff95c0:0xc0024969c0 0x2ffa2e0:0xc002496600 0x2ffa580:0xc002496660 0x2ffad60:0xc002496720 0x2ffb000:0xc002496780 0x2ffb2a0:0xc0024967e0 0x2ffba80:0xc002496a20 0x2ffbd20:0xc002496a80 0x3026980:0xc00092a720 0x3026d20:0xc002984540 0x3027800:0xc000c81980 0x3029160:0xc000e958c0 0x30298a0:0xc0029845a0 0x302a720:0xc000e95920] 0xc000b285b0 0} I0514 23:14:40.972088 3663 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b0b150 0xc000325ce0 0xc000325d50 0xc000325dc0 0xc000325e30 0xc000325ea0 0xc000325f10 0xc000b275e0 0xc00033e000 0xc00033e070 0xc000cdc370 0xc0064f7500 0xc00033e0e0 map[0x2ff9080:0xc0024966c0 0x2ff95c0:0xc0024969c0 0x2ffa2e0:0xc002496600 0x2ffa580:0xc002496660 0x2ffad60:0xc002496720 0x2ffb000:0xc002496780 0x2ffb2a0:0xc0024967e0 0x2ffba80:0xc002496a20 0x2ffbd20:0xc002496a80 0x3026980:0xc00092a720 0x3026d20:0xc002984540 0x3027800:0xc000c81980 0x3029160:0xc000e958c0 0x30298a0:0xc0029845a0 0x302a720:0xc000e95920] 0xc000b285b0 0} I0514 23:14:40.972124 3663 gateway.go:532] Reconciling gateway with updates I0514 23:14:40.977746 3663 gateway.go:339] Spawning Conntrack Rule Check Thread I0514 23:14:40.977843 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0514 23:14:40.977859 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0514 23:14:40.977860 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0514 23:14:40.977870 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0514 23:14:40.977876 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0514 23:14:40.977880 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0514 23:14:40.977886 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0514 23:14:40.977889 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0514 23:14:40.977902 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0514 23:14:40.977916 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0514 23:14:40.977925 3663 default_node_network_controller.go:944] Gateway and management port readiness took 56.71015ms I0514 23:14:40.977936 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0514 23:14:40.977946 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0514 23:14:40.977843 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0514 23:14:40.977956 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0514 23:14:40.977965 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0514 23:14:40.977967 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0514 23:14:40.977970 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0514 23:14:40.977975 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0514 23:14:40.977973 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0514 23:14:40.977981 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0514 23:14:40.977983 3663 port_claim.go:71] Svc openshift-ingress/router-default: port 31443 is already open, no action needed I0514 23:14:40.977983 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0514 23:14:40.977986 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0514 23:14:40.977992 3663 port_claim.go:71] Svc openshift-ingress/router-default: port 30875 is already open, no action needed I0514 23:14:40.977993 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0514 23:14:40.977998 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0514 23:14:40.978003 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0514 23:14:40.978002 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0514 23:14:40.978012 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0514 23:14:40.978014 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0514 23:14:40.978037 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0514 23:14:40.977988 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0514 23:14:40.978065 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0514 23:14:40.978066 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0514 23:14:40.978078 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0514 23:14:40.978084 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0514 23:14:40.978090 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0514 23:14:40.978098 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0514 23:14:40.978102 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0514 23:14:40.978101 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0514 23:14:40.977975 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0514 23:14:40.978113 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:14:40.978100 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0514 23:14:40.978065 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0514 23:14:40.978124 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0514 23:14:40.978129 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0514 23:14:40.978135 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0514 23:14:40.978136 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0514 23:14:40.978140 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0514 23:14:40.978152 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0514 23:14:40.978159 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0514 23:14:40.978160 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0514 23:14:40.977990 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0514 23:14:40.978169 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0514 23:14:40.978178 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0514 23:14:40.978184 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0514 23:14:40.978238 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0514 23:14:40.978090 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0514 23:14:40.978261 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0514 23:14:40.978286 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0514 23:14:40.978328 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0514 23:14:40.978342 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0514 23:14:40.978351 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0514 23:14:40.978147 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0514 23:14:40.978078 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0514 23:14:40.978292 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0514 23:14:40.978434 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0514 23:14:40.978448 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0514 23:14:40.978124 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0514 23:14:40.978470 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0514 23:14:40.978195 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0514 23:14:40.978066 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0514 23:14:40.978538 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0514 23:14:40.978079 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0514 23:14:40.978562 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0514 23:14:40.978092 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0514 23:14:40.978581 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0514 23:14:40.978587 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0514 23:14:40.978106 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0514 23:14:40.978106 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0514 23:14:40.978642 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0514 23:14:40.978115 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0514 23:14:40.978120 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:14:40.978118 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0514 23:14:40.978836 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0514 23:14:40.978113 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0514 23:14:40.978848 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0514 23:14:40.978857 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0514 23:14:40.978160 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0514 23:14:40.978872 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0514 23:14:40.978200 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0514 23:14:40.978978 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0514 23:14:40.978295 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0514 23:14:40.979056 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0514 23:14:40.978297 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0514 23:14:40.979129 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0514 23:14:40.978303 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:40.979198 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:40.978324 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0514 23:14:40.979253 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0514 23:14:40.978522 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0514 23:14:40.978075 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0514 23:14:40.979318 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0514 23:14:40.979327 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0514 23:14:40.979379 3663 node_ip_handler_linux.go:160] Node IP manager is running W0514 23:14:41.017272 3663 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0514 23:14:41.017453 3663 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0514 23:14:41.017579 3663 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:82:00:01/10.130.0.1) on link ovn-k8s-mp0 I0514 23:14:41.025947 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0514 23:14:41.026018 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0514 23:14:41.026040 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0514 23:14:41.026051 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0514 23:14:41.026063 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0514 23:14:41.026073 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0514 23:14:41.026106 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0514 23:14:41.026123 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0514 23:14:41.026129 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0514 23:14:41.026136 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0514 23:14:41.026141 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0514 23:14:41.026147 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0514 23:14:41.026152 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0514 23:14:41.026157 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0514 23:14:41.026162 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0514 23:14:41.026168 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0514 23:14:41.026173 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0514 23:14:41.026178 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0514 23:14:41.026183 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0514 23:14:41.026189 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0514 23:14:41.026194 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0514 23:14:41.026199 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0514 23:14:41.026204 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0514 23:14:41.026209 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0514 23:14:41.026214 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0514 23:14:41.026219 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0514 23:14:41.026224 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0514 23:14:41.026229 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0514 23:14:41.026234 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0514 23:14:41.026241 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) W0514 23:14:41.087176 3663 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-854969bbd6-gtc27 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-854969bbd6-gtc27: no pod IPs found I0514 23:14:42.294457 3663 obj_retry.go:498] Detected object openshift-image-registry/image-registry-55bc57d65c-h99kj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:14:42.295414 3663 obj_retry.go:498] Detected object openshift-image-registry/image-registry-55bc57d65c-h99kj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:14:42.313228 3663 obj_retry.go:492] Detected object openshift-image-registry/image-registry-55bc57d65c-h99kj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:14:42.313272 3663 obj_retry.go:492] Detected object openshift-image-registry/image-registry-55bc57d65c-h99kj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:14:42.324674 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-55bc57d65c-h99kj I0514 23:14:42.324694 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-55bc57d65c-h99kj I0514 23:14:42.531497 3663 default_node_network_controller.go:1380] Starting node controller node sync W0514 23:14:42.531523 3663 egressip_healthcheck.go:76] Health checking using insecure connection I0514 23:14:42.531563 3663 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.130.0.2:9107 I0514 23:14:42.542967 3663 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.45308ms I0514 23:14:42.543088 3663 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-98.ec2.internal", IP: 10.0.1.98 to PMTUD blocking rules I0514 23:14:42.543107 3663 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-98.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0514 23:14:42.543088 3663 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-152.ec2.internal", IP: 10.0.2.152 to PMTUD blocking rules I0514 23:14:42.543119 3663 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-152.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0514 23:14:42.554537 3663 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0514 23:14:42.554913 3663 egressservice_node.go:93] Setting up event handlers for Egress Services I0514 23:14:42.555072 3663 egressservice_node.go:183] Starting Egress Services Controller I0514 23:14:42.555089 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0514 23:14:42.555099 3663 shared_informer.go:356] "Caches are synced" controller="egressservices" I0514 23:14:42.555104 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0514 23:14:42.555110 3663 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0514 23:14:42.555116 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0514 23:14:42.555121 3663 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0514 23:14:42.555125 3663 egressservice_node.go:197] Repairing Egress Services I0514 23:14:42.564550 3663 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0514 23:14:42.564561 3663 external_controller.go:278] Starting Admin Policy Based Route Controller I0514 23:14:42.564705 3663 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0514 23:14:42.564714 3663 link_network_manager.go:119] Link manager is running I0514 23:14:42.564724 3663 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0514 23:14:42.564958 3663 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0514 23:14:42.565069 3663 vrf_manager.go:128] VRF manager is running I0514 23:14:42.565156 3663 node_controller_manager.go:437] Removing flows to drop GARP I0514 23:14:42.565166 3663 gateway.go:532] Reconciling gateway with updates I0514 23:14:42.589671 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0514 23:14:42.589680 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0514 23:14:42.589688 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0514 23:14:42.589695 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0514 23:14:42.589700 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0514 23:14:42.589704 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0514 23:14:42.589705 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0514 23:14:42.589705 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0514 23:14:42.589712 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0514 23:14:42.589719 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0514 23:14:42.589718 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0514 23:14:42.589724 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0514 23:14:42.589677 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0514 23:14:42.589732 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0514 23:14:42.589736 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0514 23:14:42.589739 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0514 23:14:42.589742 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0514 23:14:42.589747 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0514 23:14:42.589749 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0514 23:14:42.589754 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0514 23:14:42.589758 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0514 23:14:42.589769 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0514 23:14:42.589780 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0514 23:14:42.589781 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0514 23:14:42.589737 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0514 23:14:42.589798 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0514 23:14:42.589806 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:42.589826 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0514 23:14:42.589832 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0514 23:14:42.589835 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0514 23:14:42.589840 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0514 23:14:42.589721 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0514 23:14:42.589843 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0514 23:14:42.589848 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0514 23:14:42.589853 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0514 23:14:42.589849 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0514 23:14:42.589856 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0514 23:14:42.589859 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0514 23:14:42.589862 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0514 23:14:42.589866 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0514 23:14:42.589865 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0514 23:14:42.589869 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0514 23:14:42.589875 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0514 23:14:42.589879 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0514 23:14:42.589879 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0514 23:14:42.589882 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0514 23:14:42.589891 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0514 23:14:42.589895 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0514 23:14:42.589900 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0514 23:14:42.589704 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0514 23:14:42.589693 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0514 23:14:42.589916 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0514 23:14:42.589929 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0514 23:14:42.589880 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0514 23:14:42.589942 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0514 23:14:42.589941 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0514 23:14:42.589757 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0514 23:14:42.589835 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:42.589963 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0514 23:14:42.589842 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0514 23:14:42.589745 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0514 23:14:42.590009 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0514 23:14:42.589733 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0514 23:14:42.589830 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0514 23:14:42.590049 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0514 23:14:42.589691 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0514 23:14:42.590064 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0514 23:14:42.590062 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0514 23:14:42.590078 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0514 23:14:42.589803 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0514 23:14:42.589722 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0514 23:14:42.589722 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0514 23:14:42.590138 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0514 23:14:42.589732 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0514 23:14:42.590147 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0514 23:14:42.590151 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0514 23:14:42.589742 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0514 23:14:42.590158 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0514 23:14:42.589801 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0514 23:14:42.589836 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0514 23:14:42.590168 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0514 23:14:42.589845 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0514 23:14:42.589849 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0514 23:14:42.590215 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0514 23:14:42.590227 3663 port_claim.go:71] Svc openshift-ingress/router-default: port 31443 is already open, no action needed I0514 23:14:42.590232 3663 port_claim.go:71] Svc openshift-ingress/router-default: port 30875 is already open, no action needed I0514 23:14:42.589856 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0514 23:14:42.589867 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:14:42.590287 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:14:42.589885 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0514 23:14:42.590312 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0514 23:14:42.589963 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0514 23:14:42.590319 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0514 23:14:42.589957 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0514 23:14:42.590326 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0514 23:14:42.589991 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0514 23:14:42.590341 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0514 23:14:42.589706 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0514 23:14:42.590162 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0514 23:14:42.589802 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0514 23:14:42.617074 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0514 23:14:42.617087 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0514 23:14:42.617105 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0514 23:14:42.617113 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0514 23:14:42.617121 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0514 23:14:42.617130 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0514 23:14:42.617138 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0514 23:14:42.617150 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0514 23:14:42.617165 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0514 23:14:42.617171 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0514 23:14:42.617179 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0514 23:14:42.617185 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0514 23:14:42.617192 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0514 23:14:42.617198 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0514 23:14:42.617203 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0514 23:14:42.617208 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0514 23:14:42.617214 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0514 23:14:42.617219 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0514 23:14:42.617224 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0514 23:14:42.617229 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0514 23:14:42.617234 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0514 23:14:42.658613 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0514 23:14:42.658639 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0514 23:14:42.658648 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0514 23:14:42.658654 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0514 23:14:42.658661 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0514 23:14:42.701001 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0514 23:14:42.701010 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0514 23:14:42.701026 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0514 23:14:42.701034 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0514 23:14:42.701040 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0514 23:14:42.701047 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0514 23:14:42.701051 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0514 23:14:42.701057 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) W0514 23:14:57.962436 3663 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0514 23:14:58.009595 3663 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0514 23:14:58.129310 3663 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0514 23:14:58.200847 3663 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-d477cf66b-rhxtq IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-d477cf66b-rhxtq: no pod IPs found I0514 23:14:58.537445 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal I0514 23:14:58.537468 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 28.398µs I0514 23:14:58.731183 3663 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-6gc2p] creating logical port openshift-insights_insights-runtime-extractor-6gc2p for pod on switch ip-10-0-0-78.ec2.internal I0514 23:14:58.731267 3663 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-6gc2p I0514 23:14:58.757330 3663 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-fp5kl] creating logical port openshift-dns_dns-default-fp5kl for pod on switch ip-10-0-0-78.ec2.internal I0514 23:14:58.757450 3663 kube.go:256] Updating pod openshift-dns/dns-default-fp5kl I0514 23:14:58.786562 3663 pod.go:62] [openshift-insights/insights-runtime-extractor-6gc2p] pod update took 55.322248ms I0514 23:14:58.786581 3663 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-6gc2p] addLogicalPort annotation time took 55.344384ms I0514 23:14:58.787721 3663 pods.go:271] [openshift-insights/insights-runtime-extractor-6gc2p] addLogicalPort took 56.551497ms, libovsdb time 898.398µs I0514 23:14:58.810295 3663 pod.go:62] [openshift-dns/dns-default-fp5kl] pod update took 52.898208ms I0514 23:14:58.810312 3663 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-fp5kl] addLogicalPort annotation time took 52.920026ms I0514 23:14:58.811404 3663 pods.go:271] [openshift-dns/dns-default-fp5kl] addLogicalPort took 54.090947ms, libovsdb time 854.357µs I0514 23:14:58.884661 3663 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-8pp79] creating logical port openshift-ingress-canary_ingress-canary-8pp79 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:14:58.884744 3663 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-8pp79 I0514 23:14:58.920001 3663 pod.go:62] [openshift-ingress-canary/ingress-canary-8pp79] pod update took 35.285513ms I0514 23:14:58.920023 3663 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-8pp79] addLogicalPort annotation time took 35.312233ms I0514 23:14:58.921029 3663 pods.go:271] [openshift-ingress-canary/ingress-canary-8pp79] addLogicalPort took 36.378445ms, libovsdb time 772.646µs I0514 23:14:58.958098 3663 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7fdc74bbd-ltlc5] creating logical port openshift-image-registry_image-registry-7fdc74bbd-ltlc5 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:14:58.958182 3663 kube.go:256] Updating pod openshift-image-registry/image-registry-7fdc74bbd-ltlc5 I0514 23:14:58.986976 3663 pod.go:62] [openshift-image-registry/image-registry-7fdc74bbd-ltlc5] pod update took 28.822878ms I0514 23:14:58.986996 3663 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-7fdc74bbd-ltlc5] addLogicalPort annotation time took 28.84603ms I0514 23:14:58.988013 3663 pods.go:271] [openshift-image-registry/image-registry-7fdc74bbd-ltlc5] addLogicalPort took 29.926711ms, libovsdb time 716.814µs I0514 23:14:59.120197 3663 cni.go:452] [openshift-insights/insights-runtime-extractor-6gc2p fba0f5766f593662fc3de86f38f7e60d357f4b6305f0986251a97980859b4f34 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-6gc2p fba0f5766f593662fc3de86f38f7e60d357f4b6305f0986251a97980859b4f34 network default NAD default NAD key ] I0514 23:14:59.159573 3663 cni.go:452] [openshift-dns/dns-default-fp5kl 7b4ae63ed550b5b987f0f3ec57b79f54f36e8c134e8a880a47096f8cc7af12c0 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-fp5kl 7b4ae63ed550b5b987f0f3ec57b79f54f36e8c134e8a880a47096f8cc7af12c0 network default NAD default NAD key ] I0514 23:14:59.166650 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-6gc2p, hostIfaceName: fba0f5766f59366, network: default, NAD default, SandboxID: "fba0f5766f593662fc3de86f38f7e60d357f4b6305f0986251a97980859b4f34", PCI device ID: , UID: "54f2e241-38cc-43e4-9ba9-bcfd2acc7b2d", MAC: 0a:58:0a:82:00:06, IPs: [10.130.0.6/23] I0514 23:14:59.171083 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-fp5kl, hostIfaceName: 7b4ae63ed550b5b, network: default, NAD default, SandboxID: "7b4ae63ed550b5b987f0f3ec57b79f54f36e8c134e8a880a47096f8cc7af12c0", PCI device ID: , UID: "f429d005-2dc0-4161-887f-e841e6cbb8af", MAC: 0a:58:0a:82:00:07, IPs: [10.130.0.7/23] I0514 23:14:59.234932 3663 cni.go:473] [openshift-insights/insights-runtime-extractor-6gc2p fba0f5766f593662fc3de86f38f7e60d357f4b6305f0986251a97980859b4f34 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-6gc2p fba0f5766f593662fc3de86f38f7e60d357f4b6305f0986251a97980859b4f34 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:60:46:a6:e0:17\",\"name\":\"fba0f5766f59366\"},{\"mac\":\"0a:58:0a:82:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7451d07c-d319-4e71-aa42-40787601fa35\"}],\"ips\":[{\"address\":\"10.130.0.6/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:14:59.234975 3663 cni.go:473] [openshift-dns/dns-default-fp5kl 7b4ae63ed550b5b987f0f3ec57b79f54f36e8c134e8a880a47096f8cc7af12c0 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-fp5kl 7b4ae63ed550b5b987f0f3ec57b79f54f36e8c134e8a880a47096f8cc7af12c0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:f5:11:53:00:00\",\"name\":\"7b4ae63ed550b5b\"},{\"mac\":\"0a:58:0a:82:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f135033b-82cf-40eb-a9e2-4c87116d7f73\"}],\"ips\":[{\"address\":\"10.130.0.7/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:14:59.275761 3663 cni.go:452] [openshift-ingress-canary/ingress-canary-8pp79 8f9581067d36996d4775b32e92719914a776d66469276ef2f854a6dc40a95870 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-8pp79 8f9581067d36996d4775b32e92719914a776d66469276ef2f854a6dc40a95870 network default NAD default NAD key ] I0514 23:14:59.287550 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-8pp79, hostIfaceName: 8f9581067d36996, network: default, NAD default, SandboxID: "8f9581067d36996d4775b32e92719914a776d66469276ef2f854a6dc40a95870", PCI device ID: , UID: "f04948f2-eddf-4de4-a00e-60bcc0d5f468", MAC: 0a:58:0a:82:00:08, IPs: [10.130.0.8/23] I0514 23:14:59.333528 3663 cni.go:473] [openshift-ingress-canary/ingress-canary-8pp79 8f9581067d36996d4775b32e92719914a776d66469276ef2f854a6dc40a95870 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-8pp79 8f9581067d36996d4775b32e92719914a776d66469276ef2f854a6dc40a95870 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:0e:ca:f4:74:0a\",\"name\":\"8f9581067d36996\"},{\"mac\":\"0a:58:0a:82:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2830a639-f2d3-400f-a92e-db46bfd3b82f\"}],\"ips\":[{\"address\":\"10.130.0.8/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:14:59.348684 3663 cni.go:452] [openshift-image-registry/image-registry-7fdc74bbd-ltlc5 f4ad0746126680927ae3c5b91462d29a2507d87cb841ab514af5ee04b98b7f94 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-7fdc74bbd-ltlc5 f4ad0746126680927ae3c5b91462d29a2507d87cb841ab514af5ee04b98b7f94 network default NAD default NAD key ] I0514 23:14:59.358585 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7fdc74bbd-ltlc5, hostIfaceName: f4ad07461266809, network: default, NAD default, SandboxID: "f4ad0746126680927ae3c5b91462d29a2507d87cb841ab514af5ee04b98b7f94", PCI device ID: , UID: "79e2a344-ad93-49eb-8d75-0d091b2272e7", MAC: 0a:58:0a:82:00:09, IPs: [10.130.0.9/23] I0514 23:14:59.405418 3663 cni.go:473] [openshift-image-registry/image-registry-7fdc74bbd-ltlc5 f4ad0746126680927ae3c5b91462d29a2507d87cb841ab514af5ee04b98b7f94 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-7fdc74bbd-ltlc5 f4ad0746126680927ae3c5b91462d29a2507d87cb841ab514af5ee04b98b7f94 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:74:3b:fd:f4:55\",\"name\":\"f4ad07461266809\"},{\"mac\":\"0a:58:0a:82:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/035c6bf9-496c-4322-8e8a-87ac4a985ce0\"}],\"ips\":[{\"address\":\"10.130.0.9/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:15:00.277215 3663 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0514 23:15:00.515414 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0514 23:15:02.041369 3663 base_network_controller_pods.go:487] [default/openshift-console/console-b57f75797-f2pc2] creating logical port openshift-console_console-b57f75797-f2pc2 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:15:02.041478 3663 kube.go:256] Updating pod openshift-console/console-b57f75797-f2pc2 I0514 23:15:02.057981 3663 pod.go:62] [openshift-console/console-b57f75797-f2pc2] pod update took 16.535368ms I0514 23:15:02.058006 3663 base_network_controller_pods.go:951] [default/openshift-console/console-b57f75797-f2pc2] addLogicalPort annotation time took 16.565834ms I0514 23:15:02.059155 3663 pods.go:271] [openshift-console/console-b57f75797-f2pc2] addLogicalPort took 17.801371ms, libovsdb time 849.18µs I0514 23:15:02.421626 3663 cni.go:452] [openshift-console/console-b57f75797-f2pc2 e5f853f512fccfd5793ab68ee1ded04226f41ac11c097147dd25b8df236b9f3c network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-b57f75797-f2pc2 e5f853f512fccfd5793ab68ee1ded04226f41ac11c097147dd25b8df236b9f3c network default NAD default NAD key ] I0514 23:15:02.433416 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-b57f75797-f2pc2, hostIfaceName: e5f853f512fccfd, network: default, NAD default, SandboxID: "e5f853f512fccfd5793ab68ee1ded04226f41ac11c097147dd25b8df236b9f3c", PCI device ID: , UID: "ecd48d7b-a875-4aa0-87e4-714acb2c01bd", MAC: 0a:58:0a:82:00:0a, IPs: [10.130.0.10/23] I0514 23:15:02.483072 3663 cni.go:473] [openshift-console/console-b57f75797-f2pc2 e5f853f512fccfd5793ab68ee1ded04226f41ac11c097147dd25b8df236b9f3c network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-b57f75797-f2pc2 e5f853f512fccfd5793ab68ee1ded04226f41ac11c097147dd25b8df236b9f3c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:ee:7a:cc:81:6f\",\"name\":\"e5f853f512fccfd\"},{\"mac\":\"0a:58:0a:82:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c3ab50e0-723f-4ecf-8f66-507b3c32c99d\"}],\"ips\":[{\"address\":\"10.130.0.10/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:15:03.367906 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0514 23:15:03.367932 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0514 23:15:03.367941 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0514 23:15:03.367974 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0514 23:15:03.367984 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0514 23:15:03.367994 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0514 23:15:04.316409 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0514 23:15:04.316430 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0514 23:15:06.295075 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0514 23:15:06.544211 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0514 23:15:07.959846 3663 default_network_controller.go:1025] Node "ip-10-0-2-152.ec2.internal" in remote zone "ip-10-0-2-152.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:15:07.960609 3663 zone_ic_handler.go:195] Time taken to create transit switch: 266.062µs I0514 23:15:07.960623 3663 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-152.ec2.internal for the network default I0514 23:15:07.961552 3663 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-152.ec2.internal" on network "default" took: 1.227442ms I0514 23:15:09.011761 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/kube-state-metrics-6dfffbb599-krxlk] creating logical port openshift-monitoring_kube-state-metrics-6dfffbb599-krxlk for pod on switch ip-10-0-0-78.ec2.internal I0514 23:15:09.011904 3663 kube.go:256] Updating pod openshift-monitoring/kube-state-metrics-6dfffbb599-krxlk I0514 23:15:09.030363 3663 pod.go:62] [openshift-monitoring/kube-state-metrics-6dfffbb599-krxlk] pod update took 18.504306ms I0514 23:15:09.030386 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/kube-state-metrics-6dfffbb599-krxlk] addLogicalPort annotation time took 18.539405ms I0514 23:15:09.031561 3663 pods.go:271] [openshift-monitoring/kube-state-metrics-6dfffbb599-krxlk] addLogicalPort took 19.813801ms, libovsdb time 851.782µs I0514 23:15:09.419364 3663 cni.go:452] [openshift-monitoring/kube-state-metrics-6dfffbb599-krxlk a14fb626cc9ff386403096cf912990bfb73e83c3c985b7e19207aed7bc72ae4e network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/kube-state-metrics-6dfffbb599-krxlk a14fb626cc9ff386403096cf912990bfb73e83c3c985b7e19207aed7bc72ae4e network default NAD default NAD key ] I0514 23:15:09.429375 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: kube-state-metrics-6dfffbb599-krxlk, hostIfaceName: a14fb626cc9ff38, network: default, NAD default, SandboxID: "a14fb626cc9ff386403096cf912990bfb73e83c3c985b7e19207aed7bc72ae4e", PCI device ID: , UID: "9025cc4c-464a-4c78-8bdd-d1f57d33e3a0", MAC: 0a:58:0a:82:00:0b, IPs: [10.130.0.11/23] I0514 23:15:09.476757 3663 cni.go:473] [openshift-monitoring/kube-state-metrics-6dfffbb599-krxlk a14fb626cc9ff386403096cf912990bfb73e83c3c985b7e19207aed7bc72ae4e network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/kube-state-metrics-6dfffbb599-krxlk a14fb626cc9ff386403096cf912990bfb73e83c3c985b7e19207aed7bc72ae4e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:47:6e:d1:98:2e\",\"name\":\"a14fb626cc9ff38\"},{\"mac\":\"0a:58:0a:82:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1858e4d4-1805-4d5b-81d5-a80f7e974195\"}],\"ips\":[{\"address\":\"10.130.0.11/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:15:10.239955 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:15:10.240098 3663 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0514 23:15:10.279324 3663 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 39.274034ms I0514 23:15:10.279356 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 39.322542ms I0514 23:15:10.281433 3663 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 41.497556ms, libovsdb time 1.414186ms I0514 23:15:10.641845 3663 cni.go:452] [openshift-monitoring/alertmanager-main-0 770bd3c636ab30d397bea319d2477159c38cf1e5c0fd25d3d26dc67a44c3ecdf network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 770bd3c636ab30d397bea319d2477159c38cf1e5c0fd25d3d26dc67a44c3ecdf network default NAD default NAD key ] I0514 23:15:10.652533 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 770bd3c636ab30d, network: default, NAD default, SandboxID: "770bd3c636ab30d397bea319d2477159c38cf1e5c0fd25d3d26dc67a44c3ecdf", PCI device ID: , UID: "52eff6b9-5477-4dd9-bea9-6bca4cf96ee6", MAC: 0a:58:0a:82:00:0c, IPs: [10.130.0.12/23] I0514 23:15:10.702688 3663 cni.go:473] [openshift-monitoring/alertmanager-main-0 770bd3c636ab30d397bea319d2477159c38cf1e5c0fd25d3d26dc67a44c3ecdf network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 770bd3c636ab30d397bea319d2477159c38cf1e5c0fd25d3d26dc67a44c3ecdf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:58:da:ff:f1:81\",\"name\":\"770bd3c636ab30d\"},{\"mac\":\"0a:58:0a:82:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3d34e06d-76f1-4715-85f4-a9a86675d8a9\"}],\"ips\":[{\"address\":\"10.130.0.12/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:15:11.059509 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0514 23:15:12.103719 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-64c6d5bccf-bjn4n] creating logical port openshift-monitoring_thanos-querier-64c6d5bccf-bjn4n for pod on switch ip-10-0-0-78.ec2.internal I0514 23:15:12.103908 3663 kube.go:256] Updating pod openshift-monitoring/thanos-querier-64c6d5bccf-bjn4n I0514 23:15:12.131897 3663 pod.go:62] [openshift-monitoring/thanos-querier-64c6d5bccf-bjn4n] pod update took 28.040012ms I0514 23:15:12.131922 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-64c6d5bccf-bjn4n] addLogicalPort annotation time took 28.069089ms I0514 23:15:12.133291 3663 pods.go:271] [openshift-monitoring/thanos-querier-64c6d5bccf-bjn4n] addLogicalPort took 29.592972ms, libovsdb time 969.612µs W0514 23:15:12.294411 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0514 23:15:12.294429 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0514 23:15:12.294436 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0514 23:15:12.481573 3663 cni.go:452] [openshift-monitoring/thanos-querier-64c6d5bccf-bjn4n 79c1a1076c7223d46d8d0e3a86feb74564402eaf2e89b49ec530f675e4057b83 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-64c6d5bccf-bjn4n 79c1a1076c7223d46d8d0e3a86feb74564402eaf2e89b49ec530f675e4057b83 network default NAD default NAD key ] I0514 23:15:12.491421 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-64c6d5bccf-bjn4n, hostIfaceName: 79c1a1076c7223d, network: default, NAD default, SandboxID: "79c1a1076c7223d46d8d0e3a86feb74564402eaf2e89b49ec530f675e4057b83", PCI device ID: , UID: "784e1a57-35e9-4e57-9d17-f073c142d795", MAC: 0a:58:0a:82:00:0d, IPs: [10.130.0.13/23] I0514 23:15:12.540362 3663 cni.go:473] [openshift-monitoring/thanos-querier-64c6d5bccf-bjn4n 79c1a1076c7223d46d8d0e3a86feb74564402eaf2e89b49ec530f675e4057b83 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-64c6d5bccf-bjn4n 79c1a1076c7223d46d8d0e3a86feb74564402eaf2e89b49ec530f675e4057b83 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a6:5b:93:ac:e3:c2\",\"name\":\"79c1a1076c7223d\"},{\"mac\":\"0a:58:0a:82:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c2c156b9-62f1-4aa9-9e37-3a4bd7b56c02\"}],\"ips\":[{\"address\":\"10.130.0.13/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:15:12.580003 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0514 23:15:12.580034 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0514 23:15:13.390844 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:13.390943 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:13.390957 3663 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0514 23:15:13.661162 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-787cb4645c-pg8kw] creating logical port openshift-monitoring_metrics-server-787cb4645c-pg8kw for pod on switch ip-10-0-0-78.ec2.internal I0514 23:15:13.661262 3663 kube.go:256] Updating pod openshift-monitoring/metrics-server-787cb4645c-pg8kw I0514 23:15:13.680138 3663 pod.go:62] [openshift-monitoring/metrics-server-787cb4645c-pg8kw] pod update took 18.906826ms I0514 23:15:13.680157 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-787cb4645c-pg8kw] addLogicalPort annotation time took 18.930178ms I0514 23:15:13.681221 3663 pods.go:271] [openshift-monitoring/metrics-server-787cb4645c-pg8kw] addLogicalPort took 20.075809ms, libovsdb time 695.465µs I0514 23:15:13.770990 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-26k89] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-26k89 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:15:13.771128 3663 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-26k89 I0514 23:15:13.786029 3663 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-26k89] pod update took 14.925863ms I0514 23:15:13.786055 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-26k89] addLogicalPort annotation time took 14.954282ms I0514 23:15:13.787221 3663 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-26k89] addLogicalPort took 16.249874ms, libovsdb time 852.181µs W0514 23:15:14.454283 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0514 23:15:14.573455 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads I0514 23:15:15.198633 3663 cni.go:452] [openshift-monitoring/metrics-server-787cb4645c-pg8kw c1e163ef404e0fc8cb2cfc2f7350b0c25fdf5794ae95bb8d4f99c8be753d9fe4 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-787cb4645c-pg8kw c1e163ef404e0fc8cb2cfc2f7350b0c25fdf5794ae95bb8d4f99c8be753d9fe4 network default NAD default NAD key ] I0514 23:15:15.198720 3663 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-26k89 74cfaacdd5815b82e2f0e2f3456a7db1e03add9895dce60021891bd39bdb4a28 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-26k89 74cfaacdd5815b82e2f0e2f3456a7db1e03add9895dce60021891bd39bdb4a28 network default NAD default NAD key ] I0514 23:15:15.209757 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-26k89, hostIfaceName: 74cfaacdd5815b8, network: default, NAD default, SandboxID: "74cfaacdd5815b82e2f0e2f3456a7db1e03add9895dce60021891bd39bdb4a28", PCI device ID: , UID: "975ce43f-edb5-41fb-bd7b-fb54850c60bb", MAC: 0a:58:0a:82:00:0f, IPs: [10.130.0.15/23] I0514 23:15:15.210412 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-787cb4645c-pg8kw, hostIfaceName: c1e163ef404e0fc, network: default, NAD default, SandboxID: "c1e163ef404e0fc8cb2cfc2f7350b0c25fdf5794ae95bb8d4f99c8be753d9fe4", PCI device ID: , UID: "97961358-e9dd-446c-aa3b-9eed04dc9733", MAC: 0a:58:0a:82:00:0e, IPs: [10.130.0.14/23] I0514 23:15:15.257599 3663 cni.go:473] [openshift-monitoring/metrics-server-787cb4645c-pg8kw c1e163ef404e0fc8cb2cfc2f7350b0c25fdf5794ae95bb8d4f99c8be753d9fe4 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-787cb4645c-pg8kw c1e163ef404e0fc8cb2cfc2f7350b0c25fdf5794ae95bb8d4f99c8be753d9fe4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:de:f8:28:2d:57\",\"name\":\"c1e163ef404e0fc\"},{\"mac\":\"0a:58:0a:82:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7a0f398f-491e-4b83-9060-b07a4859764e\"}],\"ips\":[{\"address\":\"10.130.0.14/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:15:15.273269 3663 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-26k89 74cfaacdd5815b82e2f0e2f3456a7db1e03add9895dce60021891bd39bdb4a28 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-26k89 74cfaacdd5815b82e2f0e2f3456a7db1e03add9895dce60021891bd39bdb4a28 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:50:ab:4f:30:5e\",\"name\":\"74cfaacdd5815b8\"},{\"mac\":\"0a:58:0a:82:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5fb3e58c-d204-4355-bf2f-37140888254e\"}],\"ips\":[{\"address\":\"10.130.0.15/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:15:15.496792 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:15:15.496954 3663 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0514 23:15:15.526103 3663 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 29.187828ms I0514 23:15:15.526127 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 29.21705ms I0514 23:15:15.527436 3663 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 30.66613ms, libovsdb time 879.295µs I0514 23:15:15.884436 3663 cni.go:452] [openshift-monitoring/prometheus-k8s-0 612ac9ade2be16164e71f61b352336d3f099f55dd8586d317af04b5e14f1d460 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 612ac9ade2be16164e71f61b352336d3f099f55dd8586d317af04b5e14f1d460 network default NAD default NAD key ] I0514 23:15:15.896383 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 612ac9ade2be161, network: default, NAD default, SandboxID: "612ac9ade2be16164e71f61b352336d3f099f55dd8586d317af04b5e14f1d460", PCI device ID: , UID: "f698c389-d40d-4a29-ab23-72bec4378a93", MAC: 0a:58:0a:82:00:10, IPs: [10.130.0.16/23] I0514 23:15:15.951410 3663 cni.go:473] [openshift-monitoring/prometheus-k8s-0 612ac9ade2be16164e71f61b352336d3f099f55dd8586d317af04b5e14f1d460 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 612ac9ade2be16164e71f61b352336d3f099f55dd8586d317af04b5e14f1d460 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:1a:ab:3a:4b:2c\",\"name\":\"612ac9ade2be161\"},{\"mac\":\"0a:58:0a:82:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6efc2d34-4420-4555-8d7e-66b778d87f99\"}],\"ips\":[{\"address\":\"10.130.0.16/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:15:16.269577 3663 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-79bf45776b-fn62n IP address from the namespace address-set, err: pod openshift-console/console-79bf45776b-fn62n: no pod IPs found W0514 23:15:16.377403 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:15:16.377426 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:15:18.414343 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0514 23:15:18.479929 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:18.479952 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:18.479959 3663 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:18.479967 3663 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:19.644572 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0514 23:15:19.720366 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:15:19.720389 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:15:19.864433 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:19.864450 3663 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:19.864458 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0514 23:15:21.044137 3663 cni.go:452] [kube-system/global-pull-secret-syncer-gjtjf 0b1a24f365feac90b95e000894641f14a30813bc7d2540de42826bec4275b229 network default NAD default NAD key ] ADD starting CNI request [kube-system/global-pull-secret-syncer-gjtjf 0b1a24f365feac90b95e000894641f14a30813bc7d2540de42826bec4275b229 network default NAD default NAD key ] I0514 23:15:21.047035 3663 cni.go:452] [openshift-multus/network-metrics-daemon-5t9hc fcf90ffc7bad11f1a21e254df17f0d894342637a731300ea0fd39995875b880d network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-5t9hc fcf90ffc7bad11f1a21e254df17f0d894342637a731300ea0fd39995875b880d network default NAD default NAD key ] I0514 23:15:21.054957 3663 helper_linux.go:556] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-gjtjf, hostIfaceName: 0b1a24f365feac9, network: default, NAD default, SandboxID: "0b1a24f365feac90b95e000894641f14a30813bc7d2540de42826bec4275b229", PCI device ID: , UID: "a69dac3d-8b55-45d6-a8d5-ef505f22c694", MAC: 0a:58:0a:82:00:04, IPs: [10.130.0.4/23] I0514 23:15:21.058034 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-5t9hc, hostIfaceName: fcf90ffc7bad11f, network: default, NAD default, SandboxID: "fcf90ffc7bad11f1a21e254df17f0d894342637a731300ea0fd39995875b880d", PCI device ID: , UID: "c1ada182-f0d1-4fbf-8bf3-04bd88851d65", MAC: 0a:58:0a:82:00:03, IPs: [10.130.0.3/23] I0514 23:15:21.117526 3663 cni.go:473] [kube-system/global-pull-secret-syncer-gjtjf 0b1a24f365feac90b95e000894641f14a30813bc7d2540de42826bec4275b229 network default NAD default NAD key default] ADD finished CNI request [kube-system/global-pull-secret-syncer-gjtjf 0b1a24f365feac90b95e000894641f14a30813bc7d2540de42826bec4275b229 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:78:6e:07:55:72\",\"name\":\"0b1a24f365feac9\"},{\"mac\":\"0a:58:0a:82:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/171ced3d-0526-4189-8b3e-74a87e6ee3ea\"}],\"ips\":[{\"address\":\"10.130.0.4/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:15:21.132347 3663 cni.go:473] [openshift-multus/network-metrics-daemon-5t9hc fcf90ffc7bad11f1a21e254df17f0d894342637a731300ea0fd39995875b880d network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-5t9hc fcf90ffc7bad11f1a21e254df17f0d894342637a731300ea0fd39995875b880d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:ad:ea:aa:e3:04\",\"name\":\"fcf90ffc7bad11f\"},{\"mac\":\"0a:58:0a:82:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/974584f7-c2cd-4a24-bca1-20d4c176f89a\"}],\"ips\":[{\"address\":\"10.130.0.3/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:15:21.302126 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0514 23:15:21.360571 3663 cni.go:452] [openshift-network-diagnostics/network-check-target-kjlnq 6fb56e054b2c8444bf29f7682c564124692bd6ad0e82c74e569158a6027aeaa3 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-kjlnq 6fb56e054b2c8444bf29f7682c564124692bd6ad0e82c74e569158a6027aeaa3 network default NAD default NAD key ] I0514 23:15:21.370586 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-kjlnq, hostIfaceName: 6fb56e054b2c844, network: default, NAD default, SandboxID: "6fb56e054b2c8444bf29f7682c564124692bd6ad0e82c74e569158a6027aeaa3", PCI device ID: , UID: "a2169d2f-3db1-4d29-8151-89a86ccab378", MAC: 0a:58:0a:82:00:05, IPs: [10.130.0.5/23] I0514 23:15:21.417559 3663 cni.go:473] [openshift-network-diagnostics/network-check-target-kjlnq 6fb56e054b2c8444bf29f7682c564124692bd6ad0e82c74e569158a6027aeaa3 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-kjlnq 6fb56e054b2c8444bf29f7682c564124692bd6ad0e82c74e569158a6027aeaa3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:70:df:28:a8:ba\",\"name\":\"6fb56e054b2c844\"},{\"mac\":\"0a:58:0a:82:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ad91a6b9-dc5f-4f69-a517-7737f9d4539d\"}],\"ips\":[{\"address\":\"10.130.0.5/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:15:21.650639 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:21.650663 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:21.650668 3663 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:21.650672 3663 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0514 23:15:23.412606 3663 obj_retry.go:498] Detected object openshift-image-registry/image-registry-55bc57d65c-mbggh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:23.413197 3663 obj_retry.go:498] Detected object openshift-image-registry/image-registry-55bc57d65c-mbggh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:23.692104 3663 obj_retry.go:492] Detected object openshift-image-registry/image-registry-55bc57d65c-mbggh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:23.692127 3663 obj_retry.go:492] Detected object openshift-image-registry/image-registry-55bc57d65c-mbggh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:23.695700 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-55bc57d65c-mbggh I0514 23:15:23.695726 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-55bc57d65c-mbggh W0514 23:15:24.399690 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:24.399712 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:24.399720 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:24.399728 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0514 23:15:36.230799 3663 cni.go:452] [openshift-console/console-b57f75797-f2pc2 e5f853f512fccfd5793ab68ee1ded04226f41ac11c097147dd25b8df236b9f3c network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-b57f75797-f2pc2 e5f853f512fccfd5793ab68ee1ded04226f41ac11c097147dd25b8df236b9f3c network default NAD default NAD key ] I0514 23:15:36.310565 3663 cni.go:473] [openshift-console/console-b57f75797-f2pc2 e5f853f512fccfd5793ab68ee1ded04226f41ac11c097147dd25b8df236b9f3c network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-b57f75797-f2pc2 e5f853f512fccfd5793ab68ee1ded04226f41ac11c097147dd25b8df236b9f3c network default NAD default NAD key default], result "{}", err I0514 23:15:36.384348 3663 obj_retry.go:498] Detected object openshift-console/console-b57f75797-f2pc2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:36.384379 3663 pods.go:176] Deleting pod: openshift-console/console-b57f75797-f2pc2 I0514 23:15:36.385606 3663 pods.go:236] Attempting to release IPs for pod: openshift-console/console-b57f75797-f2pc2, ips: 10.130.0.10 I0514 23:15:36.385633 3663 obj_retry.go:498] Detected object openshift-console/console-b57f75797-f2pc2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:36.396113 3663 base_network_controller_pods.go:487] [default/openshift-console/console-7bf857b667-hvfs6] creating logical port openshift-console_console-7bf857b667-hvfs6 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:15:36.396241 3663 kube.go:256] Updating pod openshift-console/console-7bf857b667-hvfs6 I0514 23:15:36.410552 3663 pod.go:62] [openshift-console/console-7bf857b667-hvfs6] pod update took 14.339413ms I0514 23:15:36.410577 3663 base_network_controller_pods.go:951] [default/openshift-console/console-7bf857b667-hvfs6] addLogicalPort annotation time took 14.365515ms I0514 23:15:36.411739 3663 pods.go:271] [openshift-console/console-7bf857b667-hvfs6] addLogicalPort took 15.642189ms, libovsdb time 836.745µs I0514 23:15:36.779138 3663 obj_retry.go:492] Detected object openshift-console/console-b57f75797-f2pc2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:36.779168 3663 obj_retry.go:492] Detected object openshift-console/console-b57f75797-f2pc2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:36.787400 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-b57f75797-f2pc2 I0514 23:15:36.787428 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-b57f75797-f2pc2 I0514 23:15:36.810068 3663 cni.go:452] [openshift-console/console-7bf857b667-hvfs6 b0b52c92833b7bde29e40404e798c77bc8aef36495b3f3547632b7b54bcdf3f9 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7bf857b667-hvfs6 b0b52c92833b7bde29e40404e798c77bc8aef36495b3f3547632b7b54bcdf3f9 network default NAD default NAD key ] I0514 23:15:36.819369 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7bf857b667-hvfs6, hostIfaceName: b0b52c92833b7bd, network: default, NAD default, SandboxID: "b0b52c92833b7bde29e40404e798c77bc8aef36495b3f3547632b7b54bcdf3f9", PCI device ID: , UID: "d6b5613d-b314-4d2a-8188-360475eb8a2e", MAC: 0a:58:0a:82:00:11, IPs: [10.130.0.17/23] I0514 23:15:36.868494 3663 cni.go:473] [openshift-console/console-7bf857b667-hvfs6 b0b52c92833b7bde29e40404e798c77bc8aef36495b3f3547632b7b54bcdf3f9 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7bf857b667-hvfs6 b0b52c92833b7bde29e40404e798c77bc8aef36495b3f3547632b7b54bcdf3f9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:73:94:97:d6:f8\",\"name\":\"b0b52c92833b7bd\"},{\"mac\":\"0a:58:0a:82:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/19bb0874-3c71-47fb-b6ef-9532a769d497\"}],\"ips\":[{\"address\":\"10.130.0.17/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:15:37.573808 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0514 23:15:39.385448 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0514 23:15:40.669244 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:40.669264 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:40.669271 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0514 23:15:42.910134 3663 obj_retry.go:498] Detected object openshift-console/console-79bf45776b-h4hm6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:42.910651 3663 obj_retry.go:498] Detected object openshift-console/console-79bf45776b-h4hm6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:43.640592 3663 obj_retry.go:492] Detected object openshift-console/console-79bf45776b-h4hm6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:43.640621 3663 obj_retry.go:492] Detected object openshift-console/console-79bf45776b-h4hm6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:43.647717 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-79bf45776b-h4hm6 I0514 23:15:43.647739 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-79bf45776b-h4hm6 W0514 23:15:43.678661 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0514 23:15:44.777164 3663 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:44.777667 3663 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:44.777695 3663 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:15:44.777703 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0514 23:15:44.777719 3663 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:15:44.777728 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0514 23:15:45.815688 3663 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:45.815718 3663 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:45.815731 3663 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:45.815740 3663 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:46.690503 3663 obj_retry.go:498] Detected object openshift-image-registry/image-registry-d477cf66b-ddgmf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:46.691160 3663 obj_retry.go:498] Detected object openshift-image-registry/image-registry-d477cf66b-ddgmf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0514 23:15:47.527346 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0514 23:15:47.629584 3663 obj_retry.go:492] Detected object openshift-image-registry/image-registry-d477cf66b-ddgmf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:47.629603 3663 obj_retry.go:492] Detected object openshift-image-registry/image-registry-d477cf66b-ddgmf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:47.637741 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-d477cf66b-ddgmf I0514 23:15:47.637769 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-d477cf66b-ddgmf I0514 23:15:49.155590 3663 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0514 23:15:49.163499 3663 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0514 23:15:54.312100 3663 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0514 23:15:57.447120 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0514 23:16:02.710151 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0514 23:16:08.400869 3663 obj_retry.go:498] Detected object openshift-ingress/router-default-7f4c9f6df5-xjsx9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:08.401389 3663 obj_retry.go:498] Detected object openshift-ingress/router-default-7f4c9f6df5-xjsx9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:08.731742 3663 obj_retry.go:492] Detected object openshift-ingress/router-default-7f4c9f6df5-xjsx9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:08.731766 3663 obj_retry.go:492] Detected object openshift-ingress/router-default-7f4c9f6df5-xjsx9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:08.734656 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-7f4c9f6df5-xjsx9 I0514 23:16:08.734677 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-7f4c9f6df5-xjsx9 I0514 23:16:12.878247 3663 obj_retry.go:498] Detected object openshift-console/console-b57f75797-44prb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:12.878779 3663 obj_retry.go:498] Detected object openshift-console/console-b57f75797-44prb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:13.865419 3663 obj_retry.go:492] Detected object openshift-console/console-b57f75797-44prb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:13.865446 3663 obj_retry.go:492] Detected object openshift-console/console-b57f75797-44prb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:13.873527 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-b57f75797-44prb I0514 23:16:13.873548 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-b57f75797-44prb W0514 23:16:16.661121 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:16:16.661139 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0514 23:16:30.971666 3663 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:30.972263 3663 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:31.958540 3663 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:31.958567 3663 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:31.966243 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0514 23:16:31.966272 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0514 23:16:33.734961 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-6b594cf7b4-bm2nd] creating logical port openshift-monitoring_telemeter-client-6b594cf7b4-bm2nd for pod on switch ip-10-0-0-78.ec2.internal I0514 23:16:33.735065 3663 kube.go:256] Updating pod openshift-monitoring/telemeter-client-6b594cf7b4-bm2nd I0514 23:16:33.764019 3663 pod.go:62] [openshift-monitoring/telemeter-client-6b594cf7b4-bm2nd] pod update took 28.990153ms I0514 23:16:33.764047 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/telemeter-client-6b594cf7b4-bm2nd] addLogicalPort annotation time took 29.020897ms I0514 23:16:33.765286 3663 pods.go:271] [openshift-monitoring/telemeter-client-6b594cf7b4-bm2nd] addLogicalPort took 30.340473ms, libovsdb time 910.562µs I0514 23:16:34.095872 3663 cni.go:452] [openshift-monitoring/telemeter-client-6b594cf7b4-bm2nd 25d9a372f4a2e4e42c5bf082904e1f1242075ef147e48ca423384ded650c6a49 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/telemeter-client-6b594cf7b4-bm2nd 25d9a372f4a2e4e42c5bf082904e1f1242075ef147e48ca423384ded650c6a49 network default NAD default NAD key ] I0514 23:16:34.105571 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-6b594cf7b4-bm2nd, hostIfaceName: 25d9a372f4a2e4e, network: default, NAD default, SandboxID: "25d9a372f4a2e4e42c5bf082904e1f1242075ef147e48ca423384ded650c6a49", PCI device ID: , UID: "55c04b23-f3a0-4674-a8bf-f834c870db34", MAC: 0a:58:0a:82:00:12, IPs: [10.130.0.18/23] I0514 23:16:34.154423 3663 cni.go:473] [openshift-monitoring/telemeter-client-6b594cf7b4-bm2nd 25d9a372f4a2e4e42c5bf082904e1f1242075ef147e48ca423384ded650c6a49 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/telemeter-client-6b594cf7b4-bm2nd 25d9a372f4a2e4e42c5bf082904e1f1242075ef147e48ca423384ded650c6a49 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:92:21:a6:6a:f7\",\"name\":\"25d9a372f4a2e4e\"},{\"mac\":\"0a:58:0a:82:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d53b1c7d-da79-410f-9436-a0b72f4fa458\"}],\"ips\":[{\"address\":\"10.130.0.18/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:16:35.396376 3663 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:35.396946 3663 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:35.988360 3663 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:35.988385 3663 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:35.996211 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0514 23:16:35.996275 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0514 23:16:53.687567 3663 cni.go:452] [openshift-monitoring/alertmanager-main-0 770bd3c636ab30d397bea319d2477159c38cf1e5c0fd25d3d26dc67a44c3ecdf network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 770bd3c636ab30d397bea319d2477159c38cf1e5c0fd25d3d26dc67a44c3ecdf network default NAD default NAD key ] I0514 23:16:53.770388 3663 cni.go:473] [openshift-monitoring/alertmanager-main-0 770bd3c636ab30d397bea319d2477159c38cf1e5c0fd25d3d26dc67a44c3ecdf network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 770bd3c636ab30d397bea319d2477159c38cf1e5c0fd25d3d26dc67a44c3ecdf network default NAD default NAD key default], result "{}", err I0514 23:16:53.838027 3663 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:53.838066 3663 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0514 23:16:53.839248 3663 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.130.0.12 I0514 23:16:53.839278 3663 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:54.786007 3663 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:54.786039 3663 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:54.794152 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0514 23:16:54.794182 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0514 23:16:54.833999 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:16:54.834120 3663 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0514 23:16:54.854498 3663 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 20.423572ms I0514 23:16:54.854523 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 20.455563ms I0514 23:16:54.855895 3663 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 21.91074ms, libovsdb time 1.033267ms I0514 23:16:55.193630 3663 cni.go:452] [openshift-monitoring/alertmanager-main-0 b0621c9efea1e5c4fbbc4c2b7a40709fe9790320bccdba2bcc4f8499db68b3e3 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 b0621c9efea1e5c4fbbc4c2b7a40709fe9790320bccdba2bcc4f8499db68b3e3 network default NAD default NAD key ] I0514 23:16:55.203432 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: b0621c9efea1e5c, network: default, NAD default, SandboxID: "b0621c9efea1e5c4fbbc4c2b7a40709fe9790320bccdba2bcc4f8499db68b3e3", PCI device ID: , UID: "58bccd6b-4617-405a-b855-ec9d8d35b79d", MAC: 0a:58:0a:82:00:13, IPs: [10.130.0.19/23] I0514 23:16:55.251148 3663 cni.go:473] [openshift-monitoring/alertmanager-main-0 b0621c9efea1e5c4fbbc4c2b7a40709fe9790320bccdba2bcc4f8499db68b3e3 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 b0621c9efea1e5c4fbbc4c2b7a40709fe9790320bccdba2bcc4f8499db68b3e3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:e7:63:20:8a:7b\",\"name\":\"b0621c9efea1e5c\"},{\"mac\":\"0a:58:0a:82:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3e93d9d9-e2a5-4513-a0fc-e0c49987fcc7\"}],\"ips\":[{\"address\":\"10.130.0.19/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:17:03.289919 3663 cni.go:452] [openshift-console/console-7bf857b667-hvfs6 b0b52c92833b7bde29e40404e798c77bc8aef36495b3f3547632b7b54bcdf3f9 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7bf857b667-hvfs6 b0b52c92833b7bde29e40404e798c77bc8aef36495b3f3547632b7b54bcdf3f9 network default NAD default NAD key ] I0514 23:17:03.377787 3663 cni.go:473] [openshift-console/console-7bf857b667-hvfs6 b0b52c92833b7bde29e40404e798c77bc8aef36495b3f3547632b7b54bcdf3f9 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7bf857b667-hvfs6 b0b52c92833b7bde29e40404e798c77bc8aef36495b3f3547632b7b54bcdf3f9 network default NAD default NAD key default], result "{}", err I0514 23:17:03.422576 3663 obj_retry.go:498] Detected object openshift-console/console-7bf857b667-hvfs6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:03.422612 3663 pods.go:176] Deleting pod: openshift-console/console-7bf857b667-hvfs6 I0514 23:17:03.423804 3663 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7bf857b667-hvfs6, ips: 10.130.0.17 I0514 23:17:03.423843 3663 obj_retry.go:498] Detected object openshift-console/console-7bf857b667-hvfs6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:03.431611 3663 base_network_controller_pods.go:487] [default/openshift-console/console-6dfb89476d-28rdd] creating logical port openshift-console_console-6dfb89476d-28rdd for pod on switch ip-10-0-0-78.ec2.internal I0514 23:17:03.431710 3663 kube.go:256] Updating pod openshift-console/console-6dfb89476d-28rdd I0514 23:17:03.447925 3663 pod.go:62] [openshift-console/console-6dfb89476d-28rdd] pod update took 16.24103ms I0514 23:17:03.447943 3663 base_network_controller_pods.go:951] [default/openshift-console/console-6dfb89476d-28rdd] addLogicalPort annotation time took 16.265278ms I0514 23:17:03.449079 3663 pods.go:271] [openshift-console/console-6dfb89476d-28rdd] addLogicalPort took 17.484679ms, libovsdb time 817.653µs I0514 23:17:03.791789 3663 cni.go:452] [openshift-console/console-6dfb89476d-28rdd 6948609d6965cd7f3af13f5be89f05ccdaede89df54fa8c811e5cfeee7596c20 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-6dfb89476d-28rdd 6948609d6965cd7f3af13f5be89f05ccdaede89df54fa8c811e5cfeee7596c20 network default NAD default NAD key ] I0514 23:17:03.800996 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-6dfb89476d-28rdd, hostIfaceName: 6948609d6965cd7, network: default, NAD default, SandboxID: "6948609d6965cd7f3af13f5be89f05ccdaede89df54fa8c811e5cfeee7596c20", PCI device ID: , UID: "1ab34455-44da-4ea5-b582-af14abb66d19", MAC: 0a:58:0a:82:00:14, IPs: [10.130.0.20/23] I0514 23:17:03.830288 3663 obj_retry.go:492] Detected object openshift-console/console-7bf857b667-hvfs6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:03.830310 3663 obj_retry.go:492] Detected object openshift-console/console-7bf857b667-hvfs6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:03.837433 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7bf857b667-hvfs6 I0514 23:17:03.837459 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7bf857b667-hvfs6 I0514 23:17:03.852204 3663 cni.go:473] [openshift-console/console-6dfb89476d-28rdd 6948609d6965cd7f3af13f5be89f05ccdaede89df54fa8c811e5cfeee7596c20 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-6dfb89476d-28rdd 6948609d6965cd7f3af13f5be89f05ccdaede89df54fa8c811e5cfeee7596c20 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d2:77:3a:e4:36:1e\",\"name\":\"6948609d6965cd7\"},{\"mac\":\"0a:58:0a:82:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/50a8ccd8-84fc-452b-97c3-340a3208a426\"}],\"ips\":[{\"address\":\"10.130.0.20/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:17:13.880633 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0514 23:17:14.413326 3663 obj_retry.go:498] Detected object openshift-console/console-7bf857b667-s2qrk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:14.413883 3663 obj_retry.go:498] Detected object openshift-console/console-7bf857b667-s2qrk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:15.020117 3663 obj_retry.go:492] Detected object openshift-console/console-7bf857b667-s2qrk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:15.020141 3663 obj_retry.go:492] Detected object openshift-console/console-7bf857b667-s2qrk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:15.023110 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7bf857b667-s2qrk I0514 23:17:15.023132 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7bf857b667-s2qrk W0514 23:17:15.246470 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:17:15.246490 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:17:15.246498 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0514 23:17:29.205803 3663 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:29.209506 3663 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:30.101192 3663 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:30.101215 3663 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:38.463262 3663 cni.go:452] [openshift-monitoring/prometheus-k8s-0 612ac9ade2be16164e71f61b352336d3f099f55dd8586d317af04b5e14f1d460 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 612ac9ade2be16164e71f61b352336d3f099f55dd8586d317af04b5e14f1d460 network default NAD default NAD key ] I0514 23:17:38.538361 3663 cni.go:473] [openshift-monitoring/prometheus-k8s-0 612ac9ade2be16164e71f61b352336d3f099f55dd8586d317af04b5e14f1d460 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 612ac9ade2be16164e71f61b352336d3f099f55dd8586d317af04b5e14f1d460 network default NAD default NAD key default], result "{}", err I0514 23:17:38.597968 3663 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:38.598005 3663 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0514 23:17:38.599176 3663 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.130.0.16 I0514 23:17:38.599200 3663 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:38.971182 3663 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:38.971209 3663 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:38.979658 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0514 23:17:38.979682 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0514 23:17:39.025924 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:17:39.026024 3663 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0514 23:17:39.051655 3663 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 25.668446ms I0514 23:17:39.051680 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 25.697607ms I0514 23:17:39.052975 3663 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 27.068006ms, libovsdb time 948.8µs I0514 23:17:39.389284 3663 cni.go:452] [openshift-monitoring/prometheus-k8s-0 8bedb8924871a41b519e240c1bfb985fe49a359792b66adb9abcb037cefde819 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 8bedb8924871a41b519e240c1bfb985fe49a359792b66adb9abcb037cefde819 network default NAD default NAD key ] I0514 23:17:39.399594 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 8bedb8924871a41, network: default, NAD default, SandboxID: "8bedb8924871a41b519e240c1bfb985fe49a359792b66adb9abcb037cefde819", PCI device ID: , UID: "56f65ba7-256e-4e75-8880-a76cf4b8179a", MAC: 0a:58:0a:82:00:15, IPs: [10.130.0.21/23] I0514 23:17:39.447946 3663 cni.go:473] [openshift-monitoring/prometheus-k8s-0 8bedb8924871a41b519e240c1bfb985fe49a359792b66adb9abcb037cefde819 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 8bedb8924871a41b519e240c1bfb985fe49a359792b66adb9abcb037cefde819 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:d1:d7:04:fc:95\",\"name\":\"8bedb8924871a41\"},{\"mac\":\"0a:58:0a:82:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/96cd6a74-67c5-40d0-b0ab-5ef45eb6019c\"}],\"ips\":[{\"address\":\"10.130.0.21/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:18:40.310196 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:18:40.310220 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0514 23:19:16.008932 3663 cni.go:452] [openshift-console/console-6dfb89476d-28rdd 6948609d6965cd7f3af13f5be89f05ccdaede89df54fa8c811e5cfeee7596c20 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-6dfb89476d-28rdd 6948609d6965cd7f3af13f5be89f05ccdaede89df54fa8c811e5cfeee7596c20 network default NAD default NAD key ] I0514 23:19:16.088246 3663 cni.go:473] [openshift-console/console-6dfb89476d-28rdd 6948609d6965cd7f3af13f5be89f05ccdaede89df54fa8c811e5cfeee7596c20 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-6dfb89476d-28rdd 6948609d6965cd7f3af13f5be89f05ccdaede89df54fa8c811e5cfeee7596c20 network default NAD default NAD key default], result "{}", err I0514 23:19:16.139851 3663 obj_retry.go:498] Detected object openshift-console/console-6dfb89476d-28rdd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:19:16.139883 3663 pods.go:176] Deleting pod: openshift-console/console-6dfb89476d-28rdd I0514 23:19:16.141302 3663 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6dfb89476d-28rdd, ips: 10.130.0.20 I0514 23:19:16.141331 3663 obj_retry.go:498] Detected object openshift-console/console-6dfb89476d-28rdd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:19:16.146436 3663 base_network_controller_pods.go:487] [default/openshift-console/console-58f495ddb-rxsfk] creating logical port openshift-console_console-58f495ddb-rxsfk for pod on switch ip-10-0-0-78.ec2.internal I0514 23:19:16.146556 3663 kube.go:256] Updating pod openshift-console/console-58f495ddb-rxsfk I0514 23:19:16.161292 3663 pod.go:62] [openshift-console/console-58f495ddb-rxsfk] pod update took 14.761684ms I0514 23:19:16.161315 3663 base_network_controller_pods.go:951] [default/openshift-console/console-58f495ddb-rxsfk] addLogicalPort annotation time took 14.788285ms I0514 23:19:16.162501 3663 pods.go:271] [openshift-console/console-58f495ddb-rxsfk] addLogicalPort took 16.07838ms, libovsdb time 870.37µs I0514 23:19:16.331846 3663 obj_retry.go:492] Detected object openshift-console/console-6dfb89476d-28rdd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:19:16.331873 3663 obj_retry.go:492] Detected object openshift-console/console-6dfb89476d-28rdd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:19:16.339298 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6dfb89476d-28rdd I0514 23:19:16.339330 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6dfb89476d-28rdd I0514 23:19:16.507143 3663 cni.go:452] [openshift-console/console-58f495ddb-rxsfk 9268045959f772c7e243179ea0984d1758db4c909fd4e356710458d331dbd54c network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-58f495ddb-rxsfk 9268045959f772c7e243179ea0984d1758db4c909fd4e356710458d331dbd54c network default NAD default NAD key ] I0514 23:19:16.516841 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-58f495ddb-rxsfk, hostIfaceName: 9268045959f772c, network: default, NAD default, SandboxID: "9268045959f772c7e243179ea0984d1758db4c909fd4e356710458d331dbd54c", PCI device ID: , UID: "eb92107f-4317-481b-a427-5c84d9ac2162", MAC: 0a:58:0a:82:00:16, IPs: [10.130.0.22/23] I0514 23:19:16.563591 3663 cni.go:473] [openshift-console/console-58f495ddb-rxsfk 9268045959f772c7e243179ea0984d1758db4c909fd4e356710458d331dbd54c network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-58f495ddb-rxsfk 9268045959f772c7e243179ea0984d1758db4c909fd4e356710458d331dbd54c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:28:12:b7:b8:e2\",\"name\":\"9268045959f772c\"},{\"mac\":\"0a:58:0a:82:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ca5ef90a-2993-4618-8f7a-2c6fe93ff0ed\"}],\"ips\":[{\"address\":\"10.130.0.22/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:19:26.805612 3663 obj_retry.go:498] Detected object openshift-console/console-6dfb89476d-ktf8r of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:19:26.806229 3663 obj_retry.go:498] Detected object openshift-console/console-6dfb89476d-ktf8r of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0514 23:19:27.391275 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0514 23:19:27.712217 3663 obj_retry.go:492] Detected object openshift-console/console-6dfb89476d-ktf8r of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:19:27.712243 3663 obj_retry.go:492] Detected object openshift-console/console-6dfb89476d-ktf8r of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:19:27.722872 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6dfb89476d-ktf8r I0514 23:19:27.722899 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6dfb89476d-ktf8r I0514 23:19:43.836039 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0514 23:19:48.721211 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0514 23:20:03.846068 3663 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:20:03.846648 3663 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:20:04.742075 3663 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:20:04.742101 3663 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:20:15.723887 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=51 I0514 23:20:53.204683 3663 namespace.go:98] [tsf] adding namespace I0514 23:20:53.205517 3663 namespace.go:102] [tsf] adding namespace took 811.505µs I0514 23:20:53.229423 3663 namespace.go:142] [tsf] updating namespace I0514 23:20:53.237558 3663 namespace.go:142] [tsf] updating namespace W0514 23:20:53.951765 3663 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:20:57.617278 3663 namespace.go:98] [tsf-keycloak] adding namespace I0514 23:20:57.617432 3663 namespace.go:98] [konflux-operator] adding namespace I0514 23:20:57.617533 3663 namespace.go:98] [rhtpa-operator] adding namespace I0514 23:20:57.617639 3663 namespace.go:98] [tsf-tpa] adding namespace I0514 23:20:57.617711 3663 namespace.go:98] [konflux-ui] adding namespace I0514 23:20:57.617871 3663 namespace.go:98] [rhbk-operator] adding namespace I0514 23:20:57.618424 3663 namespace.go:102] [tsf-keycloak] adding namespace took 1.123302ms I0514 23:20:57.619136 3663 namespace.go:102] [konflux-operator] adding namespace took 1.683509ms I0514 23:20:57.619341 3663 namespace.go:98] [tsf-tas] adding namespace I0514 23:20:57.619730 3663 namespace.go:102] [rhtpa-operator] adding namespace took 2.181273ms I0514 23:20:57.619756 3663 namespace.go:98] [cert-manager-operator] adding namespace I0514 23:20:57.620260 3663 namespace.go:102] [cert-manager-operator] adding namespace took 496.947µs I0514 23:20:57.620848 3663 namespace.go:102] [tsf-tpa] adding namespace took 3.196368ms I0514 23:20:57.621285 3663 namespace.go:102] [konflux-ui] adding namespace took 3.563719ms I0514 23:20:57.621769 3663 namespace.go:102] [rhbk-operator] adding namespace took 3.88072ms I0514 23:20:57.622180 3663 namespace.go:102] [tsf-tas] adding namespace took 2.825244ms I0514 23:20:57.634962 3663 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:20:57.644175 3663 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:20:57.667444 3663 namespace.go:142] [tsf-tpa] updating namespace I0514 23:20:57.667454 3663 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:20:57.667569 3663 namespace.go:142] [konflux-operator] updating namespace I0514 23:20:57.671537 3663 namespace.go:142] [konflux-ui] updating namespace I0514 23:20:57.681258 3663 namespace.go:142] [konflux-operator] updating namespace I0514 23:20:57.683758 3663 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:20:57.683866 3663 namespace.go:142] [rhbk-operator] updating namespace I0514 23:20:57.685855 3663 namespace.go:142] [tsf-tas] updating namespace I0514 23:20:57.688151 3663 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:20:57.713473 3663 namespace.go:142] [tsf-tpa] updating namespace I0514 23:20:57.729366 3663 namespace.go:142] [tsf-tpa] updating namespace I0514 23:20:57.732057 3663 namespace.go:142] [konflux-ui] updating namespace I0514 23:20:57.744208 3663 namespace.go:142] [konflux-ui] updating namespace I0514 23:20:57.750681 3663 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:20:57.766238 3663 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:20:57.769265 3663 namespace.go:142] [rhbk-operator] updating namespace I0514 23:20:57.781137 3663 namespace.go:142] [rhbk-operator] updating namespace I0514 23:20:57.787310 3663 namespace.go:142] [tsf-tas] updating namespace I0514 23:20:57.799208 3663 namespace.go:142] [tsf-tas] updating namespace I0514 23:20:57.804929 3663 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:20:57.818005 3663 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:20:57.834390 3663 namespace.go:142] [konflux-operator] updating namespace I0514 23:20:58.811184 3663 namespace.go:142] [tsf-tpa] updating namespace I0514 23:20:58.820134 3663 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:21:05.004783 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0514 23:21:05.004853 3663 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-l9rdx IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-l9rdx: no pod IPs found I0514 23:21:06.263477 3663 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:06.264113 3663 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:06.264135 3663 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:06.264145 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:06.264167 3663 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:06.264175 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:07.292156 3663 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:07.292175 3663 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:07.292185 3663 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:07.292194 3663 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:10.812464 3663 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0514 23:21:11.204919 3663 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:11.205447 3663 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:11.205475 3663 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:11.205484 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:11.205497 3663 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:11.205502 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:12.233735 3663 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:12.233760 3663 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:12.233792 3663 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:12.233806 3663 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:13.172272 3663 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-nf56j] creating logical port tsf-tpa_rhtpa-operator-controller-manager-586cbb496b-nf56j for pod on switch ip-10-0-0-78.ec2.internal I0514 23:21:13.172402 3663 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-nf56j I0514 23:21:13.188097 3663 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-nf56j] pod update took 15.730837ms I0514 23:21:13.188120 3663 base_network_controller_pods.go:951] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-nf56j] addLogicalPort annotation time took 15.758939ms I0514 23:21:13.189322 3663 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-nf56j] addLogicalPort took 17.067735ms, libovsdb time 905.392µs I0514 23:21:13.530967 3663 cni.go:452] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-nf56j df4d7a194c3aef4be7dac6bce924b10a55484aa74aff4b3aefec572d17f7a86a network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-nf56j df4d7a194c3aef4be7dac6bce924b10a55484aa74aff4b3aefec572d17f7a86a network default NAD default NAD key ] I0514 23:21:13.539960 3663 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-586cbb496b-nf56j, hostIfaceName: df4d7a194c3aef4, network: default, NAD default, SandboxID: "df4d7a194c3aef4be7dac6bce924b10a55484aa74aff4b3aefec572d17f7a86a", PCI device ID: , UID: "4d0e8089-45c9-410f-be22-7b0607edd9a8", MAC: 0a:58:0a:82:00:17, IPs: [10.130.0.23/23] I0514 23:21:13.588413 3663 cni.go:473] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-nf56j df4d7a194c3aef4be7dac6bce924b10a55484aa74aff4b3aefec572d17f7a86a network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-nf56j df4d7a194c3aef4be7dac6bce924b10a55484aa74aff4b3aefec572d17f7a86a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:62:81:63:44:f6\",\"name\":\"df4d7a194c3aef4\"},{\"mac\":\"0a:58:0a:82:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cf68b055-2ea2-4b89-9901-4f38f30b6974\"}],\"ips\":[{\"address\":\"10.130.0.23/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:21:14.237003 3663 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:14.237536 3663 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:14.237566 3663 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:14.237574 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:14.237588 3663 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:14.237594 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:15.254055 3663 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.254646 3663 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.254677 3663 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.254687 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:15.254700 3663 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.254706 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:15.265398 3663 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:15.265424 3663 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:15.265438 3663 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:15.265448 3663 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:15.315732 3663 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.316391 3663 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.316423 3663 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.316432 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:15.316451 3663 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.316456 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:15.340176 3663 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.340705 3663 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.340733 3663 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.340743 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:15.340757 3663 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.340762 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:16.284056 3663 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.284085 3663 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.284102 3663 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.284113 3663 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.342648 3663 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.342683 3663 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.342701 3663 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.342712 3663 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.370572 3663 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.370591 3663 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.370614 3663 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.370631 3663 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:21:16.801025 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0514 23:21:20.491442 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0514 23:21:20.515155 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0514 23:21:20.527134 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0514 23:21:20.557142 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0514 23:21:24.924894 3663 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pxhzq] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-5b66478c58-pxhzq for pod on switch ip-10-0-0-78.ec2.internal I0514 23:21:24.925036 3663 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pxhzq I0514 23:21:24.947411 3663 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pxhzq] pod update took 22.403437ms I0514 23:21:24.947432 3663 base_network_controller_pods.go:951] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pxhzq] addLogicalPort annotation time took 22.430505ms I0514 23:21:24.948704 3663 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pxhzq] addLogicalPort took 23.824916ms, libovsdb time 843.681µs I0514 23:21:25.284405 3663 cni.go:452] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pxhzq 9e7851f0faab911c95e49f933bfc907214a27d092205d0bef99435d1e59a4681 network default NAD default NAD key ] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pxhzq 9e7851f0faab911c95e49f933bfc907214a27d092205d0bef99435d1e59a4681 network default NAD default NAD key ] I0514 23:21:25.293936 3663 helper_linux.go:556] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-5b66478c58-pxhzq, hostIfaceName: 9e7851f0faab911, network: default, NAD default, SandboxID: "9e7851f0faab911c95e49f933bfc907214a27d092205d0bef99435d1e59a4681", PCI device ID: , UID: "f481301b-1085-4e98-9f57-ab8d163eac44", MAC: 0a:58:0a:82:00:18, IPs: [10.130.0.24/23] I0514 23:21:25.345358 3663 cni.go:473] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pxhzq 9e7851f0faab911c95e49f933bfc907214a27d092205d0bef99435d1e59a4681 network default NAD default NAD key default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-pxhzq 9e7851f0faab911c95e49f933bfc907214a27d092205d0bef99435d1e59a4681 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:08:de:81:b9:f1\",\"name\":\"9e7851f0faab911\"},{\"mac\":\"0a:58:0a:82:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c88618f7-6118-4344-becb-01c0f90eae93\"}],\"ips\":[{\"address\":\"10.130.0.24/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:21:27.810453 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0514 23:21:29.131875 3663 namespace.go:98] [cert-manager] adding namespace I0514 23:21:29.132779 3663 namespace.go:102] [cert-manager] adding namespace took 872.961µs I0514 23:21:29.150871 3663 namespace.go:142] [cert-manager] updating namespace I0514 23:21:29.206882 3663 namespace.go:142] [cert-manager] updating namespace I0514 23:21:29.219525 3663 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-69889d554d-sjfxn] creating logical port rhbk-operator_rhbk-operator-69889d554d-sjfxn for pod on switch ip-10-0-0-78.ec2.internal I0514 23:21:29.219625 3663 kube.go:256] Updating pod rhbk-operator/rhbk-operator-69889d554d-sjfxn I0514 23:21:29.252214 3663 pod.go:62] [rhbk-operator/rhbk-operator-69889d554d-sjfxn] pod update took 32.614551ms I0514 23:21:29.252235 3663 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-69889d554d-sjfxn] addLogicalPort annotation time took 32.640992ms I0514 23:21:29.253538 3663 pods.go:271] [rhbk-operator/rhbk-operator-69889d554d-sjfxn] addLogicalPort took 34.024087ms, libovsdb time 909.423µs I0514 23:21:29.582666 3663 cni.go:452] [rhbk-operator/rhbk-operator-69889d554d-sjfxn aba799efd11be0b13d3fee24053c802b8e4699215510ae09d9e0be31a7fb1e83 network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-69889d554d-sjfxn aba799efd11be0b13d3fee24053c802b8e4699215510ae09d9e0be31a7fb1e83 network default NAD default NAD key ] I0514 23:21:29.592532 3663 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-69889d554d-sjfxn, hostIfaceName: aba799efd11be0b, network: default, NAD default, SandboxID: "aba799efd11be0b13d3fee24053c802b8e4699215510ae09d9e0be31a7fb1e83", PCI device ID: , UID: "2bd4101b-2cda-43c8-a010-2b1bcbf5fe34", MAC: 0a:58:0a:82:00:19, IPs: [10.130.0.25/23] I0514 23:21:29.640743 3663 cni.go:473] [rhbk-operator/rhbk-operator-69889d554d-sjfxn aba799efd11be0b13d3fee24053c802b8e4699215510ae09d9e0be31a7fb1e83 network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-69889d554d-sjfxn aba799efd11be0b13d3fee24053c802b8e4699215510ae09d9e0be31a7fb1e83 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:9d:97:a6:d1:18\",\"name\":\"aba799efd11be0b\"},{\"mac\":\"0a:58:0a:82:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e380b476-95e3-4ca5-9975-4a3928ae7b75\"}],\"ips\":[{\"address\":\"10.130.0.25/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:21:32.853321 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 W0514 23:21:33.242746 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0514 23:21:33.242763 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0514 23:21:34.589309 3663 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5d99449ffd-nwf4f] creating logical port cert-manager_cert-manager-cainjector-5d99449ffd-nwf4f for pod on switch ip-10-0-0-78.ec2.internal I0514 23:21:34.589441 3663 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5d99449ffd-nwf4f I0514 23:21:34.608097 3663 pod.go:62] [cert-manager/cert-manager-cainjector-5d99449ffd-nwf4f] pod update took 18.677568ms I0514 23:21:34.608124 3663 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-cainjector-5d99449ffd-nwf4f] addLogicalPort annotation time took 18.711294ms I0514 23:21:34.609699 3663 pods.go:271] [cert-manager/cert-manager-cainjector-5d99449ffd-nwf4f] addLogicalPort took 20.416821ms, libovsdb time 1.115379ms I0514 23:21:34.976730 3663 cni.go:452] [cert-manager/cert-manager-cainjector-5d99449ffd-nwf4f d3b94f1aa82fbc6075146112ecfa12fb395f6e6bdd25d9a8f6e720eca5b54066 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-nwf4f d3b94f1aa82fbc6075146112ecfa12fb395f6e6bdd25d9a8f6e720eca5b54066 network default NAD default NAD key ] I0514 23:21:34.988517 3663 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5d99449ffd-nwf4f, hostIfaceName: d3b94f1aa82fbc6, network: default, NAD default, SandboxID: "d3b94f1aa82fbc6075146112ecfa12fb395f6e6bdd25d9a8f6e720eca5b54066", PCI device ID: , UID: "55f8825d-ca8e-4c18-accb-a5ee85dcf9c6", MAC: 0a:58:0a:82:00:1a, IPs: [10.130.0.26/23] I0514 23:21:35.038206 3663 cni.go:473] [cert-manager/cert-manager-cainjector-5d99449ffd-nwf4f d3b94f1aa82fbc6075146112ecfa12fb395f6e6bdd25d9a8f6e720eca5b54066 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-nwf4f d3b94f1aa82fbc6075146112ecfa12fb395f6e6bdd25d9a8f6e720eca5b54066 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:14:6d:ce:b2:72\",\"name\":\"d3b94f1aa82fbc6\"},{\"mac\":\"0a:58:0a:82:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7ff3995f-8867-46b6-af84-2f593f1eef95\"}],\"ips\":[{\"address\":\"10.130.0.26/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:21:38.684188 3663 namespace.go:142] [openshift-operators] updating namespace I0514 23:21:38.703055 3663 namespace.go:142] [openshift-operators] updating namespace I0514 23:21:41.803569 3663 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-5bf6d87b75-dbg8w] creating logical port konflux-operator_konflux-operator-controller-manager-5bf6d87b75-dbg8w for pod on switch ip-10-0-0-78.ec2.internal I0514 23:21:41.803687 3663 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-5bf6d87b75-dbg8w I0514 23:21:41.828321 3663 pod.go:62] [konflux-operator/konflux-operator-controller-manager-5bf6d87b75-dbg8w] pod update took 24.660724ms I0514 23:21:41.828347 3663 base_network_controller_pods.go:951] [default/konflux-operator/konflux-operator-controller-manager-5bf6d87b75-dbg8w] addLogicalPort annotation time took 24.690047ms I0514 23:21:41.829634 3663 pods.go:271] [konflux-operator/konflux-operator-controller-manager-5bf6d87b75-dbg8w] addLogicalPort took 26.090388ms, libovsdb time 930.357µs I0514 23:21:42.155613 3663 cni.go:452] [konflux-operator/konflux-operator-controller-manager-5bf6d87b75-dbg8w ec912466abfc052eb1e7ac1dc5780211a2f29c0a9239c3cf84e928face411aed network default NAD default NAD key ] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-5bf6d87b75-dbg8w ec912466abfc052eb1e7ac1dc5780211a2f29c0a9239c3cf84e928face411aed network default NAD default NAD key ] I0514 23:21:42.165461 3663 helper_linux.go:556] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-5bf6d87b75-dbg8w, hostIfaceName: ec912466abfc052, network: default, NAD default, SandboxID: "ec912466abfc052eb1e7ac1dc5780211a2f29c0a9239c3cf84e928face411aed", PCI device ID: , UID: "54a17178-6333-43b3-9313-bb46c7137958", MAC: 0a:58:0a:82:00:1b, IPs: [10.130.0.27/23] I0514 23:21:42.217633 3663 cni.go:473] [konflux-operator/konflux-operator-controller-manager-5bf6d87b75-dbg8w ec912466abfc052eb1e7ac1dc5780211a2f29c0a9239c3cf84e928face411aed network default NAD default NAD key default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-5bf6d87b75-dbg8w ec912466abfc052eb1e7ac1dc5780211a2f29c0a9239c3cf84e928face411aed network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:29:a0:0d:a8:d2\",\"name\":\"ec912466abfc052\"},{\"mac\":\"0a:58:0a:82:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7c2e5ec7-aca3-4644-837e-0b38385f1ec0\"}],\"ips\":[{\"address\":\"10.130.0.27/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:21:44.967583 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0514 23:21:47.318367 3663 namespace.go:98] [trusted-artifact-signer] adding namespace I0514 23:21:47.319248 3663 namespace.go:102] [trusted-artifact-signer] adding namespace took 856.076µs W0514 23:21:47.350471 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0514 23:21:47.376853 3663 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:21:47.396300 3663 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:21:51.474204 3663 namespace.go:142] [openshift-operators] updating namespace I0514 23:21:52.434800 3663 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-5864bd5898-2gmsq] creating logical port openshift-operators_tekton-operator-webhook-5864bd5898-2gmsq for pod on switch ip-10-0-0-78.ec2.internal I0514 23:21:52.434941 3663 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-5864bd5898-2gmsq I0514 23:21:52.450399 3663 pod.go:62] [openshift-operators/tekton-operator-webhook-5864bd5898-2gmsq] pod update took 15.48346ms I0514 23:21:52.450418 3663 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-5864bd5898-2gmsq] addLogicalPort annotation time took 15.504786ms I0514 23:21:52.451610 3663 pods.go:271] [openshift-operators/tekton-operator-webhook-5864bd5898-2gmsq] addLogicalPort took 16.82424ms, libovsdb time 842.059µs I0514 23:21:54.641610 3663 cni.go:452] [openshift-operators/tekton-operator-webhook-5864bd5898-2gmsq 716585293d766ab4dfe4800de216d770f80e50f36cf31d734659fe999bfb0415 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-2gmsq 716585293d766ab4dfe4800de216d770f80e50f36cf31d734659fe999bfb0415 network default NAD default NAD key ] I0514 23:21:54.651621 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-5864bd5898-2gmsq, hostIfaceName: 716585293d766ab, network: default, NAD default, SandboxID: "716585293d766ab4dfe4800de216d770f80e50f36cf31d734659fe999bfb0415", PCI device ID: , UID: "3dd2d1a1-6a56-4493-a02d-1f5472e91d3e", MAC: 0a:58:0a:82:00:1c, IPs: [10.130.0.28/23] I0514 23:21:54.701042 3663 cni.go:473] [openshift-operators/tekton-operator-webhook-5864bd5898-2gmsq 716585293d766ab4dfe4800de216d770f80e50f36cf31d734659fe999bfb0415 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-2gmsq 716585293d766ab4dfe4800de216d770f80e50f36cf31d734659fe999bfb0415 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:04:ed:23:52:13\",\"name\":\"716585293d766ab\"},{\"mac\":\"0a:58:0a:82:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/70ba4ce1-8b30-41b0-8bac-7270b6d168de\"}],\"ips\":[{\"address\":\"10.130.0.28/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:21:55.853575 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 W0514 23:21:55.948225 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0514 23:22:03.839994 3663 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:22:03.840575 3663 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:22:03.876543 3663 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:22:03.876565 3663 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:22:03.894745 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0514 23:22:03.894774 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0514 23:22:08.863386 3663 namespace.go:98] [openshift-pipelines] adding namespace I0514 23:22:08.864223 3663 namespace.go:102] [openshift-pipelines] adding namespace took 813.057µs I0514 23:22:08.876773 3663 namespace.go:142] [openshift-pipelines] updating namespace I0514 23:22:09.701495 3663 namespace.go:142] [cert-manager] updating namespace I0514 23:22:09.900999 3663 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:22:10.304130 3663 namespace.go:142] [konflux-operator] updating namespace I0514 23:22:10.503694 3663 namespace.go:142] [konflux-ui] updating namespace I0514 23:22:10.704379 3663 namespace.go:142] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] updating namespace I0514 23:22:11.107124 3663 namespace.go:142] [rhbk-operator] updating namespace I0514 23:22:11.306902 3663 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:22:11.509789 3663 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:22:11.711013 3663 namespace.go:142] [tsf] updating namespace I0514 23:22:11.911561 3663 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:22:12.112346 3663 namespace.go:142] [tsf-tas] updating namespace I0514 23:22:12.312666 3663 namespace.go:142] [tsf-tpa] updating namespace I0514 23:22:12.450359 3663 namespace.go:142] [cert-manager] updating namespace I0514 23:22:12.459852 3663 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:22:12.470289 3663 namespace.go:142] [default] updating namespace I0514 23:22:12.480578 3663 namespace.go:142] [konflux-operator] updating namespace I0514 23:22:12.490903 3663 namespace.go:142] [konflux-ui] updating namespace I0514 23:22:12.502641 3663 namespace.go:142] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] updating namespace I0514 23:22:12.512153 3663 namespace.go:142] [openshift] updating namespace I0514 23:22:12.522551 3663 namespace.go:142] [rhbk-operator] updating namespace I0514 23:22:12.534337 3663 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:22:12.543480 3663 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:22:12.553357 3663 namespace.go:142] [tsf] updating namespace I0514 23:22:12.564647 3663 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:22:12.576198 3663 namespace.go:142] [tsf-tas] updating namespace I0514 23:22:12.595287 3663 namespace.go:142] [tsf-tpa] updating namespace I0514 23:22:12.697707 3663 namespace.go:142] [cert-manager] updating namespace I0514 23:22:12.799937 3663 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:22:12.898535 3663 namespace.go:142] [default] updating namespace I0514 23:22:13.001738 3663 namespace.go:142] [konflux-operator] updating namespace I0514 23:22:13.095995 3663 namespace.go:142] [konflux-ui] updating namespace I0514 23:22:13.200604 3663 namespace.go:142] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] updating namespace I0514 23:22:13.295340 3663 namespace.go:142] [openshift] updating namespace I0514 23:22:13.396914 3663 namespace.go:142] [rhbk-operator] updating namespace I0514 23:22:13.496305 3663 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:22:13.595607 3663 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:22:13.695032 3663 namespace.go:142] [tsf] updating namespace I0514 23:22:13.795735 3663 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:22:13.894795 3663 namespace.go:142] [tsf-tas] updating namespace I0514 23:22:13.997837 3663 namespace.go:142] [tsf-tpa] updating namespace I0514 23:22:14.255017 3663 namespace.go:142] [openshift-pipelines] updating namespace W0514 23:22:18.460463 3663 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0514 23:22:18.488340 3663 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0514 23:22:21.674409 3663 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-sc9ds] creating logical port openshift-pipelines_tekton-pipelines-controller-86bd8d9bfc-sc9ds for pod on switch ip-10-0-0-78.ec2.internal I0514 23:22:21.674515 3663 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-sc9ds I0514 23:22:21.689281 3663 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-sc9ds] pod update took 14.791749ms I0514 23:22:21.689301 3663 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-sc9ds] addLogicalPort annotation time took 14.81802ms I0514 23:22:21.690705 3663 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-sc9ds] addLogicalPort took 16.317864ms, libovsdb time 1.002408ms I0514 23:22:21.769843 3663 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-lzq6b] creating logical port openshift-pipelines_tekton-events-controller-57d7f78b8d-lzq6b for pod on switch ip-10-0-0-78.ec2.internal I0514 23:22:21.769934 3663 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-57d7f78b8d-lzq6b I0514 23:22:21.786491 3663 pod.go:62] [openshift-pipelines/tekton-events-controller-57d7f78b8d-lzq6b] pod update took 16.581537ms I0514 23:22:21.786551 3663 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-lzq6b] addLogicalPort annotation time took 16.644192ms I0514 23:22:21.787764 3663 pods.go:271] [openshift-pipelines/tekton-events-controller-57d7f78b8d-lzq6b] addLogicalPort took 17.960423ms, libovsdb time 838.585µs I0514 23:22:22.044255 3663 cni.go:452] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-sc9ds f6af56d6c8b52ff3828c67015431273abf79c9a45dd32ec32ded682022542e70 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-sc9ds f6af56d6c8b52ff3828c67015431273abf79c9a45dd32ec32ded682022542e70 network default NAD default NAD key ] I0514 23:22:22.070763 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-86bd8d9bfc-sc9ds, hostIfaceName: f6af56d6c8b52ff, network: default, NAD default, SandboxID: "f6af56d6c8b52ff3828c67015431273abf79c9a45dd32ec32ded682022542e70", PCI device ID: , UID: "f8cf65e6-051f-48d0-a5b1-59cddd30e207", MAC: 0a:58:0a:82:00:1d, IPs: [10.130.0.29/23] I0514 23:22:22.122189 3663 cni.go:473] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-sc9ds f6af56d6c8b52ff3828c67015431273abf79c9a45dd32ec32ded682022542e70 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-sc9ds f6af56d6c8b52ff3828c67015431273abf79c9a45dd32ec32ded682022542e70 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:8f:c4:dd:d0:8a\",\"name\":\"f6af56d6c8b52ff\"},{\"mac\":\"0a:58:0a:82:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9b2c2c2c-f859-4263-826f-b8ad19fb9f09\"}],\"ips\":[{\"address\":\"10.130.0.29/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:22:22.131170 3663 cni.go:452] [openshift-pipelines/tekton-events-controller-57d7f78b8d-lzq6b 69a4d640c8d1dba75bee13e8e2d1634db3b29f89adf35f512560e35913fa42c6 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-lzq6b 69a4d640c8d1dba75bee13e8e2d1634db3b29f89adf35f512560e35913fa42c6 network default NAD default NAD key ] I0514 23:22:22.139980 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-57d7f78b8d-lzq6b, hostIfaceName: 69a4d640c8d1dba, network: default, NAD default, SandboxID: "69a4d640c8d1dba75bee13e8e2d1634db3b29f89adf35f512560e35913fa42c6", PCI device ID: , UID: "2666277d-3b31-4363-8355-92c0d3c226e7", MAC: 0a:58:0a:82:00:1e, IPs: [10.130.0.30/23] I0514 23:22:22.189619 3663 cni.go:473] [openshift-pipelines/tekton-events-controller-57d7f78b8d-lzq6b 69a4d640c8d1dba75bee13e8e2d1634db3b29f89adf35f512560e35913fa42c6 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-lzq6b 69a4d640c8d1dba75bee13e8e2d1634db3b29f89adf35f512560e35913fa42c6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:24:69:00:6e:05\",\"name\":\"69a4d640c8d1dba\"},{\"mac\":\"0a:58:0a:82:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1ae036a2-cda8-474a-b8d9-7feb27f61e40\"}],\"ips\":[{\"address\":\"10.130.0.30/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:22:26.150424 3663 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0514 23:22:26.150447 3663 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0514 23:22:26.150452 3663 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0514 23:22:26.190707 3663 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0514 23:22:26.190726 3663 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0514 23:22:26.190732 3663 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0514 23:22:26.533326 3663 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:26.533346 3663 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:26.533351 3663 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:26.533355 3663 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0514 23:22:32.839797 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 W0514 23:22:37.146026 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0514 23:22:37.146046 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0514 23:22:37.146051 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0514 23:22:37.173788 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0514 23:22:37.173805 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0514 23:22:37.173810 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0514 23:22:42.731786 3663 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:22:42.732355 3663 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:22:42.751494 3663 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:22:42.751511 3663 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:22:42.759061 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0514 23:22:42.759085 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0514 23:22:44.643012 3663 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-785c845664-rgsn6] creating logical port openshift-pipelines_tekton-triggers-controller-785c845664-rgsn6 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:22:44.643110 3663 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-785c845664-rgsn6 I0514 23:22:44.656636 3663 pod.go:62] [openshift-pipelines/tekton-triggers-controller-785c845664-rgsn6] pod update took 13.554348ms I0514 23:22:44.656661 3663 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-controller-785c845664-rgsn6] addLogicalPort annotation time took 13.584323ms I0514 23:22:44.658051 3663 pods.go:271] [openshift-pipelines/tekton-triggers-controller-785c845664-rgsn6] addLogicalPort took 15.054121ms, libovsdb time 984.623µs I0514 23:22:44.805406 3663 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0514 23:22:44.806121 3663 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0514 23:22:44.806149 3663 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0514 23:22:44.808662 3663 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0514 23:22:45.002993 3663 cni.go:452] [openshift-pipelines/tekton-triggers-controller-785c845664-rgsn6 cc9c269989e1a931965ca4e1127a93dee45553bce41a0ec7f26912e067cf34c9 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-785c845664-rgsn6 cc9c269989e1a931965ca4e1127a93dee45553bce41a0ec7f26912e067cf34c9 network default NAD default NAD key ] I0514 23:22:45.012009 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-785c845664-rgsn6, hostIfaceName: cc9c269989e1a93, network: default, NAD default, SandboxID: "cc9c269989e1a931965ca4e1127a93dee45553bce41a0ec7f26912e067cf34c9", PCI device ID: , UID: "b2d43876-850f-407b-91ce-6e7670f390ce", MAC: 0a:58:0a:82:00:1f, IPs: [10.130.0.31/23] I0514 23:22:45.061624 3663 cni.go:473] [openshift-pipelines/tekton-triggers-controller-785c845664-rgsn6 cc9c269989e1a931965ca4e1127a93dee45553bce41a0ec7f26912e067cf34c9 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-785c845664-rgsn6 cc9c269989e1a931965ca4e1127a93dee45553bce41a0ec7f26912e067cf34c9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:ff:aa:cd:b4:9a\",\"name\":\"cc9c269989e1a93\"},{\"mac\":\"0a:58:0a:82:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3212bf4e-3317-43c1-9ddb-a6a2c524741e\"}],\"ips\":[{\"address\":\"10.130.0.31/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:22:47.626146 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0514 23:22:54.818271 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 W0514 23:22:55.701811 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0514 23:22:55.701854 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0514 23:22:55.701928 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0514 23:22:55.701941 3663 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service I0514 23:23:00.582287 3663 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-55fb698565-kpp4t] creating logical port openshift-pipelines_tekton-chains-controller-55fb698565-kpp4t for pod on switch ip-10-0-0-78.ec2.internal I0514 23:23:00.582404 3663 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-55fb698565-kpp4t I0514 23:23:00.597469 3663 pod.go:62] [openshift-pipelines/tekton-chains-controller-55fb698565-kpp4t] pod update took 15.086665ms I0514 23:23:00.597558 3663 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-chains-controller-55fb698565-kpp4t] addLogicalPort annotation time took 15.179747ms I0514 23:23:00.598891 3663 pods.go:271] [openshift-pipelines/tekton-chains-controller-55fb698565-kpp4t] addLogicalPort took 16.618547ms, libovsdb time 930.067µs I0514 23:23:00.950083 3663 cni.go:452] [openshift-pipelines/tekton-chains-controller-55fb698565-kpp4t efb3fb341b5f1ce511fa4756bfde8ec60e4e38f348365d6f4a079e907c7f096b network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-kpp4t efb3fb341b5f1ce511fa4756bfde8ec60e4e38f348365d6f4a079e907c7f096b network default NAD default NAD key ] I0514 23:23:00.959089 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-55fb698565-kpp4t, hostIfaceName: efb3fb341b5f1ce, network: default, NAD default, SandboxID: "efb3fb341b5f1ce511fa4756bfde8ec60e4e38f348365d6f4a079e907c7f096b", PCI device ID: , UID: "2c7888f8-0663-4d34-938a-0b12011c9d3f", MAC: 0a:58:0a:82:00:20, IPs: [10.130.0.32/23] I0514 23:23:01.010214 3663 cni.go:473] [openshift-pipelines/tekton-chains-controller-55fb698565-kpp4t efb3fb341b5f1ce511fa4756bfde8ec60e4e38f348365d6f4a079e907c7f096b network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-kpp4t efb3fb341b5f1ce511fa4756bfde8ec60e4e38f348365d6f4a079e907c7f096b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:67:e2:86:95:b1\",\"name\":\"efb3fb341b5f1ce\"},{\"mac\":\"0a:58:0a:82:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/da10d54c-d13e-46ff-8930-4055835cc950\"}],\"ips\":[{\"address\":\"10.130.0.32/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:23:05.319717 3663 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0514 23:23:05.848476 3663 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0514 23:23:15.723535 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=100 I0514 23:23:20.384208 3663 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-zqwff] creating logical port openshift-pipelines_tekton-results-watcher-7777c5dbd5-zqwff for pod on switch ip-10-0-0-78.ec2.internal I0514 23:23:20.384320 3663 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-7777c5dbd5-zqwff I0514 23:23:20.400037 3663 pod.go:62] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-zqwff] pod update took 15.746465ms I0514 23:23:20.400067 3663 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-zqwff] addLogicalPort annotation time took 15.781459ms I0514 23:23:20.402098 3663 pods.go:271] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-zqwff] addLogicalPort took 17.910948ms, libovsdb time 1.525132ms I0514 23:23:20.750928 3663 cni.go:452] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-zqwff bdecc078be57092815937093479294bc9a9e96df8cefe42c0d78f8515dbca874 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-zqwff bdecc078be57092815937093479294bc9a9e96df8cefe42c0d78f8515dbca874 network default NAD default NAD key ] I0514 23:23:20.760398 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-7777c5dbd5-zqwff, hostIfaceName: bdecc078be57092, network: default, NAD default, SandboxID: "bdecc078be57092815937093479294bc9a9e96df8cefe42c0d78f8515dbca874", PCI device ID: , UID: "230b6cee-a6bd-414a-a033-dcb23fecc702", MAC: 0a:58:0a:82:00:21, IPs: [10.130.0.33/23] I0514 23:23:20.810601 3663 cni.go:473] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-zqwff bdecc078be57092815937093479294bc9a9e96df8cefe42c0d78f8515dbca874 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-zqwff bdecc078be57092815937093479294bc9a9e96df8cefe42c0d78f8515dbca874 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:a6:c3:6f:9b:89\",\"name\":\"bdecc078be57092\"},{\"mac\":\"0a:58:0a:82:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/80769f8c-49bd-444e-8d27-2618319a4fd2\"}],\"ips\":[{\"address\":\"10.130.0.33/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:23:25.724371 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=199 W0514 23:23:25.838964 3663 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0514 23:23:25.838985 3663 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0514 23:23:25.838992 3663 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0514 23:23:31.724790 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=102 I0514 23:23:35.096329 3663 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:35.096979 3663 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:35.141054 3663 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:35.141073 3663 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:35.147918 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0514 23:23:35.147944 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0514 23:23:36.648556 3663 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-5nwdx] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-5nwdx for pod on switch ip-10-0-0-78.ec2.internal I0514 23:23:36.648658 3663 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-5nwdx I0514 23:23:36.658526 3663 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-5nwdx] pod update took 9.894673ms I0514 23:23:36.658555 3663 base_network_controller_pods.go:951] [default/openshift-pipelines/tkn-cli-serve-784b566968-5nwdx] addLogicalPort annotation time took 9.928225ms I0514 23:23:36.659915 3663 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-5nwdx] addLogicalPort took 11.379557ms, libovsdb time 949.5µs I0514 23:23:37.010418 3663 cni.go:452] [openshift-pipelines/tkn-cli-serve-784b566968-5nwdx 4f524280e12272b042d386d5e8f3010f569d3a956b6bf2c05ec1b76364af7b4f network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-5nwdx 4f524280e12272b042d386d5e8f3010f569d3a956b6bf2c05ec1b76364af7b4f network default NAD default NAD key ] I0514 23:23:37.020232 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-5nwdx, hostIfaceName: 4f524280e12272b, network: default, NAD default, SandboxID: "4f524280e12272b042d386d5e8f3010f569d3a956b6bf2c05ec1b76364af7b4f", PCI device ID: , UID: "9f166eb8-f912-4d80-9981-e228b4bcbabd", MAC: 0a:58:0a:82:00:22, IPs: [10.130.0.34/23] I0514 23:23:37.050605 3663 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-4d45n-nzds2] creating logical port tsf-tas_segment-backup-installation-4d45n-nzds2 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:23:37.050690 3663 kube.go:256] Updating pod tsf-tas/segment-backup-installation-4d45n-nzds2 I0514 23:23:37.066532 3663 pod.go:62] [tsf-tas/segment-backup-installation-4d45n-nzds2] pod update took 15.862746ms I0514 23:23:37.066555 3663 base_network_controller_pods.go:951] [default/tsf-tas/segment-backup-installation-4d45n-nzds2] addLogicalPort annotation time took 15.891263ms I0514 23:23:37.067872 3663 pods.go:271] [tsf-tas/segment-backup-installation-4d45n-nzds2] addLogicalPort took 17.284796ms, libovsdb time 880.279µs I0514 23:23:37.075564 3663 cni.go:473] [openshift-pipelines/tkn-cli-serve-784b566968-5nwdx 4f524280e12272b042d386d5e8f3010f569d3a956b6bf2c05ec1b76364af7b4f network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-5nwdx 4f524280e12272b042d386d5e8f3010f569d3a956b6bf2c05ec1b76364af7b4f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:f1:53:ee:dc:8b\",\"name\":\"4f524280e12272b\"},{\"mac\":\"0a:58:0a:82:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d60b6cd9-1c3d-40d4-bac9-93aa75764d7c\"}],\"ips\":[{\"address\":\"10.130.0.34/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:23:37.412596 3663 cni.go:452] [tsf-tas/segment-backup-installation-4d45n-nzds2 9a6f1871f969b3208dc3206da3487746fd7b700e886e19b45a0cab9f27ed7cac network default NAD default NAD key ] ADD starting CNI request [tsf-tas/segment-backup-installation-4d45n-nzds2 9a6f1871f969b3208dc3206da3487746fd7b700e886e19b45a0cab9f27ed7cac network default NAD default NAD key ] I0514 23:23:37.421938 3663 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-4d45n-nzds2, hostIfaceName: 9a6f1871f969b32, network: default, NAD default, SandboxID: "9a6f1871f969b3208dc3206da3487746fd7b700e886e19b45a0cab9f27ed7cac", PCI device ID: , UID: "9831afce-58c2-42c7-b458-7264ad30c8b4", MAC: 0a:58:0a:82:00:23, IPs: [10.130.0.35/23] I0514 23:23:37.473174 3663 cni.go:473] [tsf-tas/segment-backup-installation-4d45n-nzds2 9a6f1871f969b3208dc3206da3487746fd7b700e886e19b45a0cab9f27ed7cac network default NAD default NAD key default] ADD finished CNI request [tsf-tas/segment-backup-installation-4d45n-nzds2 9a6f1871f969b3208dc3206da3487746fd7b700e886e19b45a0cab9f27ed7cac network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:70:aa:0a:47:7d\",\"name\":\"9a6f1871f969b32\"},{\"mac\":\"0a:58:0a:82:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f2dbf0fa-ef91-4ee1-9774-7912aeda6fae\"}],\"ips\":[{\"address\":\"10.130.0.35/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:23:37.812793 3663 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-tsvkq-4kksv] creating logical port tsf-tas_ctlog-createtree-job-tsvkq-4kksv for pod on switch ip-10-0-0-78.ec2.internal I0514 23:23:37.812947 3663 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-tsvkq-4kksv I0514 23:23:37.826260 3663 pod.go:62] [tsf-tas/ctlog-createtree-job-tsvkq-4kksv] pod update took 13.348822ms I0514 23:23:37.826294 3663 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-createtree-job-tsvkq-4kksv] addLogicalPort annotation time took 13.389135ms I0514 23:23:37.828318 3663 pods.go:271] [tsf-tas/ctlog-createtree-job-tsvkq-4kksv] addLogicalPort took 15.544654ms, libovsdb time 1.44638ms I0514 23:23:38.129933 3663 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-89675b98-rm9rk] creating logical port tsf-tas_trillian-logsigner-89675b98-rm9rk for pod on switch ip-10-0-0-78.ec2.internal I0514 23:23:38.130053 3663 kube.go:256] Updating pod tsf-tas/trillian-logsigner-89675b98-rm9rk I0514 23:23:38.152847 3663 pod.go:62] [tsf-tas/trillian-logsigner-89675b98-rm9rk] pod update took 22.830126ms I0514 23:23:38.152876 3663 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logsigner-89675b98-rm9rk] addLogicalPort annotation time took 22.86385ms I0514 23:23:38.154278 3663 pods.go:271] [tsf-tas/trillian-logsigner-89675b98-rm9rk] addLogicalPort took 24.367473ms, libovsdb time 894.256µs I0514 23:23:38.206678 3663 cni.go:452] [tsf-tas/ctlog-createtree-job-tsvkq-4kksv c4cc52815316e741941fc085ed109677bb61add613d01eb8e9beaa6e822d35a7 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-createtree-job-tsvkq-4kksv c4cc52815316e741941fc085ed109677bb61add613d01eb8e9beaa6e822d35a7 network default NAD default NAD key ] I0514 23:23:38.217347 3663 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-tsvkq-4kksv, hostIfaceName: c4cc52815316e74, network: default, NAD default, SandboxID: "c4cc52815316e741941fc085ed109677bb61add613d01eb8e9beaa6e822d35a7", PCI device ID: , UID: "90da4645-64b7-45bd-94b1-e85fe7501f82", MAC: 0a:58:0a:82:00:24, IPs: [10.130.0.36/23] I0514 23:23:38.278234 3663 cni.go:473] [tsf-tas/ctlog-createtree-job-tsvkq-4kksv c4cc52815316e741941fc085ed109677bb61add613d01eb8e9beaa6e822d35a7 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-tsvkq-4kksv c4cc52815316e741941fc085ed109677bb61add613d01eb8e9beaa6e822d35a7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:7a:d7:c6:ba:48\",\"name\":\"c4cc52815316e74\"},{\"mac\":\"0a:58:0a:82:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fb1dd82e-6d69-4c2a-917c-fce06afc2f24\"}],\"ips\":[{\"address\":\"10.130.0.36/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:23:38.818883 3663 cni.go:452] [tsf-tas/trillian-logsigner-89675b98-rm9rk 1fd772d2d8e963f006bb7c1cf347ae009d55f85815a86430e37933f9553e1103 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logsigner-89675b98-rm9rk 1fd772d2d8e963f006bb7c1cf347ae009d55f85815a86430e37933f9553e1103 network default NAD default NAD key ] I0514 23:23:38.825973 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0514 23:23:38.830952 3663 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-89675b98-rm9rk, hostIfaceName: 1fd772d2d8e963f, network: default, NAD default, SandboxID: "1fd772d2d8e963f006bb7c1cf347ae009d55f85815a86430e37933f9553e1103", PCI device ID: , UID: "d03b2b12-1802-48fc-b4c5-faf04300c258", MAC: 0a:58:0a:82:00:25, IPs: [10.130.0.37/23] I0514 23:23:38.900701 3663 cni.go:473] [tsf-tas/trillian-logsigner-89675b98-rm9rk 1fd772d2d8e963f006bb7c1cf347ae009d55f85815a86430e37933f9553e1103 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logsigner-89675b98-rm9rk 1fd772d2d8e963f006bb7c1cf347ae009d55f85815a86430e37933f9553e1103 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:81:b8:fb:c6:39\",\"name\":\"1fd772d2d8e963f\"},{\"mac\":\"0a:58:0a:82:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ba89a36c-4642-4a1e-9a44-620582968377\"}],\"ips\":[{\"address\":\"10.130.0.37/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:23:40.890438 3663 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0514 23:23:40.890533 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server I0514 23:23:43.731977 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=846 W0514 23:23:46.035897 3663 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0514 23:23:46.163672 3663 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-6ccvn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:46.164332 3663 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-6ccvn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:47.068381 3663 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-6ccvn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:47.068504 3663 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-6ccvn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:23:53.959868 3663 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0514 23:23:55.613149 3663 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0514 23:23:57.214263 3663 namespace.go:142] [openshift-pipelines] updating namespace I0514 23:23:58.609226 3663 cni.go:452] [tsf-tas/segment-backup-installation-4d45n-nzds2 9a6f1871f969b3208dc3206da3487746fd7b700e886e19b45a0cab9f27ed7cac network default NAD default NAD key ] DEL starting CNI request [tsf-tas/segment-backup-installation-4d45n-nzds2 9a6f1871f969b3208dc3206da3487746fd7b700e886e19b45a0cab9f27ed7cac network default NAD default NAD key ] I0514 23:23:58.683455 3663 cni.go:473] [tsf-tas/segment-backup-installation-4d45n-nzds2 9a6f1871f969b3208dc3206da3487746fd7b700e886e19b45a0cab9f27ed7cac network default NAD default NAD key default] DEL finished CNI request [tsf-tas/segment-backup-installation-4d45n-nzds2 9a6f1871f969b3208dc3206da3487746fd7b700e886e19b45a0cab9f27ed7cac network default NAD default NAD key default], result "{}", err I0514 23:23:58.736870 3663 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-4d45n-nzds2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:58.736906 3663 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-4d45n-nzds2 I0514 23:23:58.738179 3663 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-4d45n-nzds2, ips: 10.130.0.35 I0514 23:23:58.738207 3663 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-4d45n-nzds2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:59.759909 3663 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-4d45n-nzds2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:59.759932 3663 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-4d45n-nzds2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:24:04.155489 3663 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0514 23:24:10.832602 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0514 23:24:19.813872 3663 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=11 I0514 23:24:24.745421 3663 cni.go:452] [tsf-tas/ctlog-createtree-job-tsvkq-4kksv c4cc52815316e741941fc085ed109677bb61add613d01eb8e9beaa6e822d35a7 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/ctlog-createtree-job-tsvkq-4kksv c4cc52815316e741941fc085ed109677bb61add613d01eb8e9beaa6e822d35a7 network default NAD default NAD key ] I0514 23:24:24.837588 3663 cni.go:473] [tsf-tas/ctlog-createtree-job-tsvkq-4kksv c4cc52815316e741941fc085ed109677bb61add613d01eb8e9beaa6e822d35a7 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-tsvkq-4kksv c4cc52815316e741941fc085ed109677bb61add613d01eb8e9beaa6e822d35a7 network default NAD default NAD key default], result "{}", err I0514 23:24:24.920959 3663 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-tsvkq-4kksv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:24.920987 3663 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-tsvkq-4kksv I0514 23:24:24.922310 3663 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-tsvkq-4kksv, ips: 10.130.0.36 I0514 23:24:24.922338 3663 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-tsvkq-4kksv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0514 23:24:25.702573 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner I0514 23:24:25.765809 3663 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-tsvkq-4kksv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:25.765850 3663 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-tsvkq-4kksv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:27.429333 3663 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:27.429937 3663 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:27.449780 3663 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:27.449802 3663 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:27.456344 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0514 23:24:27.456368 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0514 23:24:28.005976 3663 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-59ff668974-2dtbz] creating logical port tsf-tas_ctlog-59ff668974-2dtbz for pod on switch ip-10-0-0-78.ec2.internal I0514 23:24:28.006108 3663 kube.go:256] Updating pod tsf-tas/ctlog-59ff668974-2dtbz I0514 23:24:28.023661 3663 pod.go:62] [tsf-tas/ctlog-59ff668974-2dtbz] pod update took 17.580714ms I0514 23:24:28.023686 3663 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-59ff668974-2dtbz] addLogicalPort annotation time took 17.611259ms I0514 23:24:28.024896 3663 pods.go:271] [tsf-tas/ctlog-59ff668974-2dtbz] addLogicalPort took 18.941843ms, libovsdb time 818.833µs I0514 23:24:28.171420 3663 base_network_controller_pods.go:487] [default/tsf/patch-tekton-config-z22jk] creating logical port tsf_patch-tekton-config-z22jk for pod on switch ip-10-0-0-78.ec2.internal I0514 23:24:28.171540 3663 kube.go:256] Updating pod tsf/patch-tekton-config-z22jk I0514 23:24:28.184942 3663 pod.go:62] [tsf/patch-tekton-config-z22jk] pod update took 13.437656ms I0514 23:24:28.184968 3663 base_network_controller_pods.go:951] [default/tsf/patch-tekton-config-z22jk] addLogicalPort annotation time took 13.468037ms I0514 23:24:28.186258 3663 pods.go:271] [tsf/patch-tekton-config-z22jk] addLogicalPort took 14.859588ms, libovsdb time 897.05µs I0514 23:24:28.609083 3663 cni.go:452] [tsf-tas/ctlog-59ff668974-2dtbz a733790326012ac7765eebd58154ffa9ad6f2aef20999af86b65843968ba84b2 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-59ff668974-2dtbz a733790326012ac7765eebd58154ffa9ad6f2aef20999af86b65843968ba84b2 network default NAD default NAD key ] I0514 23:24:28.618557 3663 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-59ff668974-2dtbz, hostIfaceName: a733790326012ac, network: default, NAD default, SandboxID: "a733790326012ac7765eebd58154ffa9ad6f2aef20999af86b65843968ba84b2", PCI device ID: , UID: "cb1a2ff4-2de7-4401-b621-2afe8e6ab6f9", MAC: 0a:58:0a:82:00:26, IPs: [10.130.0.38/23] I0514 23:24:28.628995 3663 cni.go:452] [tsf/patch-tekton-config-z22jk 241c61a1d3c360897f76799404207443eba958e4845ae4d892e239e56ec03872 network default NAD default NAD key ] ADD starting CNI request [tsf/patch-tekton-config-z22jk 241c61a1d3c360897f76799404207443eba958e4845ae4d892e239e56ec03872 network default NAD default NAD key ] I0514 23:24:28.638503 3663 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: patch-tekton-config-z22jk, hostIfaceName: 241c61a1d3c3608, network: default, NAD default, SandboxID: "241c61a1d3c360897f76799404207443eba958e4845ae4d892e239e56ec03872", PCI device ID: , UID: "0a7fa79d-51c0-4f44-a62e-5cc6f1144e79", MAC: 0a:58:0a:82:00:27, IPs: [10.130.0.39/23] I0514 23:24:28.672245 3663 cni.go:473] [tsf-tas/ctlog-59ff668974-2dtbz a733790326012ac7765eebd58154ffa9ad6f2aef20999af86b65843968ba84b2 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-59ff668974-2dtbz a733790326012ac7765eebd58154ffa9ad6f2aef20999af86b65843968ba84b2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:7c:78:36:c9:f9\",\"name\":\"a733790326012ac\"},{\"mac\":\"0a:58:0a:82:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/448fc9c1-4449-4f83-8fa9-a55a5271a3f5\"}],\"ips\":[{\"address\":\"10.130.0.38/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:24:28.700393 3663 cni.go:473] [tsf/patch-tekton-config-z22jk 241c61a1d3c360897f76799404207443eba958e4845ae4d892e239e56ec03872 network default NAD default NAD key default] ADD finished CNI request [tsf/patch-tekton-config-z22jk 241c61a1d3c360897f76799404207443eba958e4845ae4d892e239e56ec03872 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:74:c6:cd:e5:0b\",\"name\":\"241c61a1d3c3608\"},{\"mac\":\"0a:58:0a:82:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fdf8bd33-0b31-412d-99eb-1e8623a87ccc\"}],\"ips\":[{\"address\":\"10.130.0.39/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:24:35.807863 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0514 23:24:36.820185 3663 cni.go:452] [tsf/patch-tekton-config-z22jk 241c61a1d3c360897f76799404207443eba958e4845ae4d892e239e56ec03872 network default NAD default NAD key ] DEL starting CNI request [tsf/patch-tekton-config-z22jk 241c61a1d3c360897f76799404207443eba958e4845ae4d892e239e56ec03872 network default NAD default NAD key ] I0514 23:24:36.908732 3663 cni.go:473] [tsf/patch-tekton-config-z22jk 241c61a1d3c360897f76799404207443eba958e4845ae4d892e239e56ec03872 network default NAD default NAD key default] DEL finished CNI request [tsf/patch-tekton-config-z22jk 241c61a1d3c360897f76799404207443eba958e4845ae4d892e239e56ec03872 network default NAD default NAD key default], result "{}", err I0514 23:24:36.957196 3663 obj_retry.go:498] Detected object tsf/patch-tekton-config-z22jk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:36.957223 3663 pods.go:176] Deleting pod: tsf/patch-tekton-config-z22jk I0514 23:24:36.958521 3663 pods.go:236] Attempting to release IPs for pod: tsf/patch-tekton-config-z22jk, ips: 10.130.0.39 I0514 23:24:36.958555 3663 obj_retry.go:498] Detected object tsf/patch-tekton-config-z22jk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:37.979772 3663 obj_retry.go:492] Detected object tsf/patch-tekton-config-z22jk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:37.979794 3663 obj_retry.go:492] Detected object tsf/patch-tekton-config-z22jk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:43.529848 3663 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-q77kq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:43.530408 3663 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-q77kq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:44.360681 3663 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-7p2l9-j2vbg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:44.361216 3663 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-7p2l9-j2vbg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:44.548629 3663 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-q77kq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:44.548657 3663 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-q77kq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:44.582605 3663 obj_retry.go:492] Detected object tsf/patch-tekton-config-z22jk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:44.582625 3663 obj_retry.go:492] Detected object tsf/patch-tekton-config-z22jk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:44.588421 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-z22jk I0514 23:24:44.588448 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-z22jk I0514 23:24:45.236874 3663 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-7p2l9-j2vbg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:45.236897 3663 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-7p2l9-j2vbg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:24:46.824940 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog I0514 23:24:48.546112 3663 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:48.546694 3663 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:48.565408 3663 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:48.565426 3663 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:48.572139 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0514 23:24:48.572161 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0514 23:24:49.361574 3663 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-5df5c6cfb8-nbb4n] creating logical port tsf-tas_rekor-redis-5df5c6cfb8-nbb4n for pod on switch ip-10-0-0-78.ec2.internal I0514 23:24:49.361674 3663 kube.go:256] Updating pod tsf-tas/rekor-redis-5df5c6cfb8-nbb4n I0514 23:24:49.381683 3663 pod.go:62] [tsf-tas/rekor-redis-5df5c6cfb8-nbb4n] pod update took 20.034903ms I0514 23:24:49.381703 3663 base_network_controller_pods.go:951] [default/tsf-tas/rekor-redis-5df5c6cfb8-nbb4n] addLogicalPort annotation time took 20.063085ms I0514 23:24:49.383029 3663 pods.go:271] [tsf-tas/rekor-redis-5df5c6cfb8-nbb4n] addLogicalPort took 21.477462ms, libovsdb time 835.249µs I0514 23:24:49.525342 3663 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-6d94cfc77c-46cgc] creating logical port tsf-tas_rekor-search-ui-6d94cfc77c-46cgc for pod on switch ip-10-0-0-78.ec2.internal I0514 23:24:49.525438 3663 kube.go:256] Updating pod tsf-tas/rekor-search-ui-6d94cfc77c-46cgc I0514 23:24:49.541729 3663 pod.go:62] [tsf-tas/rekor-search-ui-6d94cfc77c-46cgc] pod update took 16.311852ms I0514 23:24:49.541756 3663 base_network_controller_pods.go:951] [default/tsf-tas/rekor-search-ui-6d94cfc77c-46cgc] addLogicalPort annotation time took 16.344208ms I0514 23:24:49.543038 3663 pods.go:271] [tsf-tas/rekor-search-ui-6d94cfc77c-46cgc] addLogicalPort took 17.714762ms, libovsdb time 868.379µs I0514 23:24:49.885030 3663 cni.go:452] [tsf-tas/rekor-search-ui-6d94cfc77c-46cgc 6df0b9f3e8bff628ac365b0bd3bbe381f56a2d217e7fd81e94d755883564319a network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-search-ui-6d94cfc77c-46cgc 6df0b9f3e8bff628ac365b0bd3bbe381f56a2d217e7fd81e94d755883564319a network default NAD default NAD key ] I0514 23:24:49.894293 3663 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-6d94cfc77c-46cgc, hostIfaceName: 6df0b9f3e8bff62, network: default, NAD default, SandboxID: "6df0b9f3e8bff628ac365b0bd3bbe381f56a2d217e7fd81e94d755883564319a", PCI device ID: , UID: "54b45f55-252c-413f-96eb-73b37b6310bb", MAC: 0a:58:0a:82:00:29, IPs: [10.130.0.41/23] I0514 23:24:49.947473 3663 cni.go:473] [tsf-tas/rekor-search-ui-6d94cfc77c-46cgc 6df0b9f3e8bff628ac365b0bd3bbe381f56a2d217e7fd81e94d755883564319a network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-search-ui-6d94cfc77c-46cgc 6df0b9f3e8bff628ac365b0bd3bbe381f56a2d217e7fd81e94d755883564319a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:61:f9:48:bf:92\",\"name\":\"6df0b9f3e8bff62\"},{\"mac\":\"0a:58:0a:82:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3abcd518-1629-4d3a-9f50-489a5e46cc4d\"}],\"ips\":[{\"address\":\"10.130.0.41/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:24:50.318966 3663 cni.go:452] [tsf-tas/rekor-redis-5df5c6cfb8-nbb4n 9ea871f7bc484446f9d2205f5798bb22c7b876be9d9be4e112467c2d1781cc06 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-redis-5df5c6cfb8-nbb4n 9ea871f7bc484446f9d2205f5798bb22c7b876be9d9be4e112467c2d1781cc06 network default NAD default NAD key ] I0514 23:24:50.328673 3663 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-5df5c6cfb8-nbb4n, hostIfaceName: 9ea871f7bc48444, network: default, NAD default, SandboxID: "9ea871f7bc484446f9d2205f5798bb22c7b876be9d9be4e112467c2d1781cc06", PCI device ID: , UID: "db9ae34e-095b-4e1b-b532-af60c91879d9", MAC: 0a:58:0a:82:00:28, IPs: [10.130.0.40/23] I0514 23:24:50.379686 3663 cni.go:473] [tsf-tas/rekor-redis-5df5c6cfb8-nbb4n 9ea871f7bc484446f9d2205f5798bb22c7b876be9d9be4e112467c2d1781cc06 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-redis-5df5c6cfb8-nbb4n 9ea871f7bc484446f9d2205f5798bb22c7b876be9d9be4e112467c2d1781cc06 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:08:1b:11:22:63\",\"name\":\"9ea871f7bc48444\"},{\"mac\":\"0a:58:0a:82:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ced82d17-4b61-41d6-a252-669b6841f8fb\"}],\"ips\":[{\"address\":\"10.130.0.40/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:24:52.036150 3663 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-6556746489-zsbm4] creating logical port tsf-tas_rekor-server-6556746489-zsbm4 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:24:52.036439 3663 kube.go:256] Updating pod tsf-tas/rekor-server-6556746489-zsbm4 I0514 23:24:52.049839 3663 pod.go:62] [tsf-tas/rekor-server-6556746489-zsbm4] pod update took 13.409532ms I0514 23:24:52.049927 3663 base_network_controller_pods.go:951] [default/tsf-tas/rekor-server-6556746489-zsbm4] addLogicalPort annotation time took 13.528487ms I0514 23:24:52.051303 3663 pods.go:271] [tsf-tas/rekor-server-6556746489-zsbm4] addLogicalPort took 15.17872ms, libovsdb time 905.065µs W0514 23:24:52.898751 3663 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0514 23:24:52.919854 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0514 23:24:54.418395 3663 obj_retry.go:498] Detected object tsf-tpa/create-db-kjtlv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:54.418985 3663 obj_retry.go:498] Detected object tsf-tpa/create-db-kjtlv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:55.441458 3663 obj_retry.go:492] Detected object tsf-tpa/create-db-kjtlv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:55.441478 3663 obj_retry.go:492] Detected object tsf-tpa/create-db-kjtlv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:58.402121 3663 cni.go:452] [tsf-tas/rekor-server-6556746489-zsbm4 bcdb353561b8b76a05764d0a0299bc704c55b163911f335ad056cdfbe8d681d6 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-server-6556746489-zsbm4 bcdb353561b8b76a05764d0a0299bc704c55b163911f335ad056cdfbe8d681d6 network default NAD default NAD key ] I0514 23:24:58.411906 3663 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-6556746489-zsbm4, hostIfaceName: bcdb353561b8b76, network: default, NAD default, SandboxID: "bcdb353561b8b76a05764d0a0299bc704c55b163911f335ad056cdfbe8d681d6", PCI device ID: , UID: "12eb371d-c6fb-49d6-abb6-82c5051d9e3c", MAC: 0a:58:0a:82:00:2a, IPs: [10.130.0.42/23] I0514 23:24:58.443481 3663 obj_retry.go:498] Detected object tsf-tpa/migrate-db-4fs8n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:58.444000 3663 obj_retry.go:498] Detected object tsf-tpa/migrate-db-4fs8n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:58.464256 3663 cni.go:473] [tsf-tas/rekor-server-6556746489-zsbm4 bcdb353561b8b76a05764d0a0299bc704c55b163911f335ad056cdfbe8d681d6 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-server-6556746489-zsbm4 bcdb353561b8b76a05764d0a0299bc704c55b163911f335ad056cdfbe8d681d6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:48:41:52:d2:29\",\"name\":\"bcdb353561b8b76\"},{\"mac\":\"0a:58:0a:82:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cec1518b-ce94-4ad5-a4c3-c259569f2555\"}],\"ips\":[{\"address\":\"10.130.0.42/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:24:58.575189 3663 obj_retry.go:492] Detected object tsf-tpa/migrate-db-4fs8n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:58.575219 3663 obj_retry.go:492] Detected object tsf-tpa/migrate-db-4fs8n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:58.811833 3663 base_network_controller_pods.go:487] [default/tsf-tpa/create-importers-x6kkn] creating logical port tsf-tpa_create-importers-x6kkn for pod on switch ip-10-0-0-78.ec2.internal I0514 23:24:58.811935 3663 kube.go:256] Updating pod tsf-tpa/create-importers-x6kkn I0514 23:24:58.831518 3663 pod.go:62] [tsf-tpa/create-importers-x6kkn] pod update took 19.610871ms I0514 23:24:58.831539 3663 base_network_controller_pods.go:951] [default/tsf-tpa/create-importers-x6kkn] addLogicalPort annotation time took 19.637806ms I0514 23:24:58.832735 3663 pods.go:271] [tsf-tpa/create-importers-x6kkn] addLogicalPort took 20.942567ms, libovsdb time 789.008µs I0514 23:24:59.176681 3663 cni.go:452] [tsf-tpa/create-importers-x6kkn a2c6798acc52521baf8d5a4d6bd7a3a3f002aed43b43b33271ed355cc2c1893d network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-importers-x6kkn a2c6798acc52521baf8d5a4d6bd7a3a3f002aed43b43b33271ed355cc2c1893d network default NAD default NAD key ] I0514 23:24:59.186056 3663 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-importers-x6kkn, hostIfaceName: a2c6798acc52521, network: default, NAD default, SandboxID: "a2c6798acc52521baf8d5a4d6bd7a3a3f002aed43b43b33271ed355cc2c1893d", PCI device ID: , UID: "a62004c4-5908-4fab-9704-bd68998e8503", MAC: 0a:58:0a:82:00:2b, IPs: [10.130.0.43/23] I0514 23:24:59.236888 3663 cni.go:473] [tsf-tpa/create-importers-x6kkn a2c6798acc52521baf8d5a4d6bd7a3a3f002aed43b43b33271ed355cc2c1893d network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-importers-x6kkn a2c6798acc52521baf8d5a4d6bd7a3a3f002aed43b43b33271ed355cc2c1893d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:11:04:cb:aa:d3\",\"name\":\"a2c6798acc52521\"},{\"mac\":\"0a:58:0a:82:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6a6744d8-5363-4a82-ab62-02d60cbecea9\"}],\"ips\":[{\"address\":\"10.130.0.43/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:25:00.953246 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0514 23:25:02.967023 3663 cni.go:452] [tsf-tpa/create-importers-x6kkn a2c6798acc52521baf8d5a4d6bd7a3a3f002aed43b43b33271ed355cc2c1893d network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-importers-x6kkn a2c6798acc52521baf8d5a4d6bd7a3a3f002aed43b43b33271ed355cc2c1893d network default NAD default NAD key ] I0514 23:25:03.053003 3663 cni.go:473] [tsf-tpa/create-importers-x6kkn a2c6798acc52521baf8d5a4d6bd7a3a3f002aed43b43b33271ed355cc2c1893d network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-importers-x6kkn a2c6798acc52521baf8d5a4d6bd7a3a3f002aed43b43b33271ed355cc2c1893d network default NAD default NAD key default], result "{}", err I0514 23:25:03.104620 3663 obj_retry.go:498] Detected object tsf-tpa/create-importers-x6kkn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:03.104660 3663 pods.go:176] Deleting pod: tsf-tpa/create-importers-x6kkn I0514 23:25:03.106011 3663 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-importers-x6kkn, ips: 10.130.0.43 I0514 23:25:03.106036 3663 obj_retry.go:498] Detected object tsf-tpa/create-importers-x6kkn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:04.129706 3663 obj_retry.go:492] Detected object tsf-tpa/create-importers-x6kkn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:04.129731 3663 obj_retry.go:492] Detected object tsf-tpa/create-importers-x6kkn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:25:04.946545 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis W0514 23:25:04.972721 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui W0514 23:25:11.965873 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server W0514 23:25:14.415800 3663 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0514 23:25:16.721939 3663 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:16.722543 3663 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:16.742933 3663 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:16.742949 3663 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:16.749655 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0514 23:25:16.749678 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0514 23:25:17.482881 3663 namespace.go:98] [image-controller] adding namespace I0514 23:25:17.483712 3663 namespace.go:102] [image-controller] adding namespace took 803.225µs I0514 23:25:17.488762 3663 namespace.go:98] [default-tenant] adding namespace I0514 23:25:17.488782 3663 namespace.go:98] [integration-service] adding namespace I0514 23:25:17.488931 3663 namespace.go:98] [build-service] adding namespace I0514 23:25:17.489513 3663 namespace.go:102] [default-tenant] adding namespace took 732.787µs I0514 23:25:17.490184 3663 namespace.go:102] [integration-service] adding namespace took 1.386288ms I0514 23:25:17.490798 3663 namespace.go:102] [build-service] adding namespace took 1.851151ms I0514 23:25:17.510835 3663 namespace.go:142] [image-controller] updating namespace I0514 23:25:17.537023 3663 namespace.go:142] [image-controller] updating namespace I0514 23:25:17.545207 3663 namespace.go:142] [default-tenant] updating namespace I0514 23:25:17.575196 3663 namespace.go:142] [integration-service] updating namespace I0514 23:25:17.579121 3663 namespace.go:142] [default-tenant] updating namespace I0514 23:25:17.597012 3663 namespace.go:142] [integration-service] updating namespace I0514 23:25:17.601123 3663 namespace.go:142] [build-service] updating namespace I0514 23:25:17.627632 3663 namespace.go:142] [build-service] updating namespace I0514 23:25:17.747354 3663 namespace.go:142] [build-service] updating namespace I0514 23:25:17.813191 3663 namespace.go:142] [integration-service] updating namespace I0514 23:25:17.831327 3663 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-0-78.ec2.internal I0514 23:25:17.831412 3663 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0514 23:25:17.838080 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0514 23:25:17.851255 3663 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 19.865474ms I0514 23:25:17.851278 3663 base_network_controller_pods.go:951] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 19.89214ms I0514 23:25:17.852753 3663 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 21.437617ms, libovsdb time 1.02273ms I0514 23:25:17.865492 3663 namespace.go:98] [release-service] adding namespace I0514 23:25:17.866329 3663 namespace.go:102] [release-service] adding namespace took 818.931µs I0514 23:25:17.900126 3663 namespace.go:142] [release-service] updating namespace I0514 23:25:17.918233 3663 namespace.go:142] [release-service] updating namespace I0514 23:25:18.018931 3663 namespace.go:142] [konflux-ui] updating namespace I0514 23:25:18.154493 3663 namespace.go:98] [namespace-lister] adding namespace I0514 23:25:18.154509 3663 namespace.go:98] [konflux-info] adding namespace I0514 23:25:18.155410 3663 namespace.go:102] [namespace-lister] adding namespace took 894.299µs I0514 23:25:18.156200 3663 namespace.go:102] [konflux-info] adding namespace took 1.675844ms I0514 23:25:18.197342 3663 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:18.222791 3663 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:18.238759 3663 namespace.go:142] [konflux-info] updating namespace I0514 23:25:18.245982 3663 cni.go:452] [konflux-ui/test-tsf-konflux e88df640db689e97ebb2b8d887aecc631922e0d308efe6f5cf7e395d1266091e network default NAD default NAD key ] ADD starting CNI request [konflux-ui/test-tsf-konflux e88df640db689e97ebb2b8d887aecc631922e0d308efe6f5cf7e395d1266091e network default NAD default NAD key ] I0514 23:25:18.254022 3663 namespace.go:142] [konflux-info] updating namespace I0514 23:25:18.256249 3663 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: e88df640db689e9, network: default, NAD default, SandboxID: "e88df640db689e97ebb2b8d887aecc631922e0d308efe6f5cf7e395d1266091e", PCI device ID: , UID: "4e5b156a-cd9c-4ceb-8995-7c893832b8a3", MAC: 0a:58:0a:82:00:2c, IPs: [10.130.0.44/23] I0514 23:25:18.310124 3663 cni.go:473] [konflux-ui/test-tsf-konflux e88df640db689e97ebb2b8d887aecc631922e0d308efe6f5cf7e395d1266091e network default NAD default NAD key default] ADD finished CNI request [konflux-ui/test-tsf-konflux e88df640db689e97ebb2b8d887aecc631922e0d308efe6f5cf7e395d1266091e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:26:19:56:e4:cf\",\"name\":\"e88df640db689e9\"},{\"mac\":\"0a:58:0a:82:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b4a02523-b27b-4af6-88b3-907d749737b7\"}],\"ips\":[{\"address\":\"10.130.0.44/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:25:18.311588 3663 namespace.go:98] [enterprise-contract-service] adding namespace I0514 23:25:18.312372 3663 namespace.go:102] [enterprise-contract-service] adding namespace took 764.596µs I0514 23:25:18.355636 3663 namespace.go:142] [enterprise-contract-service] updating namespace I0514 23:25:18.379907 3663 namespace.go:142] [enterprise-contract-service] updating namespace I0514 23:25:18.389258 3663 namespace.go:142] [image-controller] updating namespace I0514 23:25:19.000991 3663 namespace.go:142] [default-tenant] updating namespace I0514 23:25:19.615940 3663 namespace.go:98] [konflux-cli] adding namespace I0514 23:25:19.616868 3663 namespace.go:102] [konflux-cli] adding namespace took 902.424µs I0514 23:25:19.647592 3663 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:19.670194 3663 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:19.843623 3663 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-696b855848-pp66f] creating logical port namespace-lister_namespace-lister-696b855848-pp66f for pod on switch ip-10-0-0-78.ec2.internal I0514 23:25:19.843719 3663 kube.go:256] Updating pod namespace-lister/namespace-lister-696b855848-pp66f I0514 23:25:19.861301 3663 pod.go:62] [namespace-lister/namespace-lister-696b855848-pp66f] pod update took 17.607291ms I0514 23:25:19.861322 3663 base_network_controller_pods.go:951] [default/namespace-lister/namespace-lister-696b855848-pp66f] addLogicalPort annotation time took 17.636164ms I0514 23:25:19.862858 3663 pods.go:271] [namespace-lister/namespace-lister-696b855848-pp66f] addLogicalPort took 19.251317ms, libovsdb time 1.085467ms I0514 23:25:19.883254 3663 namespace.go:98] [segment-bridge] adding namespace I0514 23:25:19.884116 3663 namespace.go:102] [segment-bridge] adding namespace took 843.307µs I0514 23:25:20.015357 3663 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:20.041585 3663 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:20.257988 3663 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0514 23:25:20.258620 3663 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0514 23:25:20.258641 3663 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0514 23:25:20.261836 3663 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0514 23:25:20.406141 3663 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0514 23:25:20.406165 3663 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0514 23:25:20.407533 3663 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0514 23:25:20.816983 3663 cni.go:452] [namespace-lister/namespace-lister-696b855848-pp66f c7a5d970c5f1ea1635e09b0b12c9d167cca37e8b40933055aea62e81cfc66525 network default NAD default NAD key ] ADD starting CNI request [namespace-lister/namespace-lister-696b855848-pp66f c7a5d970c5f1ea1635e09b0b12c9d167cca37e8b40933055aea62e81cfc66525 network default NAD default NAD key ] I0514 23:25:20.826516 3663 helper_linux.go:556] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-696b855848-pp66f, hostIfaceName: c7a5d970c5f1ea1, network: default, NAD default, SandboxID: "c7a5d970c5f1ea1635e09b0b12c9d167cca37e8b40933055aea62e81cfc66525", PCI device ID: , UID: "1edb7e15-56d5-4b9b-a598-b71ea4b0f81a", MAC: 0a:58:0a:82:00:2d, IPs: [10.130.0.45/23] I0514 23:25:20.880389 3663 cni.go:473] [namespace-lister/namespace-lister-696b855848-pp66f c7a5d970c5f1ea1635e09b0b12c9d167cca37e8b40933055aea62e81cfc66525 network default NAD default NAD key default] ADD finished CNI request [namespace-lister/namespace-lister-696b855848-pp66f c7a5d970c5f1ea1635e09b0b12c9d167cca37e8b40933055aea62e81cfc66525 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:a2:a9:60:60:0d\",\"name\":\"c7a5d970c5f1ea1\"},{\"mac\":\"0a:58:0a:82:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5b6a84c7-3ae0-440f-aa3d-8feb03b51f9d\"}],\"ips\":[{\"address\":\"10.130.0.45/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:25:21.028295 3663 namespace.go:142] [build-service] updating namespace I0514 23:25:21.875528 3663 namespace.go:142] [default-tenant] updating namespace I0514 23:25:21.999398 3663 namespace.go:142] [enterprise-contract-service] updating namespace I0514 23:25:22.102580 3663 namespace.go:142] [image-controller] updating namespace I0514 23:25:22.155491 3663 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-6f859968fc-kdn6j] creating logical port image-controller_image-controller-controller-manager-6f859968fc-kdn6j for pod on switch ip-10-0-0-78.ec2.internal I0514 23:25:22.155660 3663 kube.go:256] Updating pod image-controller/image-controller-controller-manager-6f859968fc-kdn6j I0514 23:25:22.198853 3663 pod.go:62] [image-controller/image-controller-controller-manager-6f859968fc-kdn6j] pod update took 43.183741ms I0514 23:25:22.198944 3663 base_network_controller_pods.go:951] [default/image-controller/image-controller-controller-manager-6f859968fc-kdn6j] addLogicalPort annotation time took 43.314069ms I0514 23:25:22.200586 3663 pods.go:271] [image-controller/image-controller-controller-manager-6f859968fc-kdn6j] addLogicalPort took 45.120787ms, libovsdb time 1.023989ms I0514 23:25:22.248806 3663 base_network_controller_pods.go:487] [default/konflux-ui/proxy-78867c4458-c44l9] creating logical port konflux-ui_proxy-78867c4458-c44l9 for pod on switch ip-10-0-0-78.ec2.internal I0514 23:25:22.248954 3663 kube.go:256] Updating pod konflux-ui/proxy-78867c4458-c44l9 I0514 23:25:22.351592 3663 pod.go:62] [konflux-ui/proxy-78867c4458-c44l9] pod update took 102.684793ms I0514 23:25:22.351622 3663 base_network_controller_pods.go:951] [default/konflux-ui/proxy-78867c4458-c44l9] addLogicalPort annotation time took 102.724096ms I0514 23:25:22.353299 3663 pods.go:271] [konflux-ui/proxy-78867c4458-c44l9] addLogicalPort took 104.509086ms, libovsdb time 1.042548ms I0514 23:25:23.024054 3663 namespace.go:142] [integration-service] updating namespace I0514 23:25:23.075374 3663 cni.go:452] [image-controller/image-controller-controller-manager-6f859968fc-kdn6j 78c5fb8294ca49111f2aa87857b84d5f970b12fb4e8ebc744066abb01b7191af network default NAD default NAD key ] ADD starting CNI request [image-controller/image-controller-controller-manager-6f859968fc-kdn6j 78c5fb8294ca49111f2aa87857b84d5f970b12fb4e8ebc744066abb01b7191af network default NAD default NAD key ] I0514 23:25:23.085958 3663 helper_linux.go:556] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-6f859968fc-kdn6j, hostIfaceName: 78c5fb8294ca491, network: default, NAD default, SandboxID: "78c5fb8294ca49111f2aa87857b84d5f970b12fb4e8ebc744066abb01b7191af", PCI device ID: , UID: "b1767143-5114-4d30-9f63-915e6fa41b12", MAC: 0a:58:0a:82:00:2e, IPs: [10.130.0.46/23] I0514 23:25:23.146748 3663 cni.go:473] [image-controller/image-controller-controller-manager-6f859968fc-kdn6j 78c5fb8294ca49111f2aa87857b84d5f970b12fb4e8ebc744066abb01b7191af network default NAD default NAD key default] ADD finished CNI request [image-controller/image-controller-controller-manager-6f859968fc-kdn6j 78c5fb8294ca49111f2aa87857b84d5f970b12fb4e8ebc744066abb01b7191af network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:ba:63:71:8d:a0\",\"name\":\"78c5fb8294ca491\"},{\"mac\":\"0a:58:0a:82:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ba661b53-1190-4e36-b910-7de47f3f7e38\"}],\"ips\":[{\"address\":\"10.130.0.46/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0514 23:25:23.240082 3663 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:23.266138 3663 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-78b44785b4-xklrc] creating logical port integration-service_integration-service-controller-manager-78b44785b4-xklrc for pod on switch ip-10-0-0-78.ec2.internal I0514 23:25:23.266264 3663 kube.go:256] Updating pod integration-service/integration-service-controller-manager-78b44785b4-xklrc I0514 23:25:23.280423 3663 pod.go:62] [integration-service/integration-service-controller-manager-78b44785b4-xklrc] pod update took 14.183868ms I0514 23:25:23.280447 3663 base_network_controller_pods.go:951] [default/integration-service/integration-service-controller-manager-78b44785b4-xklrc] addLogicalPort annotation time took 14.215666ms I0514 23:25:23.281957 3663 pods.go:271] [integration-service/integration-service-controller-manager-78b44785b4-xklrc] addLogicalPort took 15.835297ms, libovsdb time 951.326µs I0514 23:25:23.480266 3663 namespace.go:142] [konflux-info] updating namespace I0514 23:25:23.731647 3663 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:23.922697 3663 namespace.go:142] [release-service] updating namespace I0514 23:25:24.050608 3663 namespace.go:142] [build-service] updating namespace I0514 23:25:24.068386 3663 namespace.go:142] [default-tenant] updating namespace I0514 23:25:24.089154 3663 namespace.go:142] [enterprise-contract-service] updating namespace W0514 23:25:24.101460 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0514 23:25:24.102406 3663 namespace.go:142] [image-controller] updating namespace I0514 23:25:24.116983 3663 namespace.go:142] [integration-service] updating namespace W0514 23:25:24.132141 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister I0514 23:25:24.161172 3663 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:24.179769 3663 namespace.go:142] [konflux-info] updating namespace I0514 23:25:24.207755 3663 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:24.220711 3663 namespace.go:142] [release-service] updating namespace I0514 23:25:24.275283 3663 namespace.go:142] [build-service] updating namespace I0514 23:25:24.336088 3663 namespace.go:142] [default-tenant] updating namespace I0514 23:25:24.399755 3663 namespace.go:142] [enterprise-contract-service] updating namespace I0514 23:25:24.472097 3663 namespace.go:142] [image-controller] updating namespace I0514 23:25:24.569739 3663 namespace.go:142] [integration-service] updating namespace I0514 23:25:24.674566 3663 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:24.777138 3663 namespace.go:142] [konflux-info] updating namespace I0514 23:25:24.878880 3663 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:24.967022 3663 namespace.go:142] [release-service] updating namespace I0514 23:25:26.224156 3663 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:26.354086 3663 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:26.457538 3663 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:26.830232 3663 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-s6cwq-rwlz5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:26.830996 3663 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-s6cwq-rwlz5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0514 23:25:27.674890 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0514 23:25:27.797428 3663 cni.go:452] [integration-service/integration-service-controller-manager-78b44785b4-xklrc deb744490b3751d5efa9c1894fd9b538f2666f4dbaef411d81e8159197ceed97 network default NAD default NAD key ] ADD starting CNI request [integration-service/integration-service-controller-manager-78b44785b4-xklrc deb744490b3751d5efa9c1894fd9b538f2666f4dbaef411d81e8159197ceed97 network default NAD default NAD key ] I0514 23:25:27.807039 3663 helper_linux.go:556] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-78b44785b4-xklrc, hostIfaceName: deb744490b3751d, network: default, NAD default, SandboxID: "deb744490b3751d5efa9c1894fd9b538f2666f4dbaef411d81e8159197ceed97", PCI device ID: , UID: "44268f46-665b-4384-99ea-a5410f50edcd", MAC: 0a:58:0a:82:00:30, IPs: [10.130.0.48/23] I0514 23:25:27.852394 3663 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-s6cwq-rwlz5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:27.852421 3663 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-s6cwq-rwlz5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:27.865043 3663 cni.go:473] [integration-service/integration-service-controller-manager-78b44785b4-xklrc deb744490b3751d5efa9c1894fd9b538f2666f4dbaef411d81e8159197ceed97 network default NAD default NAD key default] ADD finished CNI request [integration-service/integration-service-controller-manager-78b44785b4-xklrc deb744490b3751d5efa9c1894fd9b538f2666f4dbaef411d81e8159197ceed97 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:b2:88:e8:ef:44\",\"name\":\"deb744490b3751d\"},{\"mac\":\"0a:58:0a:82:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d4bf7ed6-36c2-4984-ae11-3dd00c1bbb7f\"}],\"ips\":[{\"address\":\"10.130.0.48/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:25:28.116597 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0514 23:25:30.874626 3663 cni.go:452] [konflux-ui/proxy-78867c4458-c44l9 f5b11b62a857a5ad83dad2664afb8a82508966c694526eabe738312b2838c3ef network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-78867c4458-c44l9 f5b11b62a857a5ad83dad2664afb8a82508966c694526eabe738312b2838c3ef network default NAD default NAD key ] I0514 23:25:30.884976 3663 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-78867c4458-c44l9, hostIfaceName: f5b11b62a857a5a, network: default, NAD default, SandboxID: "f5b11b62a857a5ad83dad2664afb8a82508966c694526eabe738312b2838c3ef", PCI device ID: , UID: "762c7719-cc09-451f-98dd-3e9c81369834", MAC: 0a:58:0a:82:00:2f, IPs: [10.130.0.47/23] I0514 23:25:30.939730 3663 cni.go:473] [konflux-ui/proxy-78867c4458-c44l9 f5b11b62a857a5ad83dad2664afb8a82508966c694526eabe738312b2838c3ef network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-78867c4458-c44l9 f5b11b62a857a5ad83dad2664afb8a82508966c694526eabe738312b2838c3ef network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:2c:24:01:72:53\",\"name\":\"f5b11b62a857a5a\"},{\"mac\":\"0a:58:0a:82:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c5a1147e-88a9-4da4-bd6a-b4fb928dc6c2\"}],\"ips\":[{\"address\":\"10.130.0.47/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0514 23:25:31.135527 3663 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0514 23:25:31.138110 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0514 23:25:32.714375 3663 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0514 23:25:33.720709 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0514 23:25:33.722465 3663 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0514 23:25:36.175626 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0514 23:25:36.175653 3663 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0514 23:25:38.779278 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0514 23:25:38.781166 3663 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0514 23:25:38.799713 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0514 23:25:38.838829 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0514 23:25:38.838858 3663 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0514 23:25:39.128889 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0514 23:25:42.146360 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0514 23:25:42.149373 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service I0514 23:25:42.498709 3663 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:42.499342 3663 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:42.792428 3663 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:42.792487 3663 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:42.796755 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x I0514 23:25:42.796784 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x W0514 23:26:17.258266 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0514 23:26:17.258289 3663 util.go:959] Failed to build node endpoints for node ip-10-0-0-78.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0514 23:26:48.200499 3663 cni.go:452] [konflux-ui/proxy-78867c4458-c44l9 f5b11b62a857a5ad83dad2664afb8a82508966c694526eabe738312b2838c3ef network default NAD default NAD key ] DEL starting CNI request [konflux-ui/proxy-78867c4458-c44l9 f5b11b62a857a5ad83dad2664afb8a82508966c694526eabe738312b2838c3ef network default NAD default NAD key ] I0514 23:26:48.292300 3663 cni.go:473] [konflux-ui/proxy-78867c4458-c44l9 f5b11b62a857a5ad83dad2664afb8a82508966c694526eabe738312b2838c3ef network default NAD default NAD key default] DEL finished CNI request [konflux-ui/proxy-78867c4458-c44l9 f5b11b62a857a5ad83dad2664afb8a82508966c694526eabe738312b2838c3ef network default NAD default NAD key default], result "{}", err I0514 23:26:48.357274 3663 obj_retry.go:498] Detected object konflux-ui/proxy-78867c4458-c44l9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:26:48.357324 3663 pods.go:176] Deleting pod: konflux-ui/proxy-78867c4458-c44l9 I0514 23:26:48.358719 3663 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-78867c4458-c44l9, ips: 10.130.0.47 I0514 23:26:48.358747 3663 obj_retry.go:498] Detected object konflux-ui/proxy-78867c4458-c44l9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:26:48.588956 3663 obj_retry.go:492] Detected object konflux-ui/proxy-78867c4458-c44l9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:26:48.588981 3663 obj_retry.go:492] Detected object konflux-ui/proxy-78867c4458-c44l9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:26:48.598158 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-78867c4458-c44l9 I0514 23:26:48.598185 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-78867c4458-c44l9 I0514 23:26:49.589547 3663 cni.go:452] [konflux-ui/test-tsf-konflux e88df640db689e97ebb2b8d887aecc631922e0d308efe6f5cf7e395d1266091e network default NAD default NAD key ] DEL starting CNI request [konflux-ui/test-tsf-konflux e88df640db689e97ebb2b8d887aecc631922e0d308efe6f5cf7e395d1266091e network default NAD default NAD key ] I0514 23:26:49.677038 3663 cni.go:473] [konflux-ui/test-tsf-konflux e88df640db689e97ebb2b8d887aecc631922e0d308efe6f5cf7e395d1266091e network default NAD default NAD key default] DEL finished CNI request [konflux-ui/test-tsf-konflux e88df640db689e97ebb2b8d887aecc631922e0d308efe6f5cf7e395d1266091e network default NAD default NAD key default], result "{}", err I0514 23:26:49.733337 3663 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:26:49.733373 3663 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0514 23:26:49.734918 3663 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.130.0.44 I0514 23:26:49.734948 3663 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:26:49.778189 3663 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:26:49.778214 3663 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:26:49.785537 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0514 23:26:49.785564 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0514 23:27:09.563275 3663 namespace.go:98] [managed-tenant-hjy2g] adding namespace I0514 23:27:09.564215 3663 namespace.go:102] [managed-tenant-hjy2g] adding namespace took 914.375µs I0514 23:27:09.589319 3663 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:09.610264 3663 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:10.617187 3663 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:10.751625 3663 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:10.850946 3663 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:28.726627 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=58 I0514 23:28:28.723518 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=12 I0514 23:28:32.821143 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=5 I0514 23:29:26.850726 3663 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0514 23:29:29.727246 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=79 I0514 23:29:42.834506 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0514 23:29:53.814673 3663 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0514 23:29:58.734128 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=280 I0514 23:30:04.464668 3663 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:30:04.465327 3663 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:30:05.330091 3663 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:30:05.330117 3663 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:30:06.842207 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0514 23:30:07.855638 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0514 23:30:32.855798 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0514 23:32:34.726902 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=47 I0514 23:33:01.816112 3663 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0514 23:33:06.727224 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=92 I0514 23:33:22.728707 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=37 I0514 23:33:31.828854 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0514 23:33:47.726379 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0514 23:34:52.823493 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0514 23:34:52.841259 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=11 I0514 23:34:55.050999 3663 obj_retry.go:492] Detected object tsf-tpa/create-db-kjtlv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:34:55.051047 3663 obj_retry.go:492] Detected object tsf-tpa/create-db-kjtlv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:34:55.057363 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-kjtlv I0514 23:34:55.057388 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-kjtlv I0514 23:34:58.040657 3663 obj_retry.go:492] Detected object tsf-tpa/migrate-db-4fs8n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:34:58.040705 3663 obj_retry.go:492] Detected object tsf-tpa/migrate-db-4fs8n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:34:58.046120 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-4fs8n I0514 23:34:58.046147 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-4fs8n I0514 23:35:04.032974 3663 obj_retry.go:492] Detected object tsf-tpa/create-importers-x6kkn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:35:04.033020 3663 obj_retry.go:492] Detected object tsf-tpa/create-importers-x6kkn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:35:04.036694 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-x6kkn I0514 23:35:04.036727 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-x6kkn I0514 23:35:53.729582 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=7 I0514 23:36:44.817209 3663 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0514 23:38:02.736914 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=24