+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-253.ec2.internal ++ [[ -n ip-10-0-1-253.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-253.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0424 15:05:28.921453850 - disable conntrack on geneve port' I0424 15:05:28.921453850 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0424 15:05:28.939320209 - starting ovnkube-node + echo 'I0424 15:05:28.939320209 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-253.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-253.ec2.internal --init-node ip-10-0-1-253.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-253.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0424 15:05:28.978012 3680 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0424 15:05:28.978068 3680 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-1706d184cc.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0424 15:05:28.979453 3680 kube.go:431] Waiting for certificate I0424 15:05:28.979883 3680 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0424 15:05:28.979952 3680 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0424 15:05:28.980895 3680 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0424 15:05:29.016652 3680 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0424 15:05:29.016668 3680 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0424 15:05:29.020242 3680 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0424 15:05:29.020338 3680 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-q7227" I0424 15:05:29.027538 3680 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-q7227" I0424 15:05:29.027592 3680 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0424 15:05:29.980277 3680 kube.go:438] Certificate found I0424 15:05:29.981175 3680 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0424 15:05:29.982031 3680 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0424 15:05:30.035868 3680 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-25 15:00:29 +0000 UTC" deadline="2026-04-25 08:31:27.024935988 +0000 UTC" I0424 15:05:30.035907 3680 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="17h25m56.989040834s" I0424 15:05:30.043790 3680 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0424 15:05:30.058733 3680 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0424 15:05:30.058823 3680 server.go:101] MetricServer registers OVS metrics I0424 15:05:30.059564 3680 server.go:105] MetricServer registers OVN DB metrics I0424 15:05:30.060653 3680 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0424 15:05:30.060710 3680 factory.go:561] Starting watch factory I0424 15:05:30.060788 3680 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.060809 3680 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.060818 3680 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.060828 3680 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.060833 3680 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.060841 3680 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.060793 3680 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.060942 3680 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.060792 3680 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.061041 3680 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.061062 3680 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.061066 3680 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.063181 3680 controller_manager.go:370] Starting the ovnkube controller I0424 15:05:30.063192 3680 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-253.ec2.internal I0424 15:05:30.063231 3680 controller_manager.go:395] NBDB zone sync took: 33.018µs I0424 15:05:30.063240 3680 factory.go:561] Starting watch factory I0424 15:05:30.064648 3680 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0424 15:05:30.064669 3680 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0424 15:05:30.077970 3680 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0424 15:05:30.078014 3680 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0424 15:05:30.079415 3680 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.079509 3680 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.079654 3680 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.082506 3680 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0424 15:05:30.082582 3680 server.go:109] MetricServer registers OVN Controller metrics I0424 15:05:30.082583 3680 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.084412 3680 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.087692 3680 server.go:113] MetricServer registers OVN Northd metrics I0424 15:05:30.092033 3680 metrics.go:519] OVN Metrics Server starts to run ... I0424 15:05:30.092742 3680 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0424 15:05:30.166968 3680 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0424 15:05:30.166983 3680 factory.go:1977] *v1.Namespace informer cache synced successfully I0424 15:05:30.166987 3680 factory.go:1977] *v1.Node informer cache synced successfully I0424 15:05:30.166989 3680 factory.go:1977] *v1.Service informer cache synced successfully I0424 15:05:30.166992 3680 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0424 15:05:30.166996 3680 factory.go:1977] *v1.Pod informer cache synced successfully I0424 15:05:30.167070 3680 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0424 15:05:30.167079 3680 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0424 15:05:30.167089 3680 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0424 15:05:30.167092 3680 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0424 15:05:30.169365 3680 factory.go:1977] *v1.Pod informer cache synced successfully I0424 15:05:30.169376 3680 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0424 15:05:30.169380 3680 factory.go:1977] *v1.Namespace informer cache synced successfully I0424 15:05:30.169383 3680 factory.go:1977] *v1.Node informer cache synced successfully I0424 15:05:30.169386 3680 factory.go:1977] *v1.Service informer cache synced successfully I0424 15:05:30.169389 3680 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0424 15:05:30.170124 3680 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0424 15:05:30.170125 3680 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0424 15:05:30.172165 3680 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0424 15:05:30.172174 3680 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0424 15:05:30.172219 3680 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.172230 3680 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.174476 3680 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0424 15:05:30.174489 3680 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0424 15:05:30.174495 3680 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.177404 3680 factory.go:1977] *v1.EgressIP informer cache synced successfully I0424 15:05:30.177867 3680 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.177882 3680 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.179667 3680 factory.go:1977] *v1.EgressIP informer cache synced successfully I0424 15:05:30.180156 3680 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.183071 3680 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0424 15:05:30.183119 3680 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.183129 3680 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.184912 3680 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0424 15:05:30.185291 3680 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.188269 3680 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0424 15:05:30.188682 3680 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.188692 3680 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.190731 3680 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0424 15:05:30.190849 3680 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.193670 3680 factory.go:1977] *v1.EgressService informer cache synced successfully I0424 15:05:30.193719 3680 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.193731 3680 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.195788 3680 factory.go:1977] *v1.EgressService informer cache synced successfully I0424 15:05:30.195950 3680 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.198725 3680 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0424 15:05:30.198858 3680 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0424 15:05:30.198869 3680 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0424 15:05:30.200963 3680 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0424 15:05:30.201092 3680 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0424 15:05:30.204082 3680 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0424 15:05:30.204130 3680 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.204147 3680 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.204295 3680 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.204312 3680 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.206009 3680 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0424 15:05:30.206401 3680 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.206819 3680 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0424 15:05:30.209815 3680 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0424 15:05:30.209825 3680 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0424 15:05:30.209831 3680 factory.go:680] Watch Factory start up complete, took: 149.123786ms I0424 15:05:30.209886 3680 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0424 15:05:30.211140 3680 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0424 15:05:30.211150 3680 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0424 15:05:30.211156 3680 factory.go:680] Watch Factory start up complete, took: 147.915085ms I0424 15:05:30.211163 3680 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-1-253.ec2.internal" zone I0424 15:05:30.259975 3680 default_node_network_controller.go:710] Initializing the default node network controller I0424 15:05:30.342400 3680 udn_isolation.go:101] Starting UDN host isolation manager I0424 15:05:30.348116 3680 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0424 15:05:30.384984 3680 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0424 15:05:30.385059 3680 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0424 15:05:30.385075 3680 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0424 15:05:30.416115 3680 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0424 15:05:30.416172 3680 default_node_network_controller.go:809] Node ip-10-0-1-253.ec2.internal ready for ovn initialization with subnet 10.130.0.0/23 I0424 15:05:30.421655 3680 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.253"] k8s.ovn.org/zone-name:ip-10-0-1-253.ec2.internal] on node ip-10-0-1-253.ec2.internal I0424 15:05:30.442373 3680 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0424 15:05:30.448135 3680 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0424 15:05:30.448267 3680 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0424 15:05:30.448417 3680 gateway_init.go:263] Preparing Gateway I0424 15:05:30.448428 3680 gateway_shared_intf.go:1687] Creating new gateway I0424 15:05:30.461965 3680 controller_manager.go:429] Waiting for node in zone sync took: 250.790884ms I0424 15:05:30.470159 3680 controller_manager.go:611] Switching to transit router for layer2 networks I0424 15:05:30.470172 3680 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-253.ec2.internal I0424 15:05:30.486075 3680 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0424 15:05:30.492045 3680 controller_manager.go:290] SCTP support detected in OVN I0424 15:05:30.492578 3680 services_controller.go:69] Creating services controller for network=default I0424 15:05:30.492711 3680 udn_enabled_svc.go:80] Waiting for service informer to sync I0424 15:05:30.492740 3680 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0424 15:05:30.492767 3680 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0424 15:05:30.492782 3680 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0424 15:05:30.492793 3680 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0424 15:05:30.492983 3680 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0424 15:05:30.493025 3680 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0424 15:05:30.493073 3680 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 62.209µs I0424 15:05:30.493092 3680 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 60.307µs I0424 15:05:30.493098 3680 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 72.924µs I0424 15:05:30.493104 3680 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0424 15:05:30.493110 3680 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0424 15:05:30.493118 3680 default_network_controller.go:325] Starting the default network controller I0424 15:05:30.493154 3680 udn_enabled_svc.go:95] Performing full resync I0424 15:05:30.493231 3680 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 13.671µs I0424 15:05:30.493417 3680 udn_enabled_svc.go:99] Waiting for handler to sync I0424 15:05:30.493426 3680 udn_enabled_svc.go:104] Starting worker I0424 15:05:30.502181 3680 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0424 15:05:30.502270 3680 repair.go:33] Syncing exgw routes took 77.859µs I0424 15:05:30.502294 3680 default_network_controller.go:414] Starting all the Watchers... I0424 15:05:30.502460 3680 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0424 15:05:30.502461 3680 namespace.go:98] [openshift-operators-redhat] adding namespace I0424 15:05:30.502488 3680 namespace.go:98] [openshift-service-ca-operator] adding namespace I0424 15:05:30.502505 3680 namespace.go:98] [default] adding namespace I0424 15:05:30.502506 3680 namespace.go:98] [openshift-backplane-mobb] adding namespace I0424 15:05:30.502516 3680 namespace.go:98] [openshift-etcd] adding namespace I0424 15:05:30.502519 3680 namespace.go:98] [openshift-backplane-cse] adding namespace I0424 15:05:30.502526 3680 namespace.go:98] [kube-system] adding namespace I0424 15:05:30.502532 3680 namespace.go:98] [openshift-controller-manager] adding namespace I0424 15:05:30.502534 3680 namespace.go:98] [openshift-config] adding namespace I0424 15:05:30.502538 3680 namespace.go:98] [openshift-machine-api] adding namespace I0424 15:05:30.502540 3680 namespace.go:98] [openshift-authentication-operator] adding namespace I0424 15:05:30.502542 3680 namespace.go:98] [openshift-cluster-version] adding namespace I0424 15:05:30.502541 3680 namespace.go:98] [openshift-backplane-tam] adding namespace I0424 15:05:30.502558 3680 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0424 15:05:30.503045 3680 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 573.946µs I0424 15:05:30.503063 3680 namespace.go:98] [openshift-kube-apiserver] adding namespace I0424 15:05:30.503495 3680 namespace.go:102] [openshift-kube-apiserver] adding namespace took 427.557µs I0424 15:05:30.503509 3680 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0424 15:05:30.503946 3680 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 432.29µs I0424 15:05:30.503959 3680 namespace.go:98] [openshift-network-console] adding namespace I0424 15:05:30.504446 3680 namespace.go:102] [openshift-operators-redhat] adding namespace took 1.973381ms I0424 15:05:30.504456 3680 namespace.go:98] [openshift-host-network] adding namespace I0424 15:05:30.504942 3680 namespace.go:102] [openshift-service-ca-operator] adding namespace took 2.443407ms I0424 15:05:30.504958 3680 namespace.go:98] [openshift-image-registry] adding namespace I0424 15:05:30.505423 3680 namespace.go:102] [default] adding namespace took 2.911063ms I0424 15:05:30.505433 3680 namespace.go:98] [openshift-console] adding namespace I0424 15:05:30.505865 3680 namespace.go:102] [openshift-backplane-mobb] adding namespace took 3.350247ms I0424 15:05:30.505874 3680 namespace.go:98] [openshift-apiserver-operator] adding namespace I0424 15:05:30.506341 3680 namespace.go:102] [openshift-etcd] adding namespace took 3.815876ms I0424 15:05:30.506360 3680 namespace.go:98] [openshift] adding namespace I0424 15:05:30.506789 3680 namespace.go:102] [openshift-backplane-cse] adding namespace took 4.260647ms I0424 15:05:30.506800 3680 namespace.go:98] [openshift-config-managed] adding namespace I0424 15:05:30.507219 3680 namespace.go:102] [kube-system] adding namespace took 4.687956ms I0424 15:05:30.507228 3680 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0424 15:05:30.507644 3680 namespace.go:102] [openshift-controller-manager] adding namespace took 5.104032ms I0424 15:05:30.507662 3680 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0424 15:05:30.508143 3680 namespace.go:102] [openshift-config] adding namespace took 5.60136ms I0424 15:05:30.508161 3680 namespace.go:98] [openshift-backplane-csm] adding namespace I0424 15:05:30.509025 3680 namespace.go:102] [openshift-machine-api] adding namespace took 6.477359ms I0424 15:05:30.509044 3680 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0424 15:05:30.509492 3680 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.94197ms I0424 15:05:30.509507 3680 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0424 15:05:30.510040 3680 namespace.go:102] [openshift-cluster-version] adding namespace took 7.48785ms I0424 15:05:30.510055 3680 namespace.go:98] [openshift-authentication] adding namespace I0424 15:05:30.510576 3680 namespace.go:102] [openshift-backplane-tam] adding namespace took 8.020357ms I0424 15:05:30.510592 3680 namespace.go:98] [openshift-apiserver] adding namespace I0424 15:05:30.511042 3680 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 8.473561ms I0424 15:05:30.511054 3680 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0424 15:05:30.511575 3680 namespace.go:102] [openshift-network-console] adding namespace took 7.61139ms I0424 15:05:30.511585 3680 namespace.go:98] [openshift-config-operator] adding namespace I0424 15:05:30.512148 3680 namespace.go:102] [openshift-host-network] adding namespace took 7.685513ms I0424 15:05:30.512165 3680 namespace.go:98] [openshift-route-controller-manager] adding namespace I0424 15:05:30.512541 3680 namespace.go:102] [openshift-image-registry] adding namespace took 7.578164ms I0424 15:05:30.512551 3680 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0424 15:05:30.513011 3680 namespace.go:102] [openshift-console] adding namespace took 7.573581ms I0424 15:05:30.513021 3680 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0424 15:05:30.513383 3680 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.503795ms I0424 15:05:30.513395 3680 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0424 15:05:30.513778 3680 namespace.go:102] [openshift] adding namespace took 7.412564ms I0424 15:05:30.513788 3680 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0424 15:05:30.514283 3680 namespace.go:102] [openshift-config-managed] adding namespace took 7.478874ms I0424 15:05:30.514295 3680 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0424 15:05:30.514756 3680 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.522735ms I0424 15:05:30.514770 3680 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0424 15:05:30.515267 3680 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.599672ms I0424 15:05:30.515281 3680 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0424 15:05:30.515655 3680 namespace.go:102] [openshift-backplane-csm] adding namespace took 7.489616ms I0424 15:05:30.515667 3680 namespace.go:98] [openshift-dns-operator] adding namespace I0424 15:05:30.516274 3680 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.224892ms I0424 15:05:30.516285 3680 namespace.go:98] [openshift-network-diagnostics] adding namespace I0424 15:05:30.516708 3680 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.195782ms I0424 15:05:30.516720 3680 namespace.go:98] [openshift-network-operator] adding namespace I0424 15:05:30.517153 3680 namespace.go:102] [openshift-authentication] adding namespace took 7.094226ms I0424 15:05:30.517165 3680 namespace.go:98] [openshift-monitoring] adding namespace I0424 15:05:30.517580 3680 namespace.go:102] [openshift-apiserver] adding namespace took 6.981919ms I0424 15:05:30.517592 3680 namespace.go:98] [openshift-console-user-settings] adding namespace I0424 15:05:30.518017 3680 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.958036ms I0424 15:05:30.518028 3680 namespace.go:98] [openshift-package-operator] adding namespace I0424 15:05:30.518367 3680 namespace.go:102] [openshift-config-operator] adding namespace took 6.77711ms I0424 15:05:30.518376 3680 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0424 15:05:30.518853 3680 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.682806ms I0424 15:05:30.518866 3680 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0424 15:05:30.519324 3680 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.766866ms I0424 15:05:30.519338 3680 namespace.go:98] [kube-node-lease] adding namespace I0424 15:05:30.519690 3680 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.661776ms I0424 15:05:30.519709 3680 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0424 15:05:30.520125 3680 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.725569ms I0424 15:05:30.520136 3680 namespace.go:98] [dedicated-admin] adding namespace I0424 15:05:30.520499 3680 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.706077ms I0424 15:05:30.520510 3680 namespace.go:98] [openshift-dns] adding namespace I0424 15:05:30.520886 3680 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.587079ms I0424 15:05:30.520897 3680 namespace.go:98] [openshift-backplane-cee] adding namespace I0424 15:05:30.521298 3680 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.523166ms I0424 15:05:30.521310 3680 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0424 15:05:30.521690 3680 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.404366ms I0424 15:05:30.521701 3680 namespace.go:98] [openshift-backplane-srep] adding namespace W0424 15:05:30.522076 3680 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-gktv6: pod openshift-network-diagnostics/network-check-target-gktv6: no pod IPs found I0424 15:05:30.522122 3680 namespace.go:102] [openshift-dns-operator] adding namespace took 6.450035ms I0424 15:05:30.522135 3680 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0424 15:05:30.522614 3680 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.324485ms I0424 15:05:30.522626 3680 namespace.go:98] [openshift-operators] adding namespace I0424 15:05:30.523013 3680 namespace.go:102] [openshift-network-operator] adding namespace took 6.287774ms I0424 15:05:30.523026 3680 namespace.go:98] [openshift-kube-scheduler] adding namespace I0424 15:05:30.523384 3680 namespace.go:102] [openshift-monitoring] adding namespace took 6.214874ms I0424 15:05:30.523394 3680 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0424 15:05:30.523751 3680 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.154635ms I0424 15:05:30.523762 3680 namespace.go:98] [kube-public] adding namespace I0424 15:05:30.524144 3680 namespace.go:102] [openshift-package-operator] adding namespace took 6.112186ms I0424 15:05:30.524155 3680 namespace.go:98] [openshift-must-gather-operator] adding namespace I0424 15:05:30.524511 3680 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.130198ms I0424 15:05:30.524875 3680 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.004464ms I0424 15:05:30.524886 3680 namespace.go:98] [openshift-network-node-identity] adding namespace I0424 15:05:30.525329 3680 namespace.go:102] [kube-node-lease] adding namespace took 5.986396ms I0424 15:05:30.525343 3680 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0424 15:05:30.525767 3680 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.053259ms I0424 15:05:30.525780 3680 namespace.go:98] [openshift-logging] adding namespace I0424 15:05:30.526177 3680 namespace.go:102] [dedicated-admin] adding namespace took 6.036919ms I0424 15:05:30.526190 3680 namespace.go:98] [openshift-machine-config-operator] adding namespace I0424 15:05:30.526566 3680 namespace.go:102] [openshift-dns] adding namespace took 6.050485ms I0424 15:05:30.526582 3680 namespace.go:98] [openshift-console-operator] adding namespace I0424 15:05:30.526880 3680 namespace.go:102] [openshift-backplane-cee] adding namespace took 5.979326ms I0424 15:05:30.526891 3680 namespace.go:98] [open-cluster-management-2psja0jgf3iq01e6l9t02c7pqkonai87] adding namespace I0424 15:05:30.527457 3680 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.142243ms I0424 15:05:30.527469 3680 namespace.go:98] [openshift-node] adding namespace I0424 15:05:30.528056 3680 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.349793ms I0424 15:05:30.528072 3680 namespace.go:98] [openshift-insights] adding namespace I0424 15:05:30.528454 3680 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.314384ms I0424 15:05:30.528465 3680 namespace.go:98] [openshift-multus] adding namespace I0424 15:05:30.528894 3680 namespace.go:102] [openshift-operators] adding namespace took 6.263581ms I0424 15:05:30.528908 3680 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0424 15:05:30.529420 3680 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.388822ms I0424 15:05:30.529434 3680 namespace.go:98] [openshift-ingress-canary] adding namespace I0424 15:05:30.529913 3680 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.512154ms I0424 15:05:30.529944 3680 namespace.go:98] [openshift-ingress] adding namespace I0424 15:05:30.530337 3680 namespace.go:102] [kube-public] adding namespace took 6.569655ms I0424 15:05:30.530353 3680 namespace.go:98] [openshift-customer-monitoring] adding namespace I0424 15:05:30.530726 3680 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.566781ms I0424 15:05:30.530737 3680 namespace.go:98] [openshift-backplane] adding namespace I0424 15:05:30.531169 3680 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.277927ms I0424 15:05:30.531560 3680 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.211134ms I0424 15:05:30.531572 3680 namespace.go:98] [openshift-ingress-operator] adding namespace I0424 15:05:30.531945 3680 namespace.go:102] [openshift-logging] adding namespace took 6.16138ms I0424 15:05:30.532383 3680 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.187812ms I0424 15:05:30.532439 3680 iptables.go:108] Creating table: filter chain: FORWARD I0424 15:05:30.532834 3680 namespace.go:102] [openshift-console-operator] adding namespace took 6.245102ms I0424 15:05:30.532848 3680 namespace.go:98] [openshift-infra] adding namespace I0424 15:05:30.533254 3680 namespace.go:102] [open-cluster-management-2psja0jgf3iq01e6l9t02c7pqkonai87] adding namespace took 6.356623ms I0424 15:05:30.533580 3680 namespace.go:102] [openshift-node] adding namespace took 6.107255ms W0424 15:05:30.534003 3680 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-sjxwf: pod openshift-multus/network-metrics-daemon-sjxwf: no pod IPs found I0424 15:05:30.534048 3680 namespace.go:102] [openshift-insights] adding namespace took 5.971644ms I0424 15:05:30.534471 3680 namespace.go:102] [openshift-multus] adding namespace took 6.002134ms I0424 15:05:30.534849 3680 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 5.936782ms I0424 15:05:30.535223 3680 namespace.go:102] [openshift-ingress-canary] adding namespace took 5.783121ms I0424 15:05:30.535236 3680 namespace.go:98] [openshift-marketplace] adding namespace I0424 15:05:30.535654 3680 namespace.go:102] [openshift-ingress] adding namespace took 5.705619ms I0424 15:05:30.536032 3680 namespace.go:102] [openshift-customer-monitoring] adding namespace took 5.674259ms I0424 15:05:30.536664 3680 namespace.go:102] [openshift-backplane] adding namespace took 5.921488ms I0424 15:05:30.537084 3680 namespace.go:102] [openshift-ingress-operator] adding namespace took 5.507423ms I0424 15:05:30.537457 3680 namespace.go:102] [openshift-infra] adding namespace took 4.604229ms I0424 15:05:30.537803 3680 namespace.go:102] [openshift-marketplace] adding namespace took 2.562553ms W0424 15:05:30.537883 3680 master.go:458] Unable to parse local node ip-10-0-1-253.ec2.internal chassis-id annotation. Chassis may be removed during sync I0424 15:05:30.537895 3680 master.go:481] Removing stale chassis with ID/Name: f0c88587-d27e-4e2c-a156-66ee2fec42ae, hostname: ip-10-0-1-253.ec2.internal I0424 15:05:30.538359 3680 master.go:548] Adding or Updating local node "ip-10-0-1-253.ec2.internal" for network "default" I0424 15:05:30.539319 3680 iptables.go:108] Creating table: filter chain: OUTPUT I0424 15:05:30.539477 3680 zone_ic_handler.go:195] Time taken to create transit switch: 263.963µs I0424 15:05:30.539489 3680 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-196.ec2.internal for the network default E0424 15:05:30.539859 3680 obj_retry.go:575] Failed to create *v1.Node ip-10-0-2-93.ec2.internal, error: adding or updating remote node IC resources ip-10-0-2-93.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-2-93.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc0092f1eb8 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u3082904809}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0424 15:05:30.539962 3680 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-2-93.ec2.internal", UID:"e5023fde-7b88-4917-84f0-b1fddedf5ccf", APIVersion:"v1", ResourceVersion:"8918", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' adding or updating remote node IC resources ip-10-0-2-93.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-2-93.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc0092f1eb8 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u3082904809}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0424 15:05:30.540425 3680 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-196.ec2.internal" on network "default" took: 1.223367ms I0424 15:05:30.542301 3680 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-253.ec2.internal for the network default I0424 15:05:30.542511 3680 zone_ic_handler.go:195] Time taken to create transit switch: 197.278µs I0424 15:05:30.543270 3680 default_network_controller.go:823] Node add failed for ip-10-0-1-253.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-253.ec2.internal, error getting gateway config for node ip-10-0-1-253.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-253.ec2.internal", failed to update chassis to local for local node ip-10-0-1-253.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-253.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-253.ec2.internal] E0424 15:05:30.543319 3680 obj_retry.go:575] Failed to create *v1.Node ip-10-0-1-253.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-253.ec2.internal, error getting gateway config for node ip-10-0-1-253.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-253.ec2.internal", failed to update chassis to local for local node ip-10-0-1-253.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-253.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-253.ec2.internal] I0424 15:05:30.543363 3680 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-1-253.ec2.internal", UID:"8eb25bd1-13e2-40f1-a607-fca06c5a66d8", APIVersion:"v1", ResourceVersion:"8917", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-253.ec2.internal, error getting gateway config for node ip-10-0-1-253.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-253.ec2.internal", failed to update chassis to local for local node ip-10-0-1-253.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-253.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-253.ec2.internal] I0424 15:05:30.543384 3680 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0424 15:05:30.543407 3680 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0424 15:05:30.543421 3680 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0424 15:05:30.543440 3680 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-196.ec2.internal in network "default" I0424 15:05:30.543499 3680 node_tracker.go:169] Node ip-10-0-0-196.ec2.internal switch + router changed, syncing services in network "default" I0424 15:05:30.543516 3680 services_controller.go:574] Full service sync requested for network=default I0424 15:05:30.543523 3680 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-253.ec2.internal in network "default" I0424 15:05:30.543532 3680 node_tracker.go:235] Node ip-10-0-1-253.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-253.ec2.internal" I0424 15:05:30.543539 3680 node_tracker.go:169] Node ip-10-0-1-253.ec2.internal switch + router changed, syncing services in network "default" I0424 15:05:30.543543 3680 services_controller.go:574] Full service sync requested for network=default I0424 15:05:30.543549 3680 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-93.ec2.internal in network "default" I0424 15:05:30.543570 3680 node_tracker.go:169] Node ip-10-0-2-93.ec2.internal switch + router changed, syncing services in network "default" I0424 15:05:30.543577 3680 services_controller.go:574] Full service sync requested for network=default I0424 15:05:30.546435 3680 gateway_shared_intf.go:1816] Gateway Creation Complete I0424 15:05:30.546449 3680 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-253.ec2.internal","mac-address":"0a:ff:fa:07:d8:ef","ip-addresses":["10.0.1.253/24"],"ip-address":"10.0.1.253/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:f0c88587-d27e-4e2c-a156-66ee2fec42ae k8s.ovn.org/node-encap-ips:["10.0.1.253"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.253/24"} k8s.ovn.org/zone-name:ip-10-0-1-253.ec2.internal] on node ip-10-0-1-253.ec2.internal I0424 15:05:30.563458 3680 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-253.ec2.internal in network "default" I0424 15:05:30.563490 3680 master.go:548] Adding or Updating local node "ip-10-0-1-253.ec2.internal" for network "default" I0424 15:05:30.563514 3680 node_tracker.go:169] Node ip-10-0-1-253.ec2.internal switch + router changed, syncing services in network "default" I0424 15:05:30.563530 3680 services_controller.go:574] Full service sync requested for network=default I0424 15:05:30.567436 3680 default_node_network_controller.go:519] Node connection status = connected W0424 15:05:30.568291 3680 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-sjxwf: pod openshift-multus/network-metrics-daemon-sjxwf: no pod IPs found W0424 15:05:30.568314 3680 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-gktv6: pod openshift-network-diagnostics/network-check-target-gktv6: no pod IPs found I0424 15:05:30.569229 3680 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-253.ec2.internal for the network default I0424 15:05:30.569377 3680 zone_ic_handler.go:195] Time taken to create transit switch: 137.694µs I0424 15:05:30.644340 3680 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0424 15:05:30.644360 3680 services_controller.go:208] Setting up event handlers for services for network=default I0424 15:05:30.644451 3680 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0424 15:05:30.644508 3680 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0424 15:05:30.644529 3680 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0424 15:05:30.745248 3680 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0424 15:05:30.745265 3680 repair.go:60] Starting repairing loop for services I0424 15:05:30.745354 3680 repair.go:131] Deleted 0 stale service LBs I0424 15:05:30.745372 3680 repair.go:137] Deleted 0 stale Chassis Template Vars I0424 15:05:30.745387 3680 repair.go:62] Finished repairing loop for services: 123.343µs I0424 15:05:30.745416 3680 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0424 15:05:30.745426 3680 services_controller.go:254] Starting workers for network=default I0424 15:05:30.745905 3680 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-sjxwf] creating logical port openshift-multus_network-metrics-daemon-sjxwf for pod on switch ip-10-0-1-253.ec2.internal I0424 15:05:30.745905 3680 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-gktv6] creating logical port openshift-network-diagnostics_network-check-target-gktv6 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:05:30.746063 3680 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-sjxwf I0424 15:05:30.746135 3680 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-gktv6 I0424 15:05:30.768284 3680 pod.go:62] [openshift-network-diagnostics/network-check-target-gktv6] pod update took 22.197124ms I0424 15:05:30.768305 3680 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-gktv6] addLogicalPort annotation time took 22.216598ms I0424 15:05:30.768476 3680 pod.go:62] [openshift-multus/network-metrics-daemon-sjxwf] pod update took 22.473809ms I0424 15:05:30.768491 3680 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-sjxwf] addLogicalPort annotation time took 22.490845ms I0424 15:05:30.769069 3680 pods.go:271] [openshift-network-diagnostics/network-check-target-gktv6] addLogicalPort took 23.174763ms, libovsdb time 565.692µs I0424 15:05:30.769332 3680 pods.go:271] [openshift-multus/network-metrics-daemon-sjxwf] addLogicalPort took 23.432939ms, libovsdb time 674.801µs I0424 15:05:30.769772 3680 admin_network_policy_controller.go:227] Starting controller default-network-controller I0424 15:05:30.769794 3680 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0424 15:05:30.769809 3680 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0424 15:05:30.769813 3680 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0424 15:05:30.770012 3680 repair.go:29] Repairing admin network policies took 192.532µs I0424 15:05:30.770130 3680 repair.go:92] Repairing baseline admin network policies took 107.54µs I0424 15:05:30.770431 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0424 15:05:30.770451 3680 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0424 15:05:30.772122 3680 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0424 15:05:30.772141 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0424 15:05:30.772526 3680 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0424 15:05:30.772845 3680 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0424 15:05:30.772863 3680 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0424 15:05:30.772871 3680 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0424 15:05:30.774074 3680 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0424 15:05:30.774099 3680 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0424 15:05:30.774109 3680 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0424 15:05:30.776031 3680 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0424 15:05:30.776050 3680 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0424 15:05:30.776059 3680 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0424 15:05:30.777144 3680 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0424 15:05:30.777163 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0424 15:05:30.777169 3680 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0424 15:05:30.778373 3680 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0424 15:05:30.778389 3680 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0424 15:05:30.778398 3680 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0424 15:05:30.779310 3680 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0424 15:05:30.779333 3680 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0424 15:05:30.779344 3680 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0424 15:05:30.780893 3680 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0424 15:05:30.780911 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0424 15:05:30.780937 3680 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0424 15:05:30.782298 3680 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0424 15:05:30.782315 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0424 15:05:30.782324 3680 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0424 15:05:30.782758 3680 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0424 15:05:30.782780 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0424 15:05:30.782790 3680 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0424 15:05:30.783266 3680 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0424 15:05:30.783295 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0424 15:05:30.784060 3680 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0424 15:05:30.784074 3680 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0424 15:05:30.784703 3680 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0424 15:05:30.784718 3680 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0424 15:05:30.784725 3680 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0424 15:05:30.786578 3680 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0424 15:05:30.786596 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0424 15:05:30.786605 3680 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0424 15:05:30.787273 3680 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0424 15:05:30.787294 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0424 15:05:30.787306 3680 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0424 15:05:30.787784 3680 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0424 15:05:30.787801 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0424 15:05:30.787827 3680 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0424 15:05:30.788454 3680 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0424 15:05:30.788477 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0424 15:05:30.788486 3680 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0424 15:05:30.789694 3680 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0424 15:05:30.789712 3680 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0424 15:05:30.789720 3680 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0424 15:05:30.790221 3680 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0424 15:05:30.790238 3680 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0424 15:05:30.790250 3680 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0424 15:05:30.791059 3680 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0424 15:05:30.791076 3680 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0424 15:05:30.791083 3680 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0424 15:05:30.791568 3680 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0424 15:05:30.791585 3680 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0424 15:05:30.791594 3680 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0424 15:05:30.792497 3680 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0424 15:05:30.792513 3680 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0424 15:05:30.792521 3680 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0424 15:05:30.793323 3680 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0424 15:05:30.793342 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0424 15:05:30.793349 3680 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0424 15:05:30.793763 3680 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0424 15:05:30.793778 3680 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0424 15:05:30.793783 3680 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0424 15:05:30.794066 3680 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0424 15:05:30.794082 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0424 15:05:30.794089 3680 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0424 15:05:30.794530 3680 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0424 15:05:30.794544 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0424 15:05:30.794551 3680 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0424 15:05:30.795006 3680 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0424 15:05:30.795019 3680 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0424 15:05:30.795029 3680 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0424 15:05:30.796541 3680 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0424 15:05:30.796559 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0424 15:05:30.796566 3680 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0424 15:05:30.796996 3680 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0424 15:05:30.797012 3680 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0424 15:05:30.797022 3680 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0424 15:05:30.797486 3680 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0424 15:05:30.802411 3680 egressfirewall.go:374] Starting EgressFirewall controller I0424 15:05:30.802441 3680 controller.go:133] Adding controller egress-firewall-controller event handlers I0424 15:05:30.802460 3680 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0424 15:05:30.802464 3680 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0424 15:05:30.802471 3680 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0424 15:05:30.802490 3680 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0424 15:05:30.802499 3680 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0424 15:05:30.802727 3680 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0424 15:05:30.802741 3680 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0424 15:05:30.802746 3680 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0424 15:05:30.802755 3680 egressqos.go:193] Setting up event handlers for EgressQoS I0424 15:05:30.802825 3680 egressfirewall.go:723] Syncing node "ip-10-0-0-196.ec2.internal" for egress firewall I0424 15:05:30.802831 3680 egressqos.go:245] Starting EgressQoS Controller I0424 15:05:30.802834 3680 egressfirewall.go:723] Syncing node "ip-10-0-1-253.ec2.internal" for egress firewall I0424 15:05:30.802838 3680 egressfirewall.go:723] Syncing node "ip-10-0-2-93.ec2.internal" for egress firewall I0424 15:05:30.802841 3680 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0424 15:05:30.802848 3680 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0424 15:05:30.802856 3680 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0424 15:05:30.802861 3680 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0424 15:05:30.802866 3680 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0424 15:05:30.802870 3680 shared_informer.go:356] "Caches are synced" controller="egressqos" I0424 15:05:30.802874 3680 egressqos.go:259] Repairing EgressQoSes I0424 15:05:30.802878 3680 egressqos.go:399] Starting repairing loop for egressqos I0424 15:05:30.802966 3680 egressqos.go:401] Finished repairing loop for egressqos: 87.773µs I0424 15:05:30.802991 3680 egressservice_zone.go:132] Setting up event handlers for Egress Services I0424 15:05:30.803021 3680 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-253.ec2.internal I0424 15:05:30.803059 3680 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-253.ec2.internal : 39.328µs I0424 15:05:30.803118 3680 egressservice_zone.go:210] Starting Egress Services Controller I0424 15:05:30.803134 3680 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0424 15:05:30.803145 3680 shared_informer.go:356] "Caches are synced" controller="egressservices" I0424 15:05:30.803153 3680 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0424 15:05:30.803158 3680 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0424 15:05:30.803163 3680 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0424 15:05:30.803167 3680 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0424 15:05:30.803172 3680 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0424 15:05:30.803176 3680 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0424 15:05:30.803180 3680 egressservice_zone.go:228] Repairing Egress Services I0424 15:05:30.803442 3680 master_controller.go:88] Starting Admin Policy Based Route Controller I0424 15:05:30.803453 3680 external_controller.go:278] Starting Admin Policy Based Route Controller I0424 15:05:30.803527 3680 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-196.ec2.internal I0424 15:05:30.803541 3680 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-196.ec2.internal: 14.675µs I0424 15:05:30.803546 3680 default_network_controller.go:585] Completing all the Watchers took 301.24201ms I0424 15:05:30.803553 3680 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-253.ec2.internal I0424 15:05:30.803556 3680 default_network_controller.go:589] Starting unidling controllers I0424 15:05:30.803559 3680 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-253.ec2.internal: 7.706µs I0424 15:05:30.803566 3680 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-93.ec2.internal I0424 15:05:30.803568 3680 unidle.go:46] Registering OVN SB ControllerEvent handler I0424 15:05:30.803572 3680 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-93.ec2.internal: 5.818µs I0424 15:05:30.803574 3680 unidle.go:63] Populating Initial ContollerEvent events I0424 15:05:30.803595 3680 unidle.go:79] Setting up event handlers for services I0424 15:05:30.803656 3680 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0424 15:05:31.067784 3680 default_node_network_controller.go:519] Node connection status = connected I0424 15:05:31.567811 3680 default_node_network_controller.go:519] Node connection status = connected I0424 15:05:32.068698 3680 default_node_network_controller.go:519] Node connection status = connected I0424 15:05:32.567936 3680 default_node_network_controller.go:519] Node connection status = connected I0424 15:05:32.939059 3680 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0424 15:05:33.067883 3680 default_node_network_controller.go:519] Node connection status = connected I0424 15:05:33.084644 3680 bridgeconfig.go:537] Gateway is ready I0424 15:05:33.084659 3680 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0424 15:05:33.095481 3680 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.253. Updating OVN encap IP. I0424 15:05:33.100456 3680 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.253 - it is already configured I0424 15:05:33.100468 3680 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.253/24:{} 172.20.0.1/32:{}]. Updating annotations. I0424 15:05:33.100660 3680 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.253/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-253.ec2.internal","mac-address":"0a:ff:fa:07:d8:ef","ip-addresses":["10.0.1.253/24"],"ip-address":"10.0.1.253/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:f0c88587-d27e-4e2c-a156-66ee2fec42ae k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.253/24"}] on node ip-10-0-1-253.ec2.internal I0424 15:05:33.117871 3680 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-253.ec2.internal I0424 15:05:33.117887 3680 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-253.ec2.internal: 18.831µs I0424 15:05:33.117913 3680 egressfirewall.go:723] Syncing node "ip-10-0-1-253.ec2.internal" for egress firewall I0424 15:05:33.117987 3680 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-253.ec2.internal in network "default" I0424 15:05:33.118049 3680 master.go:548] Adding or Updating local node "ip-10-0-1-253.ec2.internal" for network "default" I0424 15:05:33.118056 3680 node_tracker.go:169] Node ip-10-0-1-253.ec2.internal switch + router changed, syncing services in network "default" I0424 15:05:33.118077 3680 services_controller.go:574] Full service sync requested for network=default I0424 15:05:33.118180 3680 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.1.253 I0424 15:05:33.118273 3680 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-253.ec2.internal I0424 15:05:33.122306 3680 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-253.ec2.internal. Updating no re-route policies I0424 15:05:33.122379 3680 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-gktv6 I0424 15:05:33.122386 3680 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-253.ec2.internal I0424 15:05:33.122390 3680 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-4vxml I0424 15:05:33.122395 3680 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-gktv6 I0424 15:05:33.122398 3680 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-253.ec2.internal I0424 15:05:33.122402 3680 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-4vxml I0424 15:05:33.122403 3680 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-4q55g I0424 15:05:33.122408 3680 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-4xvrh I0424 15:05:33.122416 3680 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-253.ec2.internal after 0 failed attempt(s) I0424 15:05:33.122419 3680 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-4xvrh I0424 15:05:33.122417 3680 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-4vxml after 0 failed attempt(s) I0424 15:05:33.122424 3680 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-gqhkt I0424 15:05:33.122417 3680 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-4q55g I0424 15:05:33.122431 3680 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-gktv6] creating logical port openshift-network-diagnostics_network-check-target-gktv6 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:05:33.122430 3680 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-jvgqf I0424 15:05:33.122436 3680 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-gqhkt I0424 15:05:33.122430 3680 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-sjxwf I0424 15:05:33.122440 3680 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-jvgqf I0424 15:05:33.122442 3680 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-4q55g after 0 failed attempt(s) I0424 15:05:33.122445 3680 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-sjxwf I0424 15:05:33.122444 3680 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-pjpc6 I0424 15:05:33.122449 3680 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-ckzws I0424 15:05:33.122451 3680 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-gqhkt after 0 failed attempt(s) I0424 15:05:33.122457 3680 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-jvgqf after 0 failed attempt(s) I0424 15:05:33.122450 3680 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-x42pq I0424 15:05:33.122464 3680 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-ckzws I0424 15:05:33.122470 3680 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-x42pq I0424 15:05:33.122481 3680 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-sjxwf] creating logical port openshift-multus_network-metrics-daemon-sjxwf for pod on switch ip-10-0-1-253.ec2.internal I0424 15:05:33.122381 3680 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-k8rdm I0424 15:05:33.122423 3680 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-253.ec2.internal I0424 15:05:33.122537 3680 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-k8rdm I0424 15:05:33.122537 3680 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-253.ec2.internal I0424 15:05:33.122549 3680 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-k8rdm after 0 failed attempt(s) I0424 15:05:33.122463 3680 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-pjpc6 I0424 15:05:33.122554 3680 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-253.ec2.internal after 0 failed attempt(s) I0424 15:05:33.122562 3680 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-pjpc6 after 0 failed attempt(s) I0424 15:05:33.122485 3680 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-ckzws after 0 failed attempt(s) I0424 15:05:33.122994 3680 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-4xvrh after 0 failed attempt(s) I0424 15:05:33.123011 3680 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-x42pq after 0 failed attempt(s) I0424 15:05:33.123075 3680 pods.go:271] [openshift-network-diagnostics/network-check-target-gktv6] addLogicalPort took 653.346µs, libovsdb time 237.23µs I0424 15:05:33.123085 3680 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-gktv6 after 0 failed attempt(s) I0424 15:05:33.123245 3680 pods.go:271] [openshift-multus/network-metrics-daemon-sjxwf] addLogicalPort took 777.591µs, libovsdb time 200.169µs I0424 15:05:33.123253 3680 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-sjxwf after 0 failed attempt(s) I0424 15:05:33.134553 3680 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0424 15:05:33.146149 3680 iptables.go:108] Creating table: mangle chain: OUTPUT I0424 15:05:33.149649 3680 iptables.go:108] Creating table: nat chain: OUTPUT I0424 15:05:33.177988 3680 iptables.go:108] Creating table: nat chain: PREROUTING I0424 15:05:33.229874 3680 iptables.go:108] Creating table: filter chain: FORWARD I0424 15:05:33.234491 3680 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-196.ec2.internal I0424 15:05:33.234518 3680 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-196.ec2.internal: 28.298µs I0424 15:05:33.247775 3680 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0424 15:05:33.247885 3680 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0424 15:05:33.247956 3680 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0424 15:05:33.248054 3680 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0424 15:05:33.248112 3680 gateway_init.go:325] Gateway and management port readiness took 2.684875798s I0424 15:05:33.248137 3680 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0424 15:05:33.248170 3680 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0424 15:05:33.248183 3680 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0424 15:05:33.248192 3680 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0424 15:05:33.248246 3680 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0424 15:05:33.248333 3680 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 62.703µs I0424 15:05:33.248610 3680 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0424 15:05:33.249022 3680 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 401.342µs I0424 15:05:33.249032 3680 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 422.651µs I0424 15:05:33.249038 3680 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0424 15:05:33.249047 3680 nad_controller.go:379] [node-nad-controller NAD controller]: started I0424 15:05:33.249052 3680 default_node_network_controller.go:899] Starting the default node network controller I0424 15:05:33.249057 3680 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0424 15:05:33.249139 3680 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 12.078µs I0424 15:05:33.249206 3680 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0424 15:05:33.249215 3680 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0424 15:05:33.249240 3680 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c24550 0xc0002ef730 0xc0002ef7a0 0xc0002ef810 0xc0002ef8f0 0xc0002ef960 0xc0002ef9d0 0xc000c7e190 0xc0002efa40 0xc0002efab0 0xc0008b0dc0 0xc006527340 0xc0002efb20 map[0x2ff9080:0xc0014236e0 0x2ff95c0:0xc001423860 0x2ffa2e0:0xc002618480 0x2ffa580:0xc001423680 0x2ffad60:0xc001423740 0x2ffb000:0xc0014237a0 0x2ffb2a0:0xc001423800 0x2ffba80:0xc0014238c0 0x2ffbd20:0xc004e8e4e0 0x3026980:0xc000f62c00 0x3026d20:0xc000f62c60 0x3027800:0xc000c52b40 0x3029160:0xc001bc4000 0x30298a0:0xc002f838c0 0x302a720:0xc001882780] 0xc0000d2770 0} I0424 15:05:33.249296 3680 gateway.go:151] Starting gateway service sync I0424 15:05:33.249663 3680 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0424 15:05:33.252072 3680 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0424 15:05:33.263264 3680 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0424 15:05:33.264496 3680 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0424 15:05:33.265799 3680 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0424 15:05:33.300637 3680 gateway.go:168] Gateway service sync done. Time taken: 51.328328ms I0424 15:05:33.301179 3680 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31903 I0424 15:05:33.301602 3680 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c24550 0xc0002ef730 0xc0002ef7a0 0xc0002ef810 0xc0002ef8f0 0xc0002ef960 0xc0002ef9d0 0xc000c7e190 0xc0002efa40 0xc0002efab0 0xc0008b0dc0 0xc006527340 0xc0002efb20 map[0x2ff9080:0xc0014236e0 0x2ff95c0:0xc001423860 0x2ffa2e0:0xc002618480 0x2ffa580:0xc001423680 0x2ffad60:0xc001423740 0x2ffb000:0xc0014237a0 0x2ffb2a0:0xc001423800 0x2ffba80:0xc0014238c0 0x2ffbd20:0xc004e8e4e0 0x3026980:0xc000f62c00 0x3026d20:0xc000f62c60 0x3027800:0xc000c52b40 0x3029160:0xc001bc4000 0x30298a0:0xc002f838c0 0x302a720:0xc001882780] 0xc0000d2770 0} I0424 15:05:33.302228 3680 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c24550 0xc0002ef730 0xc0002ef7a0 0xc0002ef810 0xc0002ef8f0 0xc0002ef960 0xc0002ef9d0 0xc000c7e190 0xc0002efa40 0xc0002efab0 0xc0008b0dc0 0xc006527340 0xc0002efb20 map[0x2ff9080:0xc0014236e0 0x2ff95c0:0xc001423860 0x2ffa2e0:0xc002618480 0x2ffa580:0xc001423680 0x2ffad60:0xc001423740 0x2ffb000:0xc0014237a0 0x2ffb2a0:0xc001423800 0x2ffba80:0xc0014238c0 0x2ffbd20:0xc004e8e4e0 0x3026980:0xc000f62c00 0x3026d20:0xc000f62c60 0x3027800:0xc000c52b40 0x3029160:0xc001bc4000 0x30298a0:0xc002f838c0 0x302a720:0xc001882780] 0xc0000d2770 0} I0424 15:05:33.302278 3680 gateway.go:532] Reconciling gateway with updates I0424 15:05:33.307821 3680 gateway.go:339] Spawning Conntrack Rule Check Thread I0424 15:05:33.307913 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0424 15:05:33.307930 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0424 15:05:33.307949 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0424 15:05:33.307948 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0424 15:05:33.307948 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0424 15:05:33.307953 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0424 15:05:33.307962 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0424 15:05:33.307963 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0424 15:05:33.307971 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0424 15:05:33.307978 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0424 15:05:33.307978 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0424 15:05:33.307983 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0424 15:05:33.307990 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0424 15:05:33.307954 3680 default_node_network_controller.go:944] Gateway and management port readiness took 58.730961ms I0424 15:05:33.307999 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0424 15:05:33.308003 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0424 15:05:33.308013 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0424 15:05:33.308017 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0424 15:05:33.308022 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0424 15:05:33.308023 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0424 15:05:33.307951 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0424 15:05:33.308034 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0424 15:05:33.308035 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0424 15:05:33.308039 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0424 15:05:33.308042 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0424 15:05:33.308040 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0424 15:05:33.308048 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0424 15:05:33.308054 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0424 15:05:33.308055 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0424 15:05:33.308062 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0424 15:05:33.308065 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0424 15:05:33.308070 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0424 15:05:33.308072 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0424 15:05:33.308024 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0424 15:05:33.308014 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0424 15:05:33.308078 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0424 15:05:33.308082 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0424 15:05:33.308011 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0424 15:05:33.308065 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0424 15:05:33.308098 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0424 15:05:33.308108 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0424 15:05:33.308110 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0424 15:05:33.308121 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0424 15:05:33.308124 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0424 15:05:33.308127 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0424 15:05:33.308130 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0424 15:05:33.308121 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0424 15:05:33.308137 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0424 15:05:33.308096 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0424 15:05:33.308137 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0424 15:05:33.308146 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0424 15:05:33.308124 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0424 15:05:33.308152 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0424 15:05:33.308155 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0424 15:05:33.308156 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0424 15:05:33.308160 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0424 15:05:33.307987 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0424 15:05:33.308171 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0424 15:05:33.308098 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0424 15:05:33.308176 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0424 15:05:33.308076 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0424 15:05:33.308183 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0424 15:05:33.308186 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0424 15:05:33.308183 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0424 15:05:33.308190 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0424 15:05:33.308190 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0424 15:05:33.307998 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0424 15:05:33.308189 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0424 15:05:33.308195 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0424 15:05:33.308200 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0424 15:05:33.308200 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0424 15:05:33.308205 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0424 15:05:33.308205 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0424 15:05:33.308214 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0424 15:05:33.308222 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0424 15:05:33.308224 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0424 15:05:33.308225 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0424 15:05:33.308226 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0424 15:05:33.308229 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0424 15:05:33.308235 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0424 15:05:33.308231 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0424 15:05:33.308241 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0424 15:05:33.308241 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0424 15:05:33.308111 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0424 15:05:33.308286 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0424 15:05:33.308297 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0424 15:05:33.308087 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0424 15:05:33.307956 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0424 15:05:33.308317 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0424 15:05:33.308085 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0424 15:05:33.308333 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0424 15:05:33.308348 3680 port_claim.go:71] Svc openshift-ingress/router-default: port 31929 is already open, no action needed I0424 15:05:33.308357 3680 port_claim.go:71] Svc openshift-ingress/router-default: port 32694 is already open, no action needed I0424 15:05:33.308026 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0424 15:05:33.308410 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0424 15:05:33.308087 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0424 15:05:33.308172 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0424 15:05:33.308195 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0424 15:05:33.308198 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0424 15:05:33.308210 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0424 15:05:33.308526 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0424 15:05:33.308142 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0424 15:05:33.309281 3680 node_ip_handler_linux.go:160] Node IP manager is running W0424 15:05:33.354137 3680 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0424 15:05:33.354320 3680 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0424 15:05:33.354446 3680 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:82:00:01/10.130.0.1) on link ovn-k8s-mp0 I0424 15:05:33.363818 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0424 15:05:33.363847 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0424 15:05:33.363857 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0424 15:05:33.363866 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0424 15:05:33.363876 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0424 15:05:33.363885 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0424 15:05:33.406758 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0424 15:05:33.406795 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0424 15:05:33.406810 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0424 15:05:33.406823 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0424 15:05:33.406834 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0424 15:05:33.406851 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0424 15:05:33.406867 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0424 15:05:33.406889 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0424 15:05:33.406906 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0424 15:05:33.406912 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0424 15:05:33.406932 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0424 15:05:33.406939 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0424 15:05:33.406962 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0424 15:05:33.406982 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0424 15:05:33.406995 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0424 15:05:33.407003 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0424 15:05:33.407011 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0424 15:05:33.407019 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0424 15:05:33.407027 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0424 15:05:33.407034 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0424 15:05:33.407055 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0424 15:05:33.407065 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0424 15:05:33.407073 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0424 15:05:33.407080 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0424 15:05:33.449823 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0424 15:05:33.449837 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0424 15:05:33.449829 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0424 15:05:33.941076 3680 default_network_controller.go:1025] Node "ip-10-0-2-93.ec2.internal" in remote zone "ip-10-0-2-93.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0424 15:05:33.941080 3680 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-93.ec2.internal I0424 15:05:33.941236 3680 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-93.ec2.internal: 157.703µs I0424 15:05:33.941868 3680 zone_ic_handler.go:195] Time taken to create transit switch: 308.403µs I0424 15:05:33.941883 3680 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-93.ec2.internal for the network default I0424 15:05:33.942973 3680 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-93.ec2.internal" on network "default" took: 1.430271ms I0424 15:05:35.184481 3680 default_node_network_controller.go:1380] Starting node controller node sync W0424 15:05:35.184515 3680 egressip_healthcheck.go:76] Health checking using insecure connection I0424 15:05:35.184558 3680 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.130.0.2:9107 I0424 15:05:35.196674 3680 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 12.167095ms I0424 15:05:35.196798 3680 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-93.ec2.internal", IP: 10.0.2.93 to PMTUD blocking rules I0424 15:05:35.196797 3680 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-196.ec2.internal", IP: 10.0.0.196 to PMTUD blocking rules I0424 15:05:35.196812 3680 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-93.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0424 15:05:35.196818 3680 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-196.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0424 15:05:35.208481 3680 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0424 15:05:35.208800 3680 egressservice_node.go:93] Setting up event handlers for Egress Services I0424 15:05:35.208905 3680 egressservice_node.go:183] Starting Egress Services Controller I0424 15:05:35.208936 3680 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0424 15:05:35.208948 3680 shared_informer.go:356] "Caches are synced" controller="egressservices" I0424 15:05:35.208964 3680 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0424 15:05:35.208969 3680 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0424 15:05:35.208974 3680 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0424 15:05:35.208979 3680 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0424 15:05:35.208983 3680 egressservice_node.go:197] Repairing Egress Services I0424 15:05:35.218832 3680 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0424 15:05:35.218845 3680 external_controller.go:278] Starting Admin Policy Based Route Controller I0424 15:05:35.219013 3680 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0424 15:05:35.219030 3680 link_network_manager.go:119] Link manager is running I0424 15:05:35.219045 3680 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0424 15:05:35.219201 3680 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0424 15:05:35.219666 3680 vrf_manager.go:128] VRF manager is running I0424 15:05:35.219943 3680 node_controller_manager.go:437] Removing flows to drop GARP I0424 15:05:35.219953 3680 gateway.go:532] Reconciling gateway with updates I0424 15:05:35.241905 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0424 15:05:35.241915 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0424 15:05:35.241916 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0424 15:05:35.241936 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0424 15:05:35.241942 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0424 15:05:35.241945 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0424 15:05:35.241950 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0424 15:05:35.241950 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0424 15:05:35.241961 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0424 15:05:35.241963 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0424 15:05:35.241971 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0424 15:05:35.241972 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0424 15:05:35.241976 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0424 15:05:35.241983 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0424 15:05:35.241985 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0424 15:05:35.241985 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0424 15:05:35.241986 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0424 15:05:35.241995 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0424 15:05:35.241996 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0424 15:05:35.242001 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0424 15:05:35.242008 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0424 15:05:35.242010 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0424 15:05:35.242009 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0424 15:05:35.242022 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0424 15:05:35.242022 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0424 15:05:35.241963 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0424 15:05:35.242026 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0424 15:05:35.242034 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0424 15:05:35.241949 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0424 15:05:35.242042 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0424 15:05:35.242045 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0424 15:05:35.242045 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0424 15:05:35.242053 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0424 15:05:35.242055 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0424 15:05:35.242052 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0424 15:05:35.242231 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0424 15:05:35.241993 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0424 15:05:35.242244 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0424 15:05:35.242008 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0424 15:05:35.242253 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0424 15:05:35.242260 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0424 15:05:35.242261 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0424 15:05:35.242002 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0424 15:05:35.242311 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0424 15:05:35.241949 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0424 15:05:35.242050 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0424 15:05:35.241965 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0424 15:05:35.241976 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0424 15:05:35.242445 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0424 15:05:35.241975 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0424 15:05:35.242480 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0424 15:05:35.242487 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0424 15:05:35.242031 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0424 15:05:35.242028 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0424 15:05:35.242502 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0424 15:05:35.242040 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0424 15:05:35.242543 3680 port_claim.go:71] Svc openshift-ingress/router-default: port 31929 is already open, no action needed I0424 15:05:35.242554 3680 port_claim.go:71] Svc openshift-ingress/router-default: port 32694 is already open, no action needed I0424 15:05:35.242050 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0424 15:05:35.242051 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0424 15:05:35.242616 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0424 15:05:35.242059 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0424 15:05:35.242060 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0424 15:05:35.242064 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0424 15:05:35.242686 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0424 15:05:35.242067 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0424 15:05:35.242739 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0424 15:05:35.242747 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0424 15:05:35.242065 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0424 15:05:35.242755 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0424 15:05:35.242072 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0424 15:05:35.242789 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0424 15:05:35.242797 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0424 15:05:35.242075 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0424 15:05:35.242806 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0424 15:05:35.242074 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0424 15:05:35.242842 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0424 15:05:35.242081 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0424 15:05:35.242877 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0424 15:05:35.242070 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0424 15:05:35.242944 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0424 15:05:35.242076 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0424 15:05:35.242984 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0424 15:05:35.242991 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0424 15:05:35.242077 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0424 15:05:35.243000 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0424 15:05:35.242082 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0424 15:05:35.243043 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0424 15:05:35.242086 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0424 15:05:35.243079 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0424 15:05:35.242092 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0424 15:05:35.243113 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0424 15:05:35.242092 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0424 15:05:35.243148 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0424 15:05:35.242097 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0424 15:05:35.243183 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0424 15:05:35.242089 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0424 15:05:35.243238 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0424 15:05:35.242107 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0424 15:05:35.243265 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0424 15:05:35.242062 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0424 15:05:35.243329 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0424 15:05:35.242208 3680 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0424 15:05:35.243347 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0424 15:05:35.243360 3680 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0424 15:05:35.282897 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0424 15:05:35.282937 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0424 15:05:35.282964 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0424 15:05:35.282970 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0424 15:05:35.282976 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0424 15:05:35.282983 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0424 15:05:35.282989 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0424 15:05:35.282996 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0424 15:05:35.283003 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0424 15:05:35.283010 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0424 15:05:35.283016 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0424 15:05:35.325615 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0424 15:05:35.325626 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0424 15:05:35.325642 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0424 15:05:35.325648 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0424 15:05:35.325655 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0424 15:05:35.325660 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0424 15:05:35.325666 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0424 15:05:35.365656 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0424 15:05:35.365686 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0424 15:05:35.365693 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0424 15:05:35.365698 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0424 15:05:35.365704 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0424 15:05:35.365710 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0424 15:05:35.365715 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0424 15:05:35.365721 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0424 15:05:35.365729 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0424 15:05:35.365735 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0424 15:05:35.365739 3680 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0424 15:05:36.878164 3680 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-253.ec2.internal I0424 15:05:36.878189 3680 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-253.ec2.internal: 30.936µs I0424 15:05:36.910504 3680 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-d956df968-klmhh] creating logical port openshift-image-registry_image-registry-d956df968-klmhh for pod on switch ip-10-0-1-253.ec2.internal I0424 15:05:36.910603 3680 kube.go:256] Updating pod openshift-image-registry/image-registry-d956df968-klmhh I0424 15:05:36.928965 3680 pod.go:62] [openshift-image-registry/image-registry-d956df968-klmhh] pod update took 18.383743ms I0424 15:05:36.929040 3680 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-d956df968-klmhh] addLogicalPort annotation time took 18.461188ms I0424 15:05:36.930300 3680 pods.go:271] [openshift-image-registry/image-registry-d956df968-klmhh] addLogicalPort took 19.807112ms, libovsdb time 949.757µs I0424 15:05:36.932046 3680 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-5w65p] creating logical port openshift-dns_dns-default-5w65p for pod on switch ip-10-0-1-253.ec2.internal I0424 15:05:36.932128 3680 kube.go:256] Updating pod openshift-dns/dns-default-5w65p I0424 15:05:36.937941 3680 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-gl7sh] creating logical port openshift-ingress-canary_ingress-canary-gl7sh for pod on switch ip-10-0-1-253.ec2.internal I0424 15:05:36.938018 3680 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-gl7sh I0424 15:05:36.951770 3680 pod.go:62] [openshift-dns/dns-default-5w65p] pod update took 19.670791ms I0424 15:05:36.951793 3680 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-5w65p] addLogicalPort annotation time took 19.696903ms I0424 15:05:36.952843 3680 pods.go:271] [openshift-dns/dns-default-5w65p] addLogicalPort took 20.808966ms, libovsdb time 795.206µs I0424 15:05:36.958350 3680 pod.go:62] [openshift-ingress-canary/ingress-canary-gl7sh] pod update took 20.356416ms I0424 15:05:36.958377 3680 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-gl7sh] addLogicalPort annotation time took 20.385937ms I0424 15:05:36.959401 3680 pods.go:271] [openshift-ingress-canary/ingress-canary-gl7sh] addLogicalPort took 21.487783ms, libovsdb time 813.889µs I0424 15:05:37.356221 3680 cni.go:452] [openshift-ingress-canary/ingress-canary-gl7sh 268776cc1c06ea9d9bd751e81390c16b78ee4750741d1883de2e2c8aab8d21f6 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-gl7sh 268776cc1c06ea9d9bd751e81390c16b78ee4750741d1883de2e2c8aab8d21f6 network default NAD default NAD key ] I0424 15:05:37.388583 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-gl7sh, hostIfaceName: 268776cc1c06ea9, network: default, NAD default, SandboxID: "268776cc1c06ea9d9bd751e81390c16b78ee4750741d1883de2e2c8aab8d21f6", PCI device ID: , UID: "e71d3b28-c9d7-4c64-be47-2adf858e3257", MAC: 0a:58:0a:82:00:07, IPs: [10.130.0.7/23] I0424 15:05:37.436747 3680 cni.go:473] [openshift-ingress-canary/ingress-canary-gl7sh 268776cc1c06ea9d9bd751e81390c16b78ee4750741d1883de2e2c8aab8d21f6 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-gl7sh 268776cc1c06ea9d9bd751e81390c16b78ee4750741d1883de2e2c8aab8d21f6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:74:b6:3a:9c:74\",\"name\":\"268776cc1c06ea9\"},{\"mac\":\"0a:58:0a:82:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7062310d-1fc7-4e2a-bbd3-1326ad582683\"}],\"ips\":[{\"address\":\"10.130.0.7/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:05:41.049979 3680 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0424 15:05:41.700962 3680 cni.go:452] [openshift-network-diagnostics/network-check-target-gktv6 a8edda2a25dd8ef77bcbf588fc0d371218fa532d438c476a765a1cdf1d9ec5d2 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-gktv6 a8edda2a25dd8ef77bcbf588fc0d371218fa532d438c476a765a1cdf1d9ec5d2 network default NAD default NAD key ] I0424 15:05:41.710473 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-gktv6, hostIfaceName: a8edda2a25dd8ef, network: default, NAD default, SandboxID: "a8edda2a25dd8ef77bcbf588fc0d371218fa532d438c476a765a1cdf1d9ec5d2", PCI device ID: , UID: "ac369b31-a413-4d34-be45-6f8de0d460f8", MAC: 0a:58:0a:82:00:04, IPs: [10.130.0.4/23] I0424 15:05:41.756064 3680 cni.go:473] [openshift-network-diagnostics/network-check-target-gktv6 a8edda2a25dd8ef77bcbf588fc0d371218fa532d438c476a765a1cdf1d9ec5d2 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-gktv6 a8edda2a25dd8ef77bcbf588fc0d371218fa532d438c476a765a1cdf1d9ec5d2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:eb:fc:e8:8a:7e\",\"name\":\"a8edda2a25dd8ef\"},{\"mac\":\"0a:58:0a:82:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6f1ba634-0094-43ed-ab5f-c6bfae34e43a\"}],\"ips\":[{\"address\":\"10.130.0.4/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:05:42.053872 3680 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0424 15:05:45.785395 3680 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0424 15:06:05.797524 3680 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0424 15:06:12.055813 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0424 15:06:12.055844 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0424 15:06:16.793084 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0424 15:06:42.218261 3680 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-chtnp] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-chtnp for pod on switch ip-10-0-1-253.ec2.internal I0424 15:06:42.218392 3680 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-chtnp I0424 15:06:42.231745 3680 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-rffkv] creating logical port openshift-console-operator_console-operator-66dbbb6468-rffkv for pod on switch ip-10-0-1-253.ec2.internal I0424 15:06:42.231865 3680 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-rffkv I0424 15:06:42.248224 3680 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-chtnp] pod update took 29.860762ms I0424 15:06:42.248250 3680 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-chtnp] addLogicalPort annotation time took 29.890235ms I0424 15:06:42.249293 3680 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-chtnp] addLogicalPort took 31.075103ms, libovsdb time 744.883µs I0424 15:06:42.252603 3680 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-nqfwt] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-nqfwt for pod on switch ip-10-0-1-253.ec2.internal I0424 15:06:42.252677 3680 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-nqfwt I0424 15:06:42.261062 3680 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-rffkv] pod update took 29.219537ms I0424 15:06:42.261087 3680 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-rffkv] addLogicalPort annotation time took 29.249678ms I0424 15:06:42.262099 3680 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-rffkv] addLogicalPort took 30.369742ms, libovsdb time 745.56µs I0424 15:06:42.274535 3680 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-nqfwt] pod update took 21.878442ms I0424 15:06:42.274556 3680 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-nqfwt] addLogicalPort annotation time took 21.899519ms I0424 15:06:42.275393 3680 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-nqfwt] addLogicalPort took 22.79922ms, libovsdb time 634.91µs I0424 15:06:42.531128 3680 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-6484bbc7f4-kq8zm] creating logical port openshift-ingress_router-default-6484bbc7f4-kq8zm for pod on switch ip-10-0-1-253.ec2.internal I0424 15:06:42.531199 3680 kube.go:256] Updating pod openshift-ingress/router-default-6484bbc7f4-kq8zm I0424 15:06:42.550122 3680 pod.go:62] [openshift-ingress/router-default-6484bbc7f4-kq8zm] pod update took 18.942056ms I0424 15:06:42.550142 3680 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-6484bbc7f4-kq8zm] addLogicalPort annotation time took 18.96765ms I0424 15:06:42.551231 3680 pods.go:271] [openshift-ingress/router-default-6484bbc7f4-kq8zm] addLogicalPort took 20.110774ms, libovsdb time 773.695µs I0424 15:06:42.585437 3680 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-rffkv d62675677a4e4729763439c55ab2dfd0788f2f741f8ed6cde4f69e9949c43591 network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-rffkv d62675677a4e4729763439c55ab2dfd0788f2f741f8ed6cde4f69e9949c43591 network default NAD default NAD key ] I0424 15:06:42.595733 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-rffkv, hostIfaceName: d62675677a4e472, network: default, NAD default, SandboxID: "d62675677a4e4729763439c55ab2dfd0788f2f741f8ed6cde4f69e9949c43591", PCI device ID: , UID: "1ed0ab68-f238-41d4-a534-12fe517f1759", MAC: 0a:58:0a:82:00:09, IPs: [10.130.0.9/23] I0424 15:06:42.639774 3680 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-rffkv d62675677a4e4729763439c55ab2dfd0788f2f741f8ed6cde4f69e9949c43591 network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-rffkv d62675677a4e4729763439c55ab2dfd0788f2f741f8ed6cde4f69e9949c43591 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:94:57:c3:c4:70\",\"name\":\"d62675677a4e472\"},{\"mac\":\"0a:58:0a:82:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bc44bedd-b30d-422f-b57a-84139fc39fcf\"}],\"ips\":[{\"address\":\"10.130.0.9/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:06:45.921450 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0424 15:06:47.786020 3680 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0424 15:06:47.786814 3680 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 774.47µs I0424 15:06:47.810599 3680 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0424 15:06:48.982687 3680 namespace.go:98] [openshift-service-ca] adding namespace I0424 15:06:48.983561 3680 namespace.go:102] [openshift-service-ca] adding namespace took 846.78µs I0424 15:06:49.001329 3680 namespace.go:142] [openshift-service-ca] updating namespace I0424 15:06:53.371974 3680 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db7l7m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:06:53.372510 3680 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db7l7m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:06:53.372536 3680 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db7l7m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:06:53.372546 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0424 15:06:53.372564 3680 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db7l7m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:06:53.372569 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0424 15:06:54.449973 3680 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db7l7m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:06:54.449995 3680 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db7l7m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:06:54.450009 3680 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db7l7m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:06:54.450019 3680 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db7l7m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:06:57.839915 3680 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0424 15:06:57.858856 3680 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0424 15:06:57.864753 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0424 15:07:14.090050 3680 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-chtnp 8659460eec0e63a5195d382598619315d91e3c0853de64515d09f365b4a7452d network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-chtnp 8659460eec0e63a5195d382598619315d91e3c0853de64515d09f365b4a7452d network default NAD default NAD key ] I0424 15:07:14.101127 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-chtnp, hostIfaceName: 8659460eec0e63a, network: default, NAD default, SandboxID: "8659460eec0e63a5195d382598619315d91e3c0853de64515d09f365b4a7452d", PCI device ID: , UID: "fb400c70-b691-432e-9fd8-b05cf6747747", MAC: 0a:58:0a:82:00:08, IPs: [10.130.0.8/23] I0424 15:07:14.146538 3680 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-chtnp 8659460eec0e63a5195d382598619315d91e3c0853de64515d09f365b4a7452d network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-chtnp 8659460eec0e63a5195d382598619315d91e3c0853de64515d09f365b4a7452d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:ef:18:78:fb:3a\",\"name\":\"8659460eec0e63a\"},{\"mac\":\"0a:58:0a:82:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/beb4b871-1d0c-499d-bee7-0182db32e935\"}],\"ips\":[{\"address\":\"10.130.0.8/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:07:14.415813 3680 cni.go:452] [openshift-ingress/router-default-6484bbc7f4-kq8zm 1b5d3542c1797287ef66d9703641a0e06eede2604464246c3dcc44daac14156c network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-6484bbc7f4-kq8zm 1b5d3542c1797287ef66d9703641a0e06eede2604464246c3dcc44daac14156c network default NAD default NAD key ] I0424 15:07:14.415940 3680 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-nqfwt fceeafcde676ae5e70de06f773f0233e826776bd38d8307948b646d999da68bf network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-nqfwt fceeafcde676ae5e70de06f773f0233e826776bd38d8307948b646d999da68bf network default NAD default NAD key ] I0424 15:07:14.429095 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-nqfwt, hostIfaceName: fceeafcde676ae5, network: default, NAD default, SandboxID: "fceeafcde676ae5e70de06f773f0233e826776bd38d8307948b646d999da68bf", PCI device ID: , UID: "f648aadf-9e25-43bb-b366-bd1f7ee08e6e", MAC: 0a:58:0a:82:00:0a, IPs: [10.130.0.10/23] I0424 15:07:14.429853 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-6484bbc7f4-kq8zm, hostIfaceName: 1b5d3542c179728, network: default, NAD default, SandboxID: "1b5d3542c1797287ef66d9703641a0e06eede2604464246c3dcc44daac14156c", PCI device ID: , UID: "46b64cca-8ad3-4013-b495-7c45afdee062", MAC: 0a:58:0a:82:00:0b, IPs: [10.130.0.11/23] I0424 15:07:14.473811 3680 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-nqfwt fceeafcde676ae5e70de06f773f0233e826776bd38d8307948b646d999da68bf network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-nqfwt fceeafcde676ae5e70de06f773f0233e826776bd38d8307948b646d999da68bf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:9d:8a:a8:51:1f\",\"name\":\"fceeafcde676ae5\"},{\"mac\":\"0a:58:0a:82:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/59bf3681-23a9-4f94-bef9-df34b342620c\"}],\"ips\":[{\"address\":\"10.130.0.10/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:07:14.488079 3680 cni.go:473] [openshift-ingress/router-default-6484bbc7f4-kq8zm 1b5d3542c1797287ef66d9703641a0e06eede2604464246c3dcc44daac14156c network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-6484bbc7f4-kq8zm 1b5d3542c1797287ef66d9703641a0e06eede2604464246c3dcc44daac14156c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:40:4c:8f:54:4e\",\"name\":\"1b5d3542c179728\"},{\"mac\":\"0a:58:0a:82:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/46679d32-c851-47e7-bf1f-38a2ecc23071\"}],\"ips\":[{\"address\":\"10.130.0.11/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:07:15.053591 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:15.053606 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:15.053611 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:15.056028 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0424 15:07:15.056047 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0424 15:07:15.077584 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0424 15:07:15.077600 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0424 15:07:15.089703 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:15.089714 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:15.089717 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default I0424 15:07:15.159916 3680 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-ffbkh] creating logical port openshift-insights_insights-runtime-extractor-ffbkh for pod on switch ip-10-0-1-253.ec2.internal I0424 15:07:15.160021 3680 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-ffbkh I0424 15:07:15.183722 3680 pod.go:62] [openshift-insights/insights-runtime-extractor-ffbkh] pod update took 23.73441ms I0424 15:07:15.183743 3680 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-ffbkh] addLogicalPort annotation time took 23.760127ms I0424 15:07:15.184853 3680 pods.go:271] [openshift-insights/insights-runtime-extractor-ffbkh] addLogicalPort took 24.950892ms, libovsdb time 777.194µs I0424 15:07:15.752430 3680 cni.go:452] [openshift-insights/insights-runtime-extractor-ffbkh ca43300ad52870ee3f9ee84040cda920d65df325b7190f7cf8315039200039a2 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-ffbkh ca43300ad52870ee3f9ee84040cda920d65df325b7190f7cf8315039200039a2 network default NAD default NAD key ] I0424 15:07:15.763230 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-ffbkh, hostIfaceName: ca43300ad52870e, network: default, NAD default, SandboxID: "ca43300ad52870ee3f9ee84040cda920d65df325b7190f7cf8315039200039a2", PCI device ID: , UID: "98d2d17d-df20-4ee5-af32-d0dc76355adb", MAC: 0a:58:0a:82:00:0c, IPs: [10.130.0.12/23] I0424 15:07:15.812858 3680 cni.go:473] [openshift-insights/insights-runtime-extractor-ffbkh ca43300ad52870ee3f9ee84040cda920d65df325b7190f7cf8315039200039a2 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-ffbkh ca43300ad52870ee3f9ee84040cda920d65df325b7190f7cf8315039200039a2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:99:79:52:e5:14\",\"name\":\"ca43300ad52870e\"},{\"mac\":\"0a:58:0a:82:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/acdd0445-8a71-4b89-af83-7c841d82b30e\"}],\"ips\":[{\"address\":\"10.130.0.12/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:07:16.048270 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0424 15:07:16.048363 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0424 15:07:16.048369 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0424 15:07:16.048385 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0424 15:07:16.048400 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0424 15:07:16.048408 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0424 15:07:16.048445 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:16.048455 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:16.049495 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:16.049693 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:16.049787 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:16.049819 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:16.063229 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:16.063305 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:16.063347 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0424 15:07:16.067728 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0424 15:07:16.067752 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0424 15:07:16.189778 3680 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0424 15:07:17.011326 3680 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6498cfc5d7-4d84v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:07:17.011871 3680 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6498cfc5d7-4d84v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:07:17.026975 3680 obj_retry.go:498] Detected object openshift-image-registry/image-registry-d956df968-klmhh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:07:17.027012 3680 pods.go:176] Deleting pod: openshift-image-registry/image-registry-d956df968-klmhh I0424 15:07:17.028405 3680 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-d956df968-klmhh, ips: 10.130.0.5 I0424 15:07:17.028428 3680 obj_retry.go:498] Detected object openshift-image-registry/image-registry-d956df968-klmhh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:07:17.034702 3680 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6498cfc5d7-4d84v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:07:17.034721 3680 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6498cfc5d7-4d84v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:07:17.043899 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6498cfc5d7-4d84v I0424 15:07:17.043937 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6498cfc5d7-4d84v I0424 15:07:17.054385 3680 obj_retry.go:492] Detected object openshift-image-registry/image-registry-d956df968-klmhh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:07:17.054407 3680 obj_retry.go:492] Detected object openshift-image-registry/image-registry-d956df968-klmhh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:07:17.057628 3680 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-78b9cc75f7-rnqw9] creating logical port openshift-image-registry_image-registry-78b9cc75f7-rnqw9 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:07:17.057735 3680 kube.go:256] Updating pod openshift-image-registry/image-registry-78b9cc75f7-rnqw9 I0424 15:07:17.071688 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-d956df968-klmhh I0424 15:07:17.071708 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-d956df968-klmhh I0424 15:07:17.078538 3680 pod.go:62] [openshift-image-registry/image-registry-78b9cc75f7-rnqw9] pod update took 20.82968ms I0424 15:07:17.078557 3680 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-78b9cc75f7-rnqw9] addLogicalPort annotation time took 20.851289ms I0424 15:07:17.079688 3680 pods.go:271] [openshift-image-registry/image-registry-78b9cc75f7-rnqw9] addLogicalPort took 22.070828ms, libovsdb time 888.203µs I0424 15:07:17.184302 3680 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-75v9m] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-75v9m for pod on switch ip-10-0-1-253.ec2.internal I0424 15:07:17.184387 3680 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-75v9m I0424 15:07:17.205290 3680 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-75v9m] pod update took 20.926476ms I0424 15:07:17.205314 3680 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-75v9m] addLogicalPort annotation time took 20.95639ms I0424 15:07:17.206552 3680 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-75v9m] addLogicalPort took 22.262862ms, libovsdb time 942.698µs I0424 15:07:17.414159 3680 cni.go:452] [openshift-image-registry/image-registry-78b9cc75f7-rnqw9 8cd920eca664197b4d854ee17c12c1d7db7ce4f3a6f13ccd5a22ef0752384f4b network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-78b9cc75f7-rnqw9 8cd920eca664197b4d854ee17c12c1d7db7ce4f3a6f13ccd5a22ef0752384f4b network default NAD default NAD key ] I0424 15:07:17.423668 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-78b9cc75f7-rnqw9, hostIfaceName: 8cd920eca664197, network: default, NAD default, SandboxID: "8cd920eca664197b4d854ee17c12c1d7db7ce4f3a6f13ccd5a22ef0752384f4b", PCI device ID: , UID: "a3e94359-7168-4c11-bcab-09344b23a400", MAC: 0a:58:0a:82:00:0d, IPs: [10.130.0.13/23] I0424 15:07:17.470454 3680 cni.go:473] [openshift-image-registry/image-registry-78b9cc75f7-rnqw9 8cd920eca664197b4d854ee17c12c1d7db7ce4f3a6f13ccd5a22ef0752384f4b network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-78b9cc75f7-rnqw9 8cd920eca664197b4d854ee17c12c1d7db7ce4f3a6f13ccd5a22ef0752384f4b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:e6:91:24:be:54\",\"name\":\"8cd920eca664197\"},{\"mac\":\"0a:58:0a:82:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1a9d0370-d8f8-4ef1-ab26-8d8ce725ba5a\"}],\"ips\":[{\"address\":\"10.130.0.13/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:07:17.547767 3680 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-75v9m 6b3a9cbad40ec9210e91ebfb5fca711d85f029f0559927e4385a6256e0859580 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-75v9m 6b3a9cbad40ec9210e91ebfb5fca711d85f029f0559927e4385a6256e0859580 network default NAD default NAD key ] I0424 15:07:17.558080 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-75v9m, hostIfaceName: 6b3a9cbad40ec92, network: default, NAD default, SandboxID: "6b3a9cbad40ec9210e91ebfb5fca711d85f029f0559927e4385a6256e0859580", PCI device ID: , UID: "a125bf70-c354-49aa-9e78-72af54ea06e8", MAC: 0a:58:0a:82:00:0e, IPs: [10.130.0.14/23] I0424 15:07:17.607100 3680 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-75v9m 6b3a9cbad40ec9210e91ebfb5fca711d85f029f0559927e4385a6256e0859580 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-75v9m 6b3a9cbad40ec9210e91ebfb5fca711d85f029f0559927e4385a6256e0859580 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:18:75:84:af:c2\",\"name\":\"6b3a9cbad40ec92\"},{\"mac\":\"0a:58:0a:82:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4589ff7c-192e-4cd6-87ce-087fac74f2ba\"}],\"ips\":[{\"address\":\"10.130.0.14/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:07:17.938333 3680 cni.go:452] [openshift-multus/network-metrics-daemon-sjxwf 00db3993df13daad62210904bd841c5030e0c4aebd7461195bc3cb7f453ca01f network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-sjxwf 00db3993df13daad62210904bd841c5030e0c4aebd7461195bc3cb7f453ca01f network default NAD default NAD key ] I0424 15:07:17.948344 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-sjxwf, hostIfaceName: 00db3993df13daa, network: default, NAD default, SandboxID: "00db3993df13daad62210904bd841c5030e0c4aebd7461195bc3cb7f453ca01f", PCI device ID: , UID: "c49da2ac-0290-4d41-b7ef-a4c8db614646", MAC: 0a:58:0a:82:00:03, IPs: [10.130.0.3/23] I0424 15:07:17.995952 3680 cni.go:473] [openshift-multus/network-metrics-daemon-sjxwf 00db3993df13daad62210904bd841c5030e0c4aebd7461195bc3cb7f453ca01f network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-sjxwf 00db3993df13daad62210904bd841c5030e0c4aebd7461195bc3cb7f453ca01f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:f7:fd:13:d1:76\",\"name\":\"00db3993df13daa\"},{\"mac\":\"0a:58:0a:82:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/27edce90-52a4-4c42-831e-0e12a6637a33\"}],\"ips\":[{\"address\":\"10.130.0.3/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:07:18.018641 3680 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:07:18.062812 3680 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:07:19.365653 3680 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0424 15:07:20.011162 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0424 15:07:20.072297 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0424 15:07:21.299916 3680 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-76b589d668-bgwv8] creating logical port openshift-monitoring_prometheus-operator-76b589d668-bgwv8 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:07:21.300037 3680 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-76b589d668-bgwv8 I0424 15:07:21.314796 3680 pod.go:62] [openshift-monitoring/prometheus-operator-76b589d668-bgwv8] pod update took 14.784556ms I0424 15:07:21.314817 3680 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-76b589d668-bgwv8] addLogicalPort annotation time took 14.810721ms I0424 15:07:21.315896 3680 pods.go:271] [openshift-monitoring/prometheus-operator-76b589d668-bgwv8] addLogicalPort took 15.999884ms, libovsdb time 777.01µs I0424 15:07:22.254945 3680 cni.go:452] [openshift-monitoring/prometheus-operator-76b589d668-bgwv8 be59f7a542e37675bd5e6d13ff0c7ee3cc836a852ac7ad7a8adf60a4b2e4c34c network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-76b589d668-bgwv8 be59f7a542e37675bd5e6d13ff0c7ee3cc836a852ac7ad7a8adf60a4b2e4c34c network default NAD default NAD key ] I0424 15:07:22.264522 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-76b589d668-bgwv8, hostIfaceName: be59f7a542e3767, network: default, NAD default, SandboxID: "be59f7a542e37675bd5e6d13ff0c7ee3cc836a852ac7ad7a8adf60a4b2e4c34c", PCI device ID: , UID: "2bd7aca5-649c-4656-bcbb-d58cc087fb87", MAC: 0a:58:0a:82:00:0f, IPs: [10.130.0.15/23] I0424 15:07:22.310547 3680 cni.go:473] [openshift-monitoring/prometheus-operator-76b589d668-bgwv8 be59f7a542e37675bd5e6d13ff0c7ee3cc836a852ac7ad7a8adf60a4b2e4c34c network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-76b589d668-bgwv8 be59f7a542e37675bd5e6d13ff0c7ee3cc836a852ac7ad7a8adf60a4b2e4c34c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:24:f6:a5:1f:50\",\"name\":\"be59f7a542e3767\"},{\"mac\":\"0a:58:0a:82:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/55747718-b3fa-4866-9b28-e7af7dc64f66\"}],\"ips\":[{\"address\":\"10.130.0.15/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:07:26.925510 3680 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-b8587444b-6mx6b] creating logical port openshift-monitoring_openshift-state-metrics-b8587444b-6mx6b for pod on switch ip-10-0-1-253.ec2.internal I0424 15:07:26.925614 3680 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-b8587444b-6mx6b I0424 15:07:26.948211 3680 pod.go:62] [openshift-monitoring/openshift-state-metrics-b8587444b-6mx6b] pod update took 22.622367ms I0424 15:07:26.948240 3680 base_network_controller_pods.go:951] [default/openshift-monitoring/openshift-state-metrics-b8587444b-6mx6b] addLogicalPort annotation time took 22.656302ms I0424 15:07:26.949398 3680 pods.go:271] [openshift-monitoring/openshift-state-metrics-b8587444b-6mx6b] addLogicalPort took 23.908663ms, libovsdb time 867.751µs W0424 15:07:27.245704 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics I0424 15:07:27.283194 3680 cni.go:452] [openshift-monitoring/openshift-state-metrics-b8587444b-6mx6b 5a270fbb9b918140a81a608851e1e203ee117ade0ee4bd717a5dfe97e52e3c3a network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-6mx6b 5a270fbb9b918140a81a608851e1e203ee117ade0ee4bd717a5dfe97e52e3c3a network default NAD default NAD key ] I0424 15:07:27.292608 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-b8587444b-6mx6b, hostIfaceName: 5a270fbb9b91814, network: default, NAD default, SandboxID: "5a270fbb9b918140a81a608851e1e203ee117ade0ee4bd717a5dfe97e52e3c3a", PCI device ID: , UID: "109b5a67-34d5-4af8-bf66-167da69e3406", MAC: 0a:58:0a:82:00:10, IPs: [10.130.0.16/23] I0424 15:07:27.341714 3680 cni.go:473] [openshift-monitoring/openshift-state-metrics-b8587444b-6mx6b 5a270fbb9b918140a81a608851e1e203ee117ade0ee4bd717a5dfe97e52e3c3a network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-6mx6b 5a270fbb9b918140a81a608851e1e203ee117ade0ee4bd717a5dfe97e52e3c3a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:38:ac:61:85:f3\",\"name\":\"5a270fbb9b91814\"},{\"mac\":\"0a:58:0a:82:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1458efee-4d8f-433f-9c6a-54c223c7a664\"}],\"ips\":[{\"address\":\"10.130.0.16/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:07:27.520799 3680 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-fkbvm] creating logical port openshift-console_downloads-5c54f4944f-fkbvm for pod on switch ip-10-0-1-253.ec2.internal I0424 15:07:27.520911 3680 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-fkbvm I0424 15:07:27.548143 3680 pod.go:62] [openshift-console/downloads-5c54f4944f-fkbvm] pod update took 27.254183ms I0424 15:07:27.548170 3680 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-fkbvm] addLogicalPort annotation time took 27.283674ms I0424 15:07:27.549475 3680 pods.go:271] [openshift-console/downloads-5c54f4944f-fkbvm] addLogicalPort took 28.690978ms, libovsdb time 1.014696ms I0424 15:07:27.878350 3680 cni.go:452] [openshift-console/downloads-5c54f4944f-fkbvm a5276402f6d144b62de4c48c89a81d4671b56b9158b8f2ebd20673b27a5dc76e network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-fkbvm a5276402f6d144b62de4c48c89a81d4671b56b9158b8f2ebd20673b27a5dc76e network default NAD default NAD key ] I0424 15:07:27.887907 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-fkbvm, hostIfaceName: a5276402f6d144b, network: default, NAD default, SandboxID: "a5276402f6d144b62de4c48c89a81d4671b56b9158b8f2ebd20673b27a5dc76e", PCI device ID: , UID: "1de27dc0-2a08-4aba-a9fe-e64a292bc2fb", MAC: 0a:58:0a:82:00:11, IPs: [10.130.0.17/23] I0424 15:07:27.934476 3680 cni.go:473] [openshift-console/downloads-5c54f4944f-fkbvm a5276402f6d144b62de4c48c89a81d4671b56b9158b8f2ebd20673b27a5dc76e network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-fkbvm a5276402f6d144b62de4c48c89a81d4671b56b9158b8f2ebd20673b27a5dc76e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:b5:c6:9c:89:bd\",\"name\":\"a5276402f6d144b\"},{\"mac\":\"0a:58:0a:82:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c32a441b-b157-4cf3-b84f-8a9b5e2a199f\"}],\"ips\":[{\"address\":\"10.130.0.17/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:07:28.218408 3680 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:07:28.218527 3680 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0424 15:07:28.243522 3680 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 25.034911ms I0424 15:07:28.243564 3680 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 25.08062ms I0424 15:07:28.244908 3680 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 26.514685ms, libovsdb time 1.035671ms I0424 15:07:29.178103 3680 cni.go:452] [openshift-monitoring/alertmanager-main-0 339aa99d4fa9fb7b7dfa44ad2db11cc6c14c81cf6985db373b2c67557666e9d6 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 339aa99d4fa9fb7b7dfa44ad2db11cc6c14c81cf6985db373b2c67557666e9d6 network default NAD default NAD key ] I0424 15:07:29.188731 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 339aa99d4fa9fb7, network: default, NAD default, SandboxID: "339aa99d4fa9fb7b7dfa44ad2db11cc6c14c81cf6985db373b2c67557666e9d6", PCI device ID: , UID: "f6f1dc3f-9674-4833-8fc0-64a7fd21f3bb", MAC: 0a:58:0a:82:00:12, IPs: [10.130.0.18/23] I0424 15:07:29.237454 3680 cni.go:473] [openshift-monitoring/alertmanager-main-0 339aa99d4fa9fb7b7dfa44ad2db11cc6c14c81cf6985db373b2c67557666e9d6 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 339aa99d4fa9fb7b7dfa44ad2db11cc6c14c81cf6985db373b2c67557666e9d6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:56:d7:83:d2:25\",\"name\":\"339aa99d4fa9fb7\"},{\"mac\":\"0a:58:0a:82:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/78b0c940-096d-41b3-9d32-ff59d2707794\"}],\"ips\":[{\"address\":\"10.130.0.18/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:07:30.214232 3680 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-59f8bfdd8d-htw6x] creating logical port openshift-monitoring_thanos-querier-59f8bfdd8d-htw6x for pod on switch ip-10-0-1-253.ec2.internal I0424 15:07:30.214330 3680 kube.go:256] Updating pod openshift-monitoring/thanos-querier-59f8bfdd8d-htw6x I0424 15:07:30.234954 3680 pod.go:62] [openshift-monitoring/thanos-querier-59f8bfdd8d-htw6x] pod update took 20.653286ms I0424 15:07:30.234981 3680 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-59f8bfdd8d-htw6x] addLogicalPort annotation time took 20.683137ms I0424 15:07:30.236168 3680 pods.go:271] [openshift-monitoring/thanos-querier-59f8bfdd8d-htw6x] addLogicalPort took 21.951707ms, libovsdb time 865.651µs I0424 15:07:30.572904 3680 cni.go:452] [openshift-monitoring/thanos-querier-59f8bfdd8d-htw6x 85a2754dea3a914300af080b447b93db68893796261fc1e0f7ec314fd63e1b07 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-59f8bfdd8d-htw6x 85a2754dea3a914300af080b447b93db68893796261fc1e0f7ec314fd63e1b07 network default NAD default NAD key ] I0424 15:07:30.595188 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-59f8bfdd8d-htw6x, hostIfaceName: 85a2754dea3a914, network: default, NAD default, SandboxID: "85a2754dea3a914300af080b447b93db68893796261fc1e0f7ec314fd63e1b07", PCI device ID: , UID: "68c710db-88f4-4e29-accf-b1c7ca4d8070", MAC: 0a:58:0a:82:00:13, IPs: [10.130.0.19/23] I0424 15:07:30.644776 3680 cni.go:473] [openshift-monitoring/thanos-querier-59f8bfdd8d-htw6x 85a2754dea3a914300af080b447b93db68893796261fc1e0f7ec314fd63e1b07 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-59f8bfdd8d-htw6x 85a2754dea3a914300af080b447b93db68893796261fc1e0f7ec314fd63e1b07 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:2a:fc:43:ff:c2\",\"name\":\"85a2754dea3a914\"},{\"mac\":\"0a:58:0a:82:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/015a4233-7b82-4b0b-be63-b2fcd4de7858\"}],\"ips\":[{\"address\":\"10.130.0.19/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:07:31.084000 3680 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 15:07:31.084023 3680 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 15:07:31.084027 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 15:07:31.133029 3680 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 15:07:31.133046 3680 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 15:07:31.133050 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0424 15:07:31.538798 3680 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-jr29c] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-jr29c for pod on switch ip-10-0-1-253.ec2.internal I0424 15:07:31.538944 3680 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-jr29c I0424 15:07:31.554813 3680 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-jr29c] pod update took 15.912052ms I0424 15:07:31.554836 3680 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-jr29c] addLogicalPort annotation time took 15.938359ms I0424 15:07:31.556062 3680 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-jr29c] addLogicalPort took 17.286186ms, libovsdb time 889.866µs I0424 15:07:31.905600 3680 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-jr29c 38854b3666f6c6a629b1fb9a8fbf7a394f667bd849ccc5b36eb9cc87c54f4a5f network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-jr29c 38854b3666f6c6a629b1fb9a8fbf7a394f667bd849ccc5b36eb9cc87c54f4a5f network default NAD default NAD key ] I0424 15:07:31.916713 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-jr29c, hostIfaceName: 38854b3666f6c6a, network: default, NAD default, SandboxID: "38854b3666f6c6a629b1fb9a8fbf7a394f667bd849ccc5b36eb9cc87c54f4a5f", PCI device ID: , UID: "781ec7f8-bf56-401d-a5f4-124a0c54ad8e", MAC: 0a:58:0a:82:00:14, IPs: [10.130.0.20/23] I0424 15:07:31.951188 3680 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-5d86c9ccc9-vw2zs] creating logical port openshift-monitoring_metrics-server-5d86c9ccc9-vw2zs for pod on switch ip-10-0-1-253.ec2.internal I0424 15:07:31.951351 3680 kube.go:256] Updating pod openshift-monitoring/metrics-server-5d86c9ccc9-vw2zs I0424 15:07:31.968409 3680 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-jr29c 38854b3666f6c6a629b1fb9a8fbf7a394f667bd849ccc5b36eb9cc87c54f4a5f network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-jr29c 38854b3666f6c6a629b1fb9a8fbf7a394f667bd849ccc5b36eb9cc87c54f4a5f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:81:9c:f8:85:4c\",\"name\":\"38854b3666f6c6a\"},{\"mac\":\"0a:58:0a:82:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0f286cce-3d6e-4ccf-86c3-634c508d275a\"}],\"ips\":[{\"address\":\"10.130.0.20/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:07:31.968957 3680 pod.go:62] [openshift-monitoring/metrics-server-5d86c9ccc9-vw2zs] pod update took 17.638554ms I0424 15:07:31.968982 3680 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-5d86c9ccc9-vw2zs] addLogicalPort annotation time took 17.665873ms I0424 15:07:31.970244 3680 pods.go:271] [openshift-monitoring/metrics-server-5d86c9ccc9-vw2zs] addLogicalPort took 19.070562ms, libovsdb time 863.339µs I0424 15:07:33.022132 3680 cni.go:452] [openshift-monitoring/metrics-server-5d86c9ccc9-vw2zs 5dc2f715d8c874fc3d6fc96c375cc303ea3328aceaf776d478bb3f0ab6e3ba93 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-5d86c9ccc9-vw2zs 5dc2f715d8c874fc3d6fc96c375cc303ea3328aceaf776d478bb3f0ab6e3ba93 network default NAD default NAD key ] I0424 15:07:33.033346 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-5d86c9ccc9-vw2zs, hostIfaceName: 5dc2f715d8c874f, network: default, NAD default, SandboxID: "5dc2f715d8c874fc3d6fc96c375cc303ea3328aceaf776d478bb3f0ab6e3ba93", PCI device ID: , UID: "25ffd4b7-a1c3-42d8-9a4b-b5386abec482", MAC: 0a:58:0a:82:00:15, IPs: [10.130.0.21/23] I0424 15:07:33.099389 3680 cni.go:473] [openshift-monitoring/metrics-server-5d86c9ccc9-vw2zs 5dc2f715d8c874fc3d6fc96c375cc303ea3328aceaf776d478bb3f0ab6e3ba93 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-5d86c9ccc9-vw2zs 5dc2f715d8c874fc3d6fc96c375cc303ea3328aceaf776d478bb3f0ab6e3ba93 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:e0:de:51:3c:42\",\"name\":\"5dc2f715d8c874f\"},{\"mac\":\"0a:58:0a:82:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0a730a5-7c3c-4cb9-a86d-534b1e0e4695\"}],\"ips\":[{\"address\":\"10.130.0.21/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:07:33.399723 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin I0424 15:07:33.545912 3680 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:07:33.546042 3680 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0424 15:07:33.591466 3680 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 45.465502ms I0424 15:07:33.591489 3680 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 45.496444ms I0424 15:07:33.592800 3680 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 46.90394ms, libovsdb time 925.735µs I0424 15:07:33.923640 3680 cni.go:452] [openshift-monitoring/prometheus-k8s-1 ace376844fc507d1cdc4560500eda2db7b0968fc3a951a657f30ddc88e2043b1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 ace376844fc507d1cdc4560500eda2db7b0968fc3a951a657f30ddc88e2043b1 network default NAD default NAD key ] I0424 15:07:33.936347 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: ace376844fc507d, network: default, NAD default, SandboxID: "ace376844fc507d1cdc4560500eda2db7b0968fc3a951a657f30ddc88e2043b1", PCI device ID: , UID: "5ec0f9cc-009d-4421-ae43-fc9d51711259", MAC: 0a:58:0a:82:00:16, IPs: [10.130.0.22/23] I0424 15:07:33.997795 3680 cni.go:473] [openshift-monitoring/prometheus-k8s-1 ace376844fc507d1cdc4560500eda2db7b0968fc3a951a657f30ddc88e2043b1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 ace376844fc507d1cdc4560500eda2db7b0968fc3a951a657f30ddc88e2043b1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:a0:b8:17:17:c7\",\"name\":\"ace376844fc507d\"},{\"mac\":\"0a:58:0a:82:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7f0efc4d-4f94-4263-8b6c-b3d7460df97a\"}],\"ips\":[{\"address\":\"10.130.0.22/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:07:34.103999 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0424 15:07:34.133285 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0424 15:07:35.110316 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:35.110340 3680 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:35.110346 3680 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:35.110352 3680 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:35.120655 3680 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:35.120672 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:35.120677 3680 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:35.120681 3680 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:35.195017 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0424 15:07:35.195113 3680 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 15:07:35.195130 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 15:07:35.224889 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0424 15:07:35.231974 3680 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 15:07:35.231985 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 15:07:41.131999 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:41.132027 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:41.132035 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:41.132039 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:41.132057 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:41.132064 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:41.132069 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:41.132073 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:41.144780 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:41.144797 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:41.144802 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:41.144807 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 15:07:41.431552 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0424 15:07:42.434894 3680 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:07:44.444108 3680 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:44.444135 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:44.444142 3680 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0424 15:07:46.353194 3680 cni.go:452] [openshift-dns/dns-default-5w65p 09693e862f9f3551fc19164a53c71de75daf69411942f61ba08fbd5d9d980e17 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-5w65p 09693e862f9f3551fc19164a53c71de75daf69411942f61ba08fbd5d9d980e17 network default NAD default NAD key ] I0424 15:07:46.362601 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-5w65p, hostIfaceName: 09693e862f9f355, network: default, NAD default, SandboxID: "09693e862f9f3551fc19164a53c71de75daf69411942f61ba08fbd5d9d980e17", PCI device ID: , UID: "534e997d-a2ac-4870-a989-f6f139d8a2f6", MAC: 0a:58:0a:82:00:06, IPs: [10.130.0.6/23] I0424 15:07:46.410882 3680 cni.go:473] [openshift-dns/dns-default-5w65p 09693e862f9f3551fc19164a53c71de75daf69411942f61ba08fbd5d9d980e17 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-5w65p 09693e862f9f3551fc19164a53c71de75daf69411942f61ba08fbd5d9d980e17 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:d1:fe:9b:3e:5b\",\"name\":\"09693e862f9f355\"},{\"mac\":\"0a:58:0a:82:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a30d5d9c-6e12-4bff-b97c-cca4a5ee6d3b\"}],\"ips\":[{\"address\":\"10.130.0.6/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:07:46.732605 3680 base_network_controller_pods.go:487] [default/openshift-console/console-5b4b8fd4f7-gl6cb] creating logical port openshift-console_console-5b4b8fd4f7-gl6cb for pod on switch ip-10-0-1-253.ec2.internal I0424 15:07:46.732740 3680 kube.go:256] Updating pod openshift-console/console-5b4b8fd4f7-gl6cb I0424 15:07:46.751325 3680 pod.go:62] [openshift-console/console-5b4b8fd4f7-gl6cb] pod update took 18.627099ms I0424 15:07:46.751351 3680 base_network_controller_pods.go:951] [default/openshift-console/console-5b4b8fd4f7-gl6cb] addLogicalPort annotation time took 18.659288ms I0424 15:07:46.752779 3680 pods.go:271] [openshift-console/console-5b4b8fd4f7-gl6cb] addLogicalPort took 20.193965ms, libovsdb time 1.086369ms I0424 15:07:47.104398 3680 cni.go:452] [openshift-console/console-5b4b8fd4f7-gl6cb 8ccdf42ab10559c2c2c0d39c77e2547436680398c6da7914fcca48e72f42dcb1 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-5b4b8fd4f7-gl6cb 8ccdf42ab10559c2c2c0d39c77e2547436680398c6da7914fcca48e72f42dcb1 network default NAD default NAD key ] I0424 15:07:47.116411 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-5b4b8fd4f7-gl6cb, hostIfaceName: 8ccdf42ab10559c, network: default, NAD default, SandboxID: "8ccdf42ab10559c2c2c0d39c77e2547436680398c6da7914fcca48e72f42dcb1", PCI device ID: , UID: "c4bcc65f-18b9-4589-ae01-87d2aaf9f3fa", MAC: 0a:58:0a:82:00:17, IPs: [10.130.0.23/23] I0424 15:07:47.176972 3680 cni.go:473] [openshift-console/console-5b4b8fd4f7-gl6cb 8ccdf42ab10559c2c2c0d39c77e2547436680398c6da7914fcca48e72f42dcb1 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-5b4b8fd4f7-gl6cb 8ccdf42ab10559c2c2c0d39c77e2547436680398c6da7914fcca48e72f42dcb1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:b6:3c:9c:fa:d9\",\"name\":\"8ccdf42ab10559c\"},{\"mac\":\"0a:58:0a:82:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/690e6f26-4595-4460-b7bf-5738b1839944\"}],\"ips\":[{\"address\":\"10.130.0.23/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:07:47.179807 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0424 15:07:47.219865 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0424 15:07:48.197467 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0424 15:07:49.195371 3680 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:49.195389 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:49.195399 3680 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:50.189610 3680 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:50.189630 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:50.189634 3680 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:50.221331 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0424 15:07:54.460797 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:54.460819 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:54.460825 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:54.460840 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:54.460846 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:54.460850 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:56.114397 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0424 15:07:56.114417 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0424 15:07:57.145570 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0424 15:07:57.243743 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0424 15:07:59.221871 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:59.221890 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:59.221896 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0424 15:07:59.268912 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 15:07:59.268951 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 15:07:59.268959 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 15:08:02.513803 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0424 15:08:03.458345 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:03.458377 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:07.383965 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:07.383990 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:07.427017 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:12.279560 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0424 15:08:13.481624 3680 obj_retry.go:498] Detected object openshift-console/console-5466d6cbff-d9m9c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:08:13.482208 3680 obj_retry.go:498] Detected object openshift-console/console-5466d6cbff-d9m9c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:08:14.322382 3680 obj_retry.go:492] Detected object openshift-console/console-5466d6cbff-d9m9c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:08:14.322406 3680 obj_retry.go:492] Detected object openshift-console/console-5466d6cbff-d9m9c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:08:14.327775 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5466d6cbff-d9m9c I0424 15:08:14.327788 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5466d6cbff-d9m9c W0424 15:08:17.305811 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0424 15:08:21.293464 3680 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0424 15:08:22.575242 3680 obj_retry.go:498] Detected object openshift-console/console-5466d6cbff-6vgpk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:08:22.575767 3680 obj_retry.go:498] Detected object openshift-console/console-5466d6cbff-6vgpk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:08:22.891349 3680 obj_retry.go:492] Detected object openshift-console/console-5466d6cbff-6vgpk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:08:22.891373 3680 obj_retry.go:492] Detected object openshift-console/console-5466d6cbff-6vgpk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:08:22.899173 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5466d6cbff-6vgpk I0424 15:08:22.899199 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5466d6cbff-6vgpk W0424 15:08:24.359999 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:24.360030 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:24.360086 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0424 15:08:31.397661 3680 obj_retry.go:498] Detected object openshift-ingress/router-default-84f5db4787-477bd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:08:31.398216 3680 obj_retry.go:498] Detected object openshift-ingress/router-default-84f5db4787-477bd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:08:32.366286 3680 obj_retry.go:492] Detected object openshift-ingress/router-default-84f5db4787-477bd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:08:32.366312 3680 obj_retry.go:492] Detected object openshift-ingress/router-default-84f5db4787-477bd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:08:32.370035 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-84f5db4787-477bd I0424 15:08:32.370060 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-84f5db4787-477bd I0424 15:08:33.755029 3680 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6498cfc5d7-rsztn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:08:33.757340 3680 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6498cfc5d7-rsztn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0424 15:08:33.761743 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:33.761776 3680 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:33.761784 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:33.761792 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:34.419039 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 15:08:34.419063 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 15:08:34.419078 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 15:08:34.419084 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 15:08:34.435233 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 15:08:34.435264 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0424 15:08:34.645044 3680 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6498cfc5d7-rsztn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:08:34.645064 3680 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6498cfc5d7-rsztn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:08:34.653148 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6498cfc5d7-rsztn I0424 15:08:34.653177 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6498cfc5d7-rsztn W0424 15:08:45.356916 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:45.356957 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0424 15:08:45.366557 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0424 15:08:49.035413 3680 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:08:49.036033 3680 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:08:49.442068 3680 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:08:49.442088 3680 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:08:49.453083 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0424 15:08:49.453108 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0424 15:08:53.309522 3680 cni.go:452] [openshift-monitoring/prometheus-k8s-1 ace376844fc507d1cdc4560500eda2db7b0968fc3a951a657f30ddc88e2043b1 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 ace376844fc507d1cdc4560500eda2db7b0968fc3a951a657f30ddc88e2043b1 network default NAD default NAD key ] I0424 15:08:53.396275 3680 cni.go:473] [openshift-monitoring/prometheus-k8s-1 ace376844fc507d1cdc4560500eda2db7b0968fc3a951a657f30ddc88e2043b1 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 ace376844fc507d1cdc4560500eda2db7b0968fc3a951a657f30ddc88e2043b1 network default NAD default NAD key default], result "{}", err I0424 15:08:53.464620 3680 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:08:53.464650 3680 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0424 15:08:53.465811 3680 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.130.0.22 I0424 15:08:53.465840 3680 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:08:53.763371 3680 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:08:53.763392 3680 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:08:53.773619 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0424 15:08:53.773643 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0424 15:08:53.814516 3680 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:08:53.814617 3680 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0424 15:08:53.836764 3680 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 22.18549ms I0424 15:08:53.836784 3680 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 22.210022ms I0424 15:08:53.838003 3680 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 23.501458ms, libovsdb time 832.56µs I0424 15:08:54.171301 3680 cni.go:452] [openshift-monitoring/prometheus-k8s-1 4d5ff4c39920a138e8301a5c495591f4fe4a52cec6e7a1909b89614632ad7c5d network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 4d5ff4c39920a138e8301a5c495591f4fe4a52cec6e7a1909b89614632ad7c5d network default NAD default NAD key ] I0424 15:08:54.180685 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 4d5ff4c39920a13, network: default, NAD default, SandboxID: "4d5ff4c39920a138e8301a5c495591f4fe4a52cec6e7a1909b89614632ad7c5d", PCI device ID: , UID: "d1e49d5a-e37c-4de4-ac06-02f84dca387f", MAC: 0a:58:0a:82:00:18, IPs: [10.130.0.24/23] I0424 15:08:54.228906 3680 cni.go:473] [openshift-monitoring/prometheus-k8s-1 4d5ff4c39920a138e8301a5c495591f4fe4a52cec6e7a1909b89614632ad7c5d network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 4d5ff4c39920a138e8301a5c495591f4fe4a52cec6e7a1909b89614632ad7c5d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:ab:c6:7c:45:62\",\"name\":\"4d5ff4c39920a13\"},{\"mac\":\"0a:58:0a:82:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bea982b3-2ee1-4d04-9eea-8c5c763664b6\"}],\"ips\":[{\"address\":\"10.130.0.24/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:09:21.079460 3680 cni.go:452] [openshift-monitoring/alertmanager-main-0 339aa99d4fa9fb7b7dfa44ad2db11cc6c14c81cf6985db373b2c67557666e9d6 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 339aa99d4fa9fb7b7dfa44ad2db11cc6c14c81cf6985db373b2c67557666e9d6 network default NAD default NAD key ] I0424 15:09:21.161850 3680 cni.go:473] [openshift-monitoring/alertmanager-main-0 339aa99d4fa9fb7b7dfa44ad2db11cc6c14c81cf6985db373b2c67557666e9d6 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 339aa99d4fa9fb7b7dfa44ad2db11cc6c14c81cf6985db373b2c67557666e9d6 network default NAD default NAD key default], result "{}", err I0424 15:09:21.222656 3680 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:09:21.222686 3680 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0424 15:09:21.223873 3680 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.130.0.18 I0424 15:09:21.223902 3680 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:09:21.590612 3680 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:09:21.590665 3680 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:09:21.599631 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0424 15:09:21.599666 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0424 15:09:21.642701 3680 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:09:21.642830 3680 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0424 15:09:21.664958 3680 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 22.162282ms I0424 15:09:21.664989 3680 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 22.196147ms I0424 15:09:21.666437 3680 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 23.758778ms, libovsdb time 1.011572ms I0424 15:09:22.002232 3680 cni.go:452] [openshift-monitoring/alertmanager-main-0 adebb8a9318664ce2ed46d07c6d5a86c4c01234b5fb7012e25d94936c97c14c4 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 adebb8a9318664ce2ed46d07c6d5a86c4c01234b5fb7012e25d94936c97c14c4 network default NAD default NAD key ] I0424 15:09:22.011737 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: adebb8a9318664c, network: default, NAD default, SandboxID: "adebb8a9318664ce2ed46d07c6d5a86c4c01234b5fb7012e25d94936c97c14c4", PCI device ID: , UID: "3ff7917a-c389-4d1a-8a03-991324298957", MAC: 0a:58:0a:82:00:19, IPs: [10.130.0.25/23] I0424 15:09:22.024027 3680 obj_retry.go:498] Detected object openshift-console/console-5b4b8fd4f7-gtdjv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:09:22.024527 3680 obj_retry.go:498] Detected object openshift-console/console-5b4b8fd4f7-gtdjv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:09:22.062022 3680 cni.go:473] [openshift-monitoring/alertmanager-main-0 adebb8a9318664ce2ed46d07c6d5a86c4c01234b5fb7012e25d94936c97c14c4 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 adebb8a9318664ce2ed46d07c6d5a86c4c01234b5fb7012e25d94936c97c14c4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:20:df:1b:31:98\",\"name\":\"adebb8a9318664c\"},{\"mac\":\"0a:58:0a:82:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8928ff8d-1072-41ac-900e-ed8e7ad306b2\"}],\"ips\":[{\"address\":\"10.130.0.25/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:09:22.624360 3680 obj_retry.go:492] Detected object openshift-console/console-5b4b8fd4f7-gtdjv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:09:22.624386 3680 obj_retry.go:492] Detected object openshift-console/console-5b4b8fd4f7-gtdjv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:09:22.638886 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5b4b8fd4f7-gtdjv I0424 15:09:22.639005 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5b4b8fd4f7-gtdjv I0424 15:09:32.981307 3680 cni.go:452] [openshift-console/console-5b4b8fd4f7-gl6cb 8ccdf42ab10559c2c2c0d39c77e2547436680398c6da7914fcca48e72f42dcb1 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-5b4b8fd4f7-gl6cb 8ccdf42ab10559c2c2c0d39c77e2547436680398c6da7914fcca48e72f42dcb1 network default NAD default NAD key ] I0424 15:09:33.063421 3680 cni.go:473] [openshift-console/console-5b4b8fd4f7-gl6cb 8ccdf42ab10559c2c2c0d39c77e2547436680398c6da7914fcca48e72f42dcb1 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-5b4b8fd4f7-gl6cb 8ccdf42ab10559c2c2c0d39c77e2547436680398c6da7914fcca48e72f42dcb1 network default NAD default NAD key default], result "{}", err I0424 15:09:33.112283 3680 obj_retry.go:498] Detected object openshift-console/console-5b4b8fd4f7-gl6cb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:09:33.112320 3680 pods.go:176] Deleting pod: openshift-console/console-5b4b8fd4f7-gl6cb I0424 15:09:33.113744 3680 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5b4b8fd4f7-gl6cb, ips: 10.130.0.23 I0424 15:09:33.113774 3680 obj_retry.go:498] Detected object openshift-console/console-5b4b8fd4f7-gl6cb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:09:33.648822 3680 obj_retry.go:492] Detected object openshift-console/console-5b4b8fd4f7-gl6cb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:09:33.648846 3680 obj_retry.go:492] Detected object openshift-console/console-5b4b8fd4f7-gl6cb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:09:33.660061 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5b4b8fd4f7-gl6cb I0424 15:09:33.660096 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5b4b8fd4f7-gl6cb W0424 15:09:52.060895 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 15:09:52.060917 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 15:09:52.060941 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 15:09:54.773976 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 15:09:54.773995 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0424 15:09:56.580177 3680 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:09:56.580734 3680 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:09:56.740394 3680 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:09:56.740420 3680 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:09:56.749217 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0424 15:09:56.749249 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0424 15:10:05.111341 3680 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29617390-2hh85 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:10:05.111870 3680 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29617390-2hh85 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:10:06.018394 3680 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29617390-2hh85 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:10:06.018421 3680 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29617390-2hh85 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:10:45.172559 3680 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=5 I0424 15:11:02.095012 3680 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=563 I0424 15:11:02.209372 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0424 15:11:04.643716 3680 base_network_controller_pods.go:487] [default/openshift-console/console-6b85c448d-wfs26] creating logical port openshift-console_console-6b85c448d-wfs26 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:11:04.643824 3680 kube.go:256] Updating pod openshift-console/console-6b85c448d-wfs26 I0424 15:11:04.666387 3680 pod.go:62] [openshift-console/console-6b85c448d-wfs26] pod update took 22.592361ms I0424 15:11:04.666415 3680 base_network_controller_pods.go:951] [default/openshift-console/console-6b85c448d-wfs26] addLogicalPort annotation time took 22.625788ms I0424 15:11:04.667573 3680 pods.go:271] [openshift-console/console-6b85c448d-wfs26] addLogicalPort took 23.876761ms, libovsdb time 824.622µs I0424 15:11:05.006078 3680 cni.go:452] [openshift-console/console-6b85c448d-wfs26 d96ecb8b54924c29aadce35774da7a30cc66d063ce1c7d532d555422a4342c70 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-6b85c448d-wfs26 d96ecb8b54924c29aadce35774da7a30cc66d063ce1c7d532d555422a4342c70 network default NAD default NAD key ] I0424 15:11:05.015552 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-6b85c448d-wfs26, hostIfaceName: d96ecb8b54924c2, network: default, NAD default, SandboxID: "d96ecb8b54924c29aadce35774da7a30cc66d063ce1c7d532d555422a4342c70", PCI device ID: , UID: "57cb2aad-3c8b-4000-9462-21f5cdac6a6a", MAC: 0a:58:0a:82:00:1a, IPs: [10.130.0.26/23] I0424 15:11:05.063737 3680 cni.go:473] [openshift-console/console-6b85c448d-wfs26 d96ecb8b54924c29aadce35774da7a30cc66d063ce1c7d532d555422a4342c70 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-6b85c448d-wfs26 d96ecb8b54924c29aadce35774da7a30cc66d063ce1c7d532d555422a4342c70 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:e7:56:c9:21:76\",\"name\":\"d96ecb8b54924c2\"},{\"mac\":\"0a:58:0a:82:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/721f48a4-051a-4698-bad2-283decf5fc0d\"}],\"ips\":[{\"address\":\"10.130.0.26/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:11:15.051315 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0424 15:11:16.197989 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=5 I0424 15:11:21.083212 3680 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=140 I0424 15:11:29.931813 3680 obj_retry.go:498] Detected object openshift-console/console-6b99c95d6c-7rsz2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:11:29.932413 3680 obj_retry.go:498] Detected object openshift-console/console-6b99c95d6c-7rsz2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:11:30.125187 3680 obj_retry.go:492] Detected object openshift-console/console-6b99c95d6c-7rsz2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:11:30.125207 3680 obj_retry.go:492] Detected object openshift-console/console-6b99c95d6c-7rsz2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:11:30.133987 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6b99c95d6c-7rsz2 I0424 15:11:30.134011 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6b99c95d6c-7rsz2 I0424 15:11:37.082476 3680 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=62 I0424 15:11:40.368044 3680 obj_retry.go:498] Detected object openshift-console/console-6b99c95d6c-28ghz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:11:40.368604 3680 obj_retry.go:498] Detected object openshift-console/console-6b99c95d6c-28ghz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:11:41.358379 3680 obj_retry.go:492] Detected object openshift-console/console-6b99c95d6c-28ghz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:11:41.358405 3680 obj_retry.go:492] Detected object openshift-console/console-6b99c95d6c-28ghz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:11:41.362331 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6b99c95d6c-28ghz I0424 15:11:41.362352 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6b99c95d6c-28ghz I0424 15:12:18.087702 3680 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0424 15:12:31.204186 3680 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0424 15:13:10.085521 3680 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=71 I0424 15:13:33.193818 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0424 15:13:51.177402 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0424 15:14:18.209535 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0424 15:14:25.188411 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0424 15:15:03.183006 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0424 15:15:22.173096 3680 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0424 15:15:23.082065 3680 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0424 15:16:07.211435 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0424 15:16:25.331286 3680 namespace.go:98] [tsf] adding namespace I0424 15:16:25.332100 3680 namespace.go:102] [tsf] adding namespace took 795.536µs I0424 15:16:25.351588 3680 namespace.go:142] [tsf] updating namespace I0424 15:16:25.363741 3680 namespace.go:142] [tsf] updating namespace I0424 15:16:29.275747 3680 namespace.go:98] [tsf-tas] adding namespace I0424 15:16:29.275808 3680 namespace.go:98] [tsf-keycloak] adding namespace I0424 15:16:29.276713 3680 namespace.go:102] [tsf-tas] adding namespace took 941.088µs I0424 15:16:29.277346 3680 namespace.go:102] [tsf-keycloak] adding namespace took 1.522167ms I0424 15:16:29.277820 3680 namespace.go:98] [rhbk-operator] adding namespace I0424 15:16:29.278280 3680 namespace.go:98] [konflux-operator] adding namespace I0424 15:16:29.278460 3680 namespace.go:102] [rhbk-operator] adding namespace took 625.096µs I0424 15:16:29.278987 3680 namespace.go:102] [konflux-operator] adding namespace took 687.271µs I0424 15:16:29.282651 3680 namespace.go:98] [rhtpa-operator] adding namespace I0424 15:16:29.282793 3680 namespace.go:98] [cert-manager-operator] adding namespace I0424 15:16:29.282822 3680 namespace.go:98] [konflux-ui] adding namespace I0424 15:16:29.283385 3680 namespace.go:102] [rhtpa-operator] adding namespace took 721.579µs I0424 15:16:29.283556 3680 namespace.go:98] [tsf-tpa] adding namespace I0424 15:16:29.284117 3680 namespace.go:102] [cert-manager-operator] adding namespace took 1.310446ms I0424 15:16:29.284693 3680 namespace.go:102] [konflux-ui] adding namespace took 1.85258ms I0424 15:16:29.285341 3680 namespace.go:102] [tsf-tpa] adding namespace took 1.769623ms I0424 15:16:29.304841 3680 namespace.go:142] [tsf-tas] updating namespace I0424 15:16:29.323448 3680 namespace.go:142] [tsf-tas] updating namespace I0424 15:16:29.329590 3680 namespace.go:142] [konflux-operator] updating namespace I0424 15:16:29.329625 3680 namespace.go:142] [rhbk-operator] updating namespace I0424 15:16:29.329772 3680 namespace.go:142] [rhtpa-operator] updating namespace I0424 15:16:29.329834 3680 namespace.go:142] [tsf-keycloak] updating namespace I0424 15:16:29.348785 3680 namespace.go:142] [tsf-tpa] updating namespace I0424 15:16:29.348811 3680 namespace.go:142] [tsf-tas] updating namespace I0424 15:16:29.349008 3680 namespace.go:142] [konflux-ui] updating namespace I0424 15:16:29.349032 3680 namespace.go:142] [cert-manager-operator] updating namespace I0424 15:16:29.383118 3680 namespace.go:142] [rhbk-operator] updating namespace I0424 15:16:29.403628 3680 namespace.go:142] [rhbk-operator] updating namespace I0424 15:16:29.411726 3680 namespace.go:142] [konflux-operator] updating namespace I0424 15:16:29.425611 3680 namespace.go:142] [konflux-operator] updating namespace I0424 15:16:29.429276 3680 namespace.go:142] [rhtpa-operator] updating namespace I0424 15:16:29.443661 3680 namespace.go:142] [rhtpa-operator] updating namespace I0424 15:16:29.454790 3680 namespace.go:142] [cert-manager-operator] updating namespace I0424 15:16:29.469043 3680 namespace.go:142] [cert-manager-operator] updating namespace I0424 15:16:29.478487 3680 namespace.go:142] [konflux-ui] updating namespace I0424 15:16:29.492887 3680 namespace.go:142] [konflux-ui] updating namespace I0424 15:16:29.498427 3680 namespace.go:142] [tsf-tpa] updating namespace I0424 15:16:29.512205 3680 namespace.go:142] [tsf-tpa] updating namespace I0424 15:16:29.518495 3680 namespace.go:142] [tsf-keycloak] updating namespace I0424 15:16:29.536255 3680 namespace.go:142] [tsf-keycloak] updating namespace I0424 15:16:30.423723 3680 namespace.go:142] [tsf-tpa] updating namespace I0424 15:16:30.439271 3680 namespace.go:142] [tsf-keycloak] updating namespace I0424 15:16:37.231197 3680 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158vrr47 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:16:37.231673 3680 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158vrr47 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:16:37.231696 3680 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158vrr47 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:16:37.231705 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0424 15:16:37.231713 3680 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158vrr47 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:16:37.231717 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0424 15:16:38.257857 3680 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158vrr47 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:38.257878 3680 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158vrr47 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:38.257892 3680 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158vrr47 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:38.257902 3680 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158vrr47 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:38.269566 3680 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gxl95 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:16:38.270088 3680 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gxl95 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:16:38.270109 3680 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gxl95 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:16:38.270117 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0424 15:16:38.270124 3680 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gxl95 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:16:38.270129 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0424 15:16:39.296442 3680 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gxl95 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:39.296464 3680 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gxl95 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:39.296478 3680 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gxl95 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:39.296517 3680 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gxl95 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:44.635047 3680 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7tdw9k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:16:44.635529 3680 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7tdw9k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:16:44.635549 3680 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7tdw9k of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:16:44.635556 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0424 15:16:44.635566 3680 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7tdw9k of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:16:44.635573 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0424 15:16:45.669440 3680 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7tdw9k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:45.669465 3680 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7tdw9k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:45.669481 3680 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7tdw9k of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:45.669493 3680 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7tdw9k of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:45.669550 3680 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gw4n9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:16:45.670046 3680 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gw4n9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:16:45.670071 3680 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gw4n9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:16:45.670078 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0424 15:16:45.670086 3680 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gw4n9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:16:45.670090 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0424 15:16:45.697366 3680 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tz6sf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:16:45.697719 3680 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tz6sf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:16:45.697737 3680 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tz6sf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:16:45.697743 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0424 15:16:45.697749 3680 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tz6sf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:16:45.697753 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0424 15:16:46.083074 3680 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mknsg] creating logical port tsf-tpa_rhtpa-operator-controller-manager-586cbb496b-mknsg for pod on switch ip-10-0-1-253.ec2.internal I0424 15:16:46.083165 3680 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mknsg I0424 15:16:46.102425 3680 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mknsg] pod update took 19.285887ms I0424 15:16:46.102445 3680 base_network_controller_pods.go:951] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mknsg] addLogicalPort annotation time took 19.309442ms I0424 15:16:46.103563 3680 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mknsg] addLogicalPort took 20.501646ms, libovsdb time 810.021µs I0424 15:16:46.442160 3680 cni.go:452] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mknsg 0de1cee9f72ea1078b6e504e1d2217b2c0c4825abbfe9cf1e42d93f9586a84f3 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mknsg 0de1cee9f72ea1078b6e504e1d2217b2c0c4825abbfe9cf1e42d93f9586a84f3 network default NAD default NAD key ] I0424 15:16:46.451049 3680 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-586cbb496b-mknsg, hostIfaceName: 0de1cee9f72ea10, network: default, NAD default, SandboxID: "0de1cee9f72ea1078b6e504e1d2217b2c0c4825abbfe9cf1e42d93f9586a84f3", PCI device ID: , UID: "98151614-4646-497c-9c33-60412162c952", MAC: 0a:58:0a:82:00:1b, IPs: [10.130.0.27/23] I0424 15:16:46.499410 3680 cni.go:473] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mknsg 0de1cee9f72ea1078b6e504e1d2217b2c0c4825abbfe9cf1e42d93f9586a84f3 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mknsg 0de1cee9f72ea1078b6e504e1d2217b2c0c4825abbfe9cf1e42d93f9586a84f3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:f1:b1:0f:35:c3\",\"name\":\"0de1cee9f72ea10\"},{\"mac\":\"0a:58:0a:82:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/db71ea85-a232-4e95-aeac-90db988aafa2\"}],\"ips\":[{\"address\":\"10.130.0.27/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:16:46.659645 3680 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d89jcz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:16:46.660109 3680 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d89jcz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:16:46.660138 3680 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d89jcz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:16:46.660145 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0424 15:16:46.660154 3680 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d89jcz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 15:16:46.660158 3680 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0424 15:16:46.702246 3680 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gw4n9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:46.702263 3680 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gw4n9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:46.702281 3680 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gw4n9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:46.702290 3680 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gw4n9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:46.729550 3680 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tz6sf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:46.729567 3680 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tz6sf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:46.729583 3680 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tz6sf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:46.729593 3680 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tz6sf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:47.688366 3680 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d89jcz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:47.688398 3680 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d89jcz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:47.688422 3680 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d89jcz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:16:47.688437 3680 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d89jcz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0424 15:16:49.368500 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0424 15:16:54.441286 3680 namespace.go:142] [openshift-operators] updating namespace I0424 15:16:54.451131 3680 namespace.go:142] [openshift-operators] updating namespace W0424 15:16:55.483758 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0424 15:16:56.991267 3680 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-fb86cd8d9-6wk7f] creating logical port rhbk-operator_rhbk-operator-fb86cd8d9-6wk7f for pod on switch ip-10-0-1-253.ec2.internal I0424 15:16:56.991479 3680 kube.go:256] Updating pod rhbk-operator/rhbk-operator-fb86cd8d9-6wk7f I0424 15:16:57.013169 3680 pod.go:62] [rhbk-operator/rhbk-operator-fb86cd8d9-6wk7f] pod update took 21.726556ms I0424 15:16:57.013190 3680 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-fb86cd8d9-6wk7f] addLogicalPort annotation time took 21.752649ms I0424 15:16:57.014407 3680 pods.go:271] [rhbk-operator/rhbk-operator-fb86cd8d9-6wk7f] addLogicalPort took 23.15622ms, libovsdb time 887.316µs I0424 15:16:57.349263 3680 cni.go:452] [rhbk-operator/rhbk-operator-fb86cd8d9-6wk7f a69d837e19045831344d521ca8614d700e494d35ba67c439a962f48360a69e53 network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-6wk7f a69d837e19045831344d521ca8614d700e494d35ba67c439a962f48360a69e53 network default NAD default NAD key ] I0424 15:16:57.359037 3680 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-fb86cd8d9-6wk7f, hostIfaceName: a69d837e1904583, network: default, NAD default, SandboxID: "a69d837e19045831344d521ca8614d700e494d35ba67c439a962f48360a69e53", PCI device ID: , UID: "368e824a-beab-4df1-b576-eea50c9d2e83", MAC: 0a:58:0a:82:00:1c, IPs: [10.130.0.28/23] I0424 15:16:57.409002 3680 cni.go:473] [rhbk-operator/rhbk-operator-fb86cd8d9-6wk7f a69d837e19045831344d521ca8614d700e494d35ba67c439a962f48360a69e53 network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-6wk7f a69d837e19045831344d521ca8614d700e494d35ba67c439a962f48360a69e53 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:01:6a:26:1b:70\",\"name\":\"a69d837e1904583\"},{\"mac\":\"0a:58:0a:82:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0fc6ab7c-bf1e-426c-819e-783c4f898bc7\"}],\"ips\":[{\"address\":\"10.130.0.28/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:16:59.980211 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0424 15:16:59.993549 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0424 15:17:00.302041 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0424 15:17:00.310347 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0424 15:17:00.394715 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0424 15:17:04.517980 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0424 15:17:04.609385 3680 namespace.go:98] [trusted-artifact-signer] adding namespace I0424 15:17:04.610286 3680 namespace.go:102] [trusted-artifact-signer] adding namespace took 874.763µs I0424 15:17:04.650685 3680 namespace.go:142] [trusted-artifact-signer] updating namespace I0424 15:17:04.674670 3680 namespace.go:142] [trusted-artifact-signer] updating namespace I0424 15:17:08.080802 3680 namespace.go:142] [openshift-operators] updating namespace I0424 15:17:19.188550 3680 namespace.go:98] [cert-manager] adding namespace I0424 15:17:19.189389 3680 namespace.go:102] [cert-manager] adding namespace took 820.054µs I0424 15:17:19.254484 3680 namespace.go:142] [cert-manager] updating namespace I0424 15:17:19.295304 3680 namespace.go:142] [cert-manager] updating namespace I0424 15:17:20.084650 3680 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=31 I0424 15:17:20.243549 3680 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-7dc76cbb94-rpl9g] creating logical port cert-manager_cert-manager-webhook-7dc76cbb94-rpl9g for pod on switch ip-10-0-1-253.ec2.internal I0424 15:17:20.243649 3680 kube.go:256] Updating pod cert-manager/cert-manager-webhook-7dc76cbb94-rpl9g I0424 15:17:20.264904 3680 pod.go:62] [cert-manager/cert-manager-webhook-7dc76cbb94-rpl9g] pod update took 21.291316ms I0424 15:17:20.264939 3680 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-webhook-7dc76cbb94-rpl9g] addLogicalPort annotation time took 21.314311ms I0424 15:17:20.266090 3680 pods.go:271] [cert-manager/cert-manager-webhook-7dc76cbb94-rpl9g] addLogicalPort took 22.559951ms, libovsdb time 833.198µs I0424 15:17:20.642651 3680 cni.go:452] [cert-manager/cert-manager-webhook-7dc76cbb94-rpl9g f72d0eaa18342a16f9b4c980912a8d0aea46863362102a9b3bfa0aecf5cecf43 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-rpl9g f72d0eaa18342a16f9b4c980912a8d0aea46863362102a9b3bfa0aecf5cecf43 network default NAD default NAD key ] I0424 15:17:20.652611 3680 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-7dc76cbb94-rpl9g, hostIfaceName: f72d0eaa18342a1, network: default, NAD default, SandboxID: "f72d0eaa18342a16f9b4c980912a8d0aea46863362102a9b3bfa0aecf5cecf43", PCI device ID: , UID: "fe90184a-252b-4633-812e-2a1394047ba0", MAC: 0a:58:0a:82:00:1d, IPs: [10.130.0.29/23] I0424 15:17:20.704640 3680 cni.go:473] [cert-manager/cert-manager-webhook-7dc76cbb94-rpl9g f72d0eaa18342a16f9b4c980912a8d0aea46863362102a9b3bfa0aecf5cecf43 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-rpl9g f72d0eaa18342a16f9b4c980912a8d0aea46863362102a9b3bfa0aecf5cecf43 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:7b:de:3e:4f:1b\",\"name\":\"f72d0eaa18342a1\"},{\"mac\":\"0a:58:0a:82:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b8c97b71-464c-49c5-9bcb-de39db38e6ec\"}],\"ips\":[{\"address\":\"10.130.0.29/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:17:23.542271 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0424 15:17:23.542290 3680 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0424 15:17:26.046744 3680 namespace.go:98] [openshift-pipelines] adding namespace I0424 15:17:26.047621 3680 namespace.go:102] [openshift-pipelines] adding namespace took 854.551µs I0424 15:17:26.074670 3680 namespace.go:142] [openshift-pipelines] updating namespace I0424 15:17:27.645881 3680 namespace.go:142] [cert-manager] updating namespace I0424 15:17:27.984053 3680 namespace.go:142] [cert-manager-operator] updating namespace I0424 15:17:28.406693 3680 namespace.go:142] [konflux-operator] updating namespace I0424 15:17:28.614818 3680 namespace.go:142] [konflux-ui] updating namespace I0424 15:17:28.815154 3680 namespace.go:142] [open-cluster-management-2psja0jgf3iq01e6l9t02c7pqkonai87] updating namespace I0424 15:17:29.208139 3680 namespace.go:142] [rhbk-operator] updating namespace I0424 15:17:29.410715 3680 namespace.go:142] [rhtpa-operator] updating namespace W0424 15:17:29.545871 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0424 15:17:29.545895 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0424 15:17:29.608431 3680 namespace.go:142] [trusted-artifact-signer] updating namespace I0424 15:17:29.811643 3680 namespace.go:142] [tsf] updating namespace I0424 15:17:30.010429 3680 namespace.go:142] [tsf-keycloak] updating namespace I0424 15:17:30.208577 3680 namespace.go:142] [tsf-tas] updating namespace I0424 15:17:30.410468 3680 namespace.go:142] [tsf-tpa] updating namespace I0424 15:17:30.547791 3680 namespace.go:142] [cert-manager] updating namespace I0424 15:17:30.563386 3680 namespace.go:142] [cert-manager-operator] updating namespace I0424 15:17:30.573144 3680 namespace.go:142] [default] updating namespace I0424 15:17:30.584264 3680 namespace.go:142] [konflux-operator] updating namespace I0424 15:17:30.594715 3680 namespace.go:142] [konflux-ui] updating namespace I0424 15:17:30.603862 3680 namespace.go:142] [open-cluster-management-2psja0jgf3iq01e6l9t02c7pqkonai87] updating namespace I0424 15:17:30.614761 3680 namespace.go:142] [openshift] updating namespace I0424 15:17:30.627004 3680 namespace.go:142] [rhbk-operator] updating namespace I0424 15:17:30.636409 3680 namespace.go:142] [rhtpa-operator] updating namespace I0424 15:17:30.645910 3680 namespace.go:142] [trusted-artifact-signer] updating namespace I0424 15:17:30.654628 3680 namespace.go:142] [tsf] updating namespace I0424 15:17:30.665174 3680 namespace.go:142] [tsf-keycloak] updating namespace I0424 15:17:30.675238 3680 namespace.go:142] [tsf-tas] updating namespace I0424 15:17:30.691767 3680 namespace.go:142] [tsf-tpa] updating namespace I0424 15:17:30.806735 3680 namespace.go:142] [cert-manager] updating namespace I0424 15:17:30.896400 3680 namespace.go:142] [cert-manager-operator] updating namespace I0424 15:17:30.993993 3680 namespace.go:142] [default] updating namespace I0424 15:17:31.095317 3680 namespace.go:142] [konflux-operator] updating namespace I0424 15:17:31.198089 3680 namespace.go:142] [konflux-ui] updating namespace I0424 15:17:31.295181 3680 namespace.go:142] [open-cluster-management-2psja0jgf3iq01e6l9t02c7pqkonai87] updating namespace I0424 15:17:31.393173 3680 namespace.go:142] [openshift] updating namespace I0424 15:17:31.495639 3680 namespace.go:142] [rhbk-operator] updating namespace I0424 15:17:31.595407 3680 namespace.go:142] [rhtpa-operator] updating namespace I0424 15:17:31.693799 3680 namespace.go:142] [trusted-artifact-signer] updating namespace I0424 15:17:31.795852 3680 namespace.go:142] [tsf] updating namespace I0424 15:17:31.894271 3680 namespace.go:142] [tsf-keycloak] updating namespace I0424 15:17:31.995577 3680 namespace.go:142] [tsf-tas] updating namespace I0424 15:17:32.100009 3680 namespace.go:142] [tsf-tpa] updating namespace I0424 15:17:32.506426 3680 namespace.go:142] [openshift-pipelines] updating namespace I0424 15:17:32.906021 3680 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:17:32.906572 3680 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:17:32.943993 3680 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:17:32.944015 3680 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:17:32.953982 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0424 15:17:32.954011 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0424 15:17:34.207673 3680 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=5 I0424 15:17:40.231599 3680 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-4njrn] creating logical port openshift-pipelines_tekton-pipelines-controller-86bd8d9bfc-4njrn for pod on switch ip-10-0-1-253.ec2.internal I0424 15:17:40.231691 3680 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-4njrn I0424 15:17:40.249980 3680 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-4njrn] pod update took 18.321236ms I0424 15:17:40.250004 3680 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-4njrn] addLogicalPort annotation time took 18.346073ms I0424 15:17:40.251130 3680 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-4njrn] addLogicalPort took 19.546946ms, libovsdb time 772.701µs I0424 15:17:40.590206 3680 cni.go:452] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-4njrn c3ca6ee17f03c5f705b72d537fd03aa3940f7be49c2137390b801a11a351a28b network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-4njrn c3ca6ee17f03c5f705b72d537fd03aa3940f7be49c2137390b801a11a351a28b network default NAD default NAD key ] I0424 15:17:40.599390 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-86bd8d9bfc-4njrn, hostIfaceName: c3ca6ee17f03c5f, network: default, NAD default, SandboxID: "c3ca6ee17f03c5f705b72d537fd03aa3940f7be49c2137390b801a11a351a28b", PCI device ID: , UID: "a9d4f4d5-2713-4336-95c9-1428e4b7ef0b", MAC: 0a:58:0a:82:00:1e, IPs: [10.130.0.30/23] I0424 15:17:40.649550 3680 cni.go:473] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-4njrn c3ca6ee17f03c5f705b72d537fd03aa3940f7be49c2137390b801a11a351a28b network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-4njrn c3ca6ee17f03c5f705b72d537fd03aa3940f7be49c2137390b801a11a351a28b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:c9:84:32:57:d9\",\"name\":\"c3ca6ee17f03c5f\"},{\"mac\":\"0a:58:0a:82:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/50516171-f9a7-470e-8c99-a00c0cf4133a\"}],\"ips\":[{\"address\":\"10.130.0.30/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:17:43.635801 3680 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0424 15:17:43.635826 3680 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0424 15:17:43.635832 3680 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0424 15:17:43.717447 3680 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0424 15:17:43.717464 3680 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0424 15:17:43.717469 3680 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0424 15:17:44.200809 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 W0424 15:17:44.722005 3680 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0424 15:17:44.722021 3680 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0424 15:17:44.722025 3680 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0424 15:17:44.722029 3680 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0424 15:17:48.861514 3680 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0424 15:17:48.883367 3680 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0424 15:17:54.629060 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0424 15:17:54.629078 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0424 15:17:54.629084 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0424 15:18:02.510191 3680 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-785c845664-84lw6] creating logical port openshift-pipelines_tekton-triggers-controller-785c845664-84lw6 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:18:02.510293 3680 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-785c845664-84lw6 I0424 15:18:02.528828 3680 pod.go:62] [openshift-pipelines/tekton-triggers-controller-785c845664-84lw6] pod update took 18.565441ms I0424 15:18:02.528850 3680 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-controller-785c845664-84lw6] addLogicalPort annotation time took 18.592183ms I0424 15:18:02.530069 3680 pods.go:271] [openshift-pipelines/tekton-triggers-controller-785c845664-84lw6] addLogicalPort took 19.891999ms, libovsdb time 843.155µs I0424 15:18:02.645454 3680 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z4nmn] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-z4nmn for pod on switch ip-10-0-1-253.ec2.internal I0424 15:18:02.645545 3680 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z4nmn I0424 15:18:02.661533 3680 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z4nmn] pod update took 16.012675ms I0424 15:18:02.661557 3680 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z4nmn] addLogicalPort annotation time took 16.03955ms I0424 15:18:02.662775 3680 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z4nmn] addLogicalPort took 17.335317ms, libovsdb time 861.664µs I0424 15:18:02.749475 3680 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-sx899] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-76f587ddc9-sx899 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:18:02.749570 3680 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-sx899 I0424 15:18:02.766043 3680 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-sx899] pod update took 16.495018ms I0424 15:18:02.766066 3680 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-sx899] addLogicalPort annotation time took 16.525673ms I0424 15:18:02.767189 3680 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-sx899] addLogicalPort took 17.731686ms, libovsdb time 786.292µs I0424 15:18:02.865710 3680 cni.go:452] [openshift-pipelines/tekton-triggers-controller-785c845664-84lw6 53a28fd55319304697c4c714657164cc343015aedd647e8ee4b62dd7eeb26439 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-785c845664-84lw6 53a28fd55319304697c4c714657164cc343015aedd647e8ee4b62dd7eeb26439 network default NAD default NAD key ] I0424 15:18:02.875072 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-785c845664-84lw6, hostIfaceName: 53a28fd55319304, network: default, NAD default, SandboxID: "53a28fd55319304697c4c714657164cc343015aedd647e8ee4b62dd7eeb26439", PCI device ID: , UID: "61234b1e-5303-4ee7-8747-8ce25c5aac38", MAC: 0a:58:0a:82:00:1f, IPs: [10.130.0.31/23] I0424 15:18:02.924433 3680 cni.go:473] [openshift-pipelines/tekton-triggers-controller-785c845664-84lw6 53a28fd55319304697c4c714657164cc343015aedd647e8ee4b62dd7eeb26439 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-785c845664-84lw6 53a28fd55319304697c4c714657164cc343015aedd647e8ee4b62dd7eeb26439 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:f5:e8:32:a7:41\",\"name\":\"53a28fd55319304\"},{\"mac\":\"0a:58:0a:82:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/45acac80-80cb-42a2-87e3-6af6fc0a6770\"}],\"ips\":[{\"address\":\"10.130.0.31/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:18:03.003585 3680 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z4nmn edac0b73676227ecaf9be8a9e5a0823fa59cdfc391329f25d50e0517547859f8 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z4nmn edac0b73676227ecaf9be8a9e5a0823fa59cdfc391329f25d50e0517547859f8 network default NAD default NAD key ] I0424 15:18:03.013607 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-z4nmn, hostIfaceName: edac0b73676227e, network: default, NAD default, SandboxID: "edac0b73676227ecaf9be8a9e5a0823fa59cdfc391329f25d50e0517547859f8", PCI device ID: , UID: "5ac246f9-add4-4a55-876b-49aac5443369", MAC: 0a:58:0a:82:00:20, IPs: [10.130.0.32/23] I0424 15:18:03.064468 3680 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z4nmn edac0b73676227ecaf9be8a9e5a0823fa59cdfc391329f25d50e0517547859f8 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-z4nmn edac0b73676227ecaf9be8a9e5a0823fa59cdfc391329f25d50e0517547859f8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:f2:53:28:c0:43\",\"name\":\"edac0b73676227e\"},{\"mac\":\"0a:58:0a:82:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1f9a6746-2f0f-44d6-86e3-4e3e7b1e4cf7\"}],\"ips\":[{\"address\":\"10.130.0.32/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:18:03.107326 3680 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-sx899 a61ad60624616a15ce5aa6f282d86989c6943e87e0ad0bcc60acddf582d3d91f network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-sx899 a61ad60624616a15ce5aa6f282d86989c6943e87e0ad0bcc60acddf582d3d91f network default NAD default NAD key ] I0424 15:18:03.116964 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-76f587ddc9-sx899, hostIfaceName: a61ad60624616a1, network: default, NAD default, SandboxID: "a61ad60624616a15ce5aa6f282d86989c6943e87e0ad0bcc60acddf582d3d91f", PCI device ID: , UID: "6d02768c-dab1-4936-a87d-3cce72ce01a0", MAC: 0a:58:0a:82:00:21, IPs: [10.130.0.33/23] I0424 15:18:03.167709 3680 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-sx899 a61ad60624616a15ce5aa6f282d86989c6943e87e0ad0bcc60acddf582d3d91f network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-sx899 a61ad60624616a15ce5aa6f282d86989c6943e87e0ad0bcc60acddf582d3d91f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:11:cb:82:04:b5\",\"name\":\"a61ad60624616a1\"},{\"mac\":\"0a:58:0a:82:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/906274d5-2b02-4ce6-869c-a6602ff2a8f6\"}],\"ips\":[{\"address\":\"10.130.0.33/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:18:06.485493 3680 namespace.go:142] [openshift-pipelines] updating namespace W0424 15:18:06.803569 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0424 15:18:13.123341 3680 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:18:13.123860 3680 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:18:13.144964 3680 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:18:13.144984 3680 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:18:13.153071 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0424 15:18:13.153092 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0424 15:18:15.098042 3680 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=264 I0424 15:18:15.120404 3680 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0424 15:18:15.121095 3680 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0424 15:18:15.121114 3680 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0424 15:18:15.123457 3680 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel W0424 15:18:17.743845 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0424 15:18:21.057502 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0424 15:18:21.057519 3680 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0424 15:18:21.060324 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0424 15:18:21.060339 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0424 15:18:24.175481 3680 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0424 15:18:25.076999 3680 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0424 15:18:28.165260 3680 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:18:28.165351 3680 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0424 15:18:28.180894 3680 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 15.566087ms I0424 15:18:28.180917 3680 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 15.593961ms I0424 15:18:28.182120 3680 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 16.888395ms, libovsdb time 809.148µs I0424 15:18:34.526402 3680 cni.go:452] [openshift-pipelines/tekton-results-postgres-0 de6f49883500e89d15b297c12ff1ee4c84c85e5e9fa02bcb4c495b579e5368a5 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 de6f49883500e89d15b297c12ff1ee4c84c85e5e9fa02bcb4c495b579e5368a5 network default NAD default NAD key ] I0424 15:18:34.536297 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: de6f49883500e89, network: default, NAD default, SandboxID: "de6f49883500e89d15b297c12ff1ee4c84c85e5e9fa02bcb4c495b579e5368a5", PCI device ID: , UID: "cc8c4b78-6a52-4361-bea5-53b2a6b96778", MAC: 0a:58:0a:82:00:22, IPs: [10.130.0.34/23] I0424 15:18:34.588521 3680 cni.go:473] [openshift-pipelines/tekton-results-postgres-0 de6f49883500e89d15b297c12ff1ee4c84c85e5e9fa02bcb4c495b579e5368a5 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 de6f49883500e89d15b297c12ff1ee4c84c85e5e9fa02bcb4c495b579e5368a5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:28:1e:89:14:53\",\"name\":\"de6f49883500e89\"},{\"mac\":\"0a:58:0a:82:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3dd1a3f0-db89-4286-a672-8a96e8d23481\"}],\"ips\":[{\"address\":\"10.130.0.34/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:18:40.928757 3680 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0424 15:18:42.375586 3680 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vlcfv] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-585694b54-vlcfv for pod on switch ip-10-0-1-253.ec2.internal I0424 15:18:42.375664 3680 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vlcfv I0424 15:18:42.389533 3680 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vlcfv] pod update took 13.883159ms I0424 15:18:42.389558 3680 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vlcfv] addLogicalPort annotation time took 13.914978ms I0424 15:18:42.390851 3680 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vlcfv] addLogicalPort took 15.277263ms, libovsdb time 867.872µs I0424 15:18:42.474559 3680 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-6l2kl] creating logical port openshift-pipelines_tekton-results-watcher-7777c5dbd5-6l2kl for pod on switch ip-10-0-1-253.ec2.internal I0424 15:18:42.474663 3680 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-7777c5dbd5-6l2kl I0424 15:18:42.490626 3680 pod.go:62] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-6l2kl] pod update took 15.98095ms I0424 15:18:42.490648 3680 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-6l2kl] addLogicalPort annotation time took 16.008655ms I0424 15:18:42.491939 3680 pods.go:271] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-6l2kl] addLogicalPort took 17.37608ms, libovsdb time 866.763µs I0424 15:18:42.734982 3680 cni.go:452] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vlcfv 653eec45b9722c55e666f90f468ddc424b1b972c04e886a32f5696c8538173ba network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vlcfv 653eec45b9722c55e666f90f468ddc424b1b972c04e886a32f5696c8538173ba network default NAD default NAD key ] I0424 15:18:42.744793 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-585694b54-vlcfv, hostIfaceName: 653eec45b9722c5, network: default, NAD default, SandboxID: "653eec45b9722c55e666f90f468ddc424b1b972c04e886a32f5696c8538173ba", PCI device ID: , UID: "1fc8a4a3-f83c-453f-b0e2-95555573dbf9", MAC: 0a:58:0a:82:00:23, IPs: [10.130.0.35/23] I0424 15:18:42.795143 3680 cni.go:473] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vlcfv 653eec45b9722c55e666f90f468ddc424b1b972c04e886a32f5696c8538173ba network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vlcfv 653eec45b9722c55e666f90f468ddc424b1b972c04e886a32f5696c8538173ba network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:7e:25:58:8e:5d\",\"name\":\"653eec45b9722c5\"},{\"mac\":\"0a:58:0a:82:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4d9742b2-1054-439d-81d1-d67e5ff8c731\"}],\"ips\":[{\"address\":\"10.130.0.35/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:18:42.835307 3680 cni.go:452] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-6l2kl a3c1018484c2560f824a69c1cbbb12c7d9bd6a9e8338d45360e06f0b4d552d7e network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-6l2kl a3c1018484c2560f824a69c1cbbb12c7d9bd6a9e8338d45360e06f0b4d552d7e network default NAD default NAD key ] I0424 15:18:42.860265 3680 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-7777c5dbd5-6l2kl, hostIfaceName: a3c1018484c2560, network: default, NAD default, SandboxID: "a3c1018484c2560f824a69c1cbbb12c7d9bd6a9e8338d45360e06f0b4d552d7e", PCI device ID: , UID: "3c01d6c8-38e8-4b13-aece-628a5f6d6680", MAC: 0a:58:0a:82:00:24, IPs: [10.130.0.36/23] I0424 15:18:42.908512 3680 cni.go:473] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-6l2kl a3c1018484c2560f824a69c1cbbb12c7d9bd6a9e8338d45360e06f0b4d552d7e network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-6l2kl a3c1018484c2560f824a69c1cbbb12c7d9bd6a9e8338d45360e06f0b4d552d7e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:92:94:9b:2d:46\",\"name\":\"a3c1018484c2560\"},{\"mac\":\"0a:58:0a:82:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4d4f9678-52ad-4d62-95a2-a932636998a3\"}],\"ips\":[{\"address\":\"10.130.0.36/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:18:46.030769 3680 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0424 15:18:46.030797 3680 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0424 15:18:46.030804 3680 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0424 15:18:47.352600 3680 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:18:47.353104 3680 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:18:47.402883 3680 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:18:47.402899 3680 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:18:47.409672 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0424 15:18:47.409695 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0424 15:18:50.243746 3680 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-v5dkc-4tljb] creating logical port tsf-tas_ctlog-createtree-job-v5dkc-4tljb for pod on switch ip-10-0-1-253.ec2.internal I0424 15:18:50.243842 3680 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-v5dkc-4tljb I0424 15:18:50.258716 3680 pod.go:62] [tsf-tas/ctlog-createtree-job-v5dkc-4tljb] pod update took 14.896029ms I0424 15:18:50.258739 3680 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-createtree-job-v5dkc-4tljb] addLogicalPort annotation time took 14.924019ms I0424 15:18:50.260068 3680 pods.go:271] [tsf-tas/ctlog-createtree-job-v5dkc-4tljb] addLogicalPort took 16.336734ms, libovsdb time 945.296µs I0424 15:18:50.324724 3680 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-79f7dcd86c-q8sz5] creating logical port tsf-tas_trillian-logserver-79f7dcd86c-q8sz5 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:18:50.324798 3680 kube.go:256] Updating pod tsf-tas/trillian-logserver-79f7dcd86c-q8sz5 I0424 15:18:50.340681 3680 pod.go:62] [tsf-tas/trillian-logserver-79f7dcd86c-q8sz5] pod update took 15.903187ms I0424 15:18:50.340703 3680 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logserver-79f7dcd86c-q8sz5] addLogicalPort annotation time took 15.927898ms I0424 15:18:50.341808 3680 pods.go:271] [tsf-tas/trillian-logserver-79f7dcd86c-q8sz5] addLogicalPort took 17.103876ms, libovsdb time 728.431µs I0424 15:18:50.567336 3680 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-np2zx-lqt7s] creating logical port tsf-tas_rekor-createtree-job-np2zx-lqt7s for pod on switch ip-10-0-1-253.ec2.internal I0424 15:18:50.567414 3680 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-np2zx-lqt7s I0424 15:18:50.585532 3680 pod.go:62] [tsf-tas/rekor-createtree-job-np2zx-lqt7s] pod update took 18.135832ms I0424 15:18:50.585560 3680 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-np2zx-lqt7s] addLogicalPort annotation time took 18.167872ms I0424 15:18:50.586981 3680 pods.go:271] [tsf-tas/rekor-createtree-job-np2zx-lqt7s] addLogicalPort took 19.665793ms, libovsdb time 974.77µs I0424 15:18:50.605796 3680 cni.go:452] [tsf-tas/ctlog-createtree-job-v5dkc-4tljb b3c6a0a71e3a21a7eb546becba4bfae6d090700c044aeea13922c28c4181c7b6 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-createtree-job-v5dkc-4tljb b3c6a0a71e3a21a7eb546becba4bfae6d090700c044aeea13922c28c4181c7b6 network default NAD default NAD key ] I0424 15:18:50.615461 3680 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-v5dkc-4tljb, hostIfaceName: b3c6a0a71e3a21a, network: default, NAD default, SandboxID: "b3c6a0a71e3a21a7eb546becba4bfae6d090700c044aeea13922c28c4181c7b6", PCI device ID: , UID: "6535b2a3-f666-47c8-b024-0c20acf481aa", MAC: 0a:58:0a:82:00:25, IPs: [10.130.0.37/23] I0424 15:18:50.667558 3680 cni.go:473] [tsf-tas/ctlog-createtree-job-v5dkc-4tljb b3c6a0a71e3a21a7eb546becba4bfae6d090700c044aeea13922c28c4181c7b6 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-v5dkc-4tljb b3c6a0a71e3a21a7eb546becba4bfae6d090700c044aeea13922c28c4181c7b6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:5c:d2:33:4c:3b\",\"name\":\"b3c6a0a71e3a21a\"},{\"mac\":\"0a:58:0a:82:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0432b53-d3e6-4414-8584-6f6aa1172a6e\"}],\"ips\":[{\"address\":\"10.130.0.37/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:18:50.937579 3680 cni.go:452] [tsf-tas/rekor-createtree-job-np2zx-lqt7s afd469d700e8f171b3610f13bb9e3d266f56ae764fd327a00797a5807c333bdb network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-np2zx-lqt7s afd469d700e8f171b3610f13bb9e3d266f56ae764fd327a00797a5807c333bdb network default NAD default NAD key ] I0424 15:18:50.947392 3680 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-np2zx-lqt7s, hostIfaceName: afd469d700e8f17, network: default, NAD default, SandboxID: "afd469d700e8f171b3610f13bb9e3d266f56ae764fd327a00797a5807c333bdb", PCI device ID: , UID: "57c4cdf9-0843-4743-b9c0-03905f98e603", MAC: 0a:58:0a:82:00:27, IPs: [10.130.0.39/23] I0424 15:18:50.998249 3680 cni.go:473] [tsf-tas/rekor-createtree-job-np2zx-lqt7s afd469d700e8f171b3610f13bb9e3d266f56ae764fd327a00797a5807c333bdb network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-np2zx-lqt7s afd469d700e8f171b3610f13bb9e3d266f56ae764fd327a00797a5807c333bdb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:c4:23:19:cf:7b\",\"name\":\"afd469d700e8f17\"},{\"mac\":\"0a:58:0a:82:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e1b99127-9d88-4fa3-ab38-9ec94fc1a736\"}],\"ips\":[{\"address\":\"10.130.0.39/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:18:51.281189 3680 cni.go:452] [tsf-tas/trillian-logserver-79f7dcd86c-q8sz5 d325436d3571a769f67c7237392b69ccccf6aa23f45716ef91fee9261c48d0e8 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logserver-79f7dcd86c-q8sz5 d325436d3571a769f67c7237392b69ccccf6aa23f45716ef91fee9261c48d0e8 network default NAD default NAD key ] I0424 15:18:51.291273 3680 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-79f7dcd86c-q8sz5, hostIfaceName: d325436d3571a76, network: default, NAD default, SandboxID: "d325436d3571a769f67c7237392b69ccccf6aa23f45716ef91fee9261c48d0e8", PCI device ID: , UID: "6e48f62e-23c9-4d4a-94e1-7b3f7df07714", MAC: 0a:58:0a:82:00:26, IPs: [10.130.0.38/23] I0424 15:18:51.341524 3680 cni.go:473] [tsf-tas/trillian-logserver-79f7dcd86c-q8sz5 d325436d3571a769f67c7237392b69ccccf6aa23f45716ef91fee9261c48d0e8 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logserver-79f7dcd86c-q8sz5 d325436d3571a769f67c7237392b69ccccf6aa23f45716ef91fee9261c48d0e8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:ce:80:53:75:f4\",\"name\":\"d325436d3571a76\"},{\"mac\":\"0a:58:0a:82:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0dcaf1b5-4874-410c-bfad-b6554aaf5f52\"}],\"ips\":[{\"address\":\"10.130.0.38/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:18:52.062517 3680 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0424 15:18:52.062547 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0424 15:18:58.008003 3680 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0424 15:19:03.449137 3680 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-mbmrj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:19:03.449677 3680 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-mbmrj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0424 15:19:04.143224 3680 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0424 15:19:04.349354 3680 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-mbmrj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:04.349384 3680 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-mbmrj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:08.585525 3680 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-ndwnb-h48rq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:19:08.586143 3680 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-ndwnb-h48rq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0424 15:19:09.181064 3680 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0424 15:19:09.189988 3680 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-ndwnb-h48rq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:09.190010 3680 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-ndwnb-h48rq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0424 15:19:24.508716 3680 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0424 15:19:31.090907 3680 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=99 W0424 15:19:37.159672 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0424 15:19:38.191682 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0424 15:19:38.661220 3680 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:19:38.661776 3680 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:19:38.683429 3680 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:38.683445 3680 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:38.691143 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0424 15:19:38.691164 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0424 15:19:49.716858 3680 obj_retry.go:498] Detected object tsf/patch-tekton-config-n9hpq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:19:49.717401 3680 obj_retry.go:498] Detected object tsf/patch-tekton-config-n9hpq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:19:50.741956 3680 obj_retry.go:492] Detected object tsf/patch-tekton-config-n9hpq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:50.741978 3680 obj_retry.go:492] Detected object tsf/patch-tekton-config-n9hpq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:51.269768 3680 cni.go:452] [tsf-tas/ctlog-createtree-job-v5dkc-4tljb b3c6a0a71e3a21a7eb546becba4bfae6d090700c044aeea13922c28c4181c7b6 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/ctlog-createtree-job-v5dkc-4tljb b3c6a0a71e3a21a7eb546becba4bfae6d090700c044aeea13922c28c4181c7b6 network default NAD default NAD key ] I0424 15:19:51.361734 3680 cni.go:473] [tsf-tas/ctlog-createtree-job-v5dkc-4tljb b3c6a0a71e3a21a7eb546becba4bfae6d090700c044aeea13922c28c4181c7b6 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-v5dkc-4tljb b3c6a0a71e3a21a7eb546becba4bfae6d090700c044aeea13922c28c4181c7b6 network default NAD default NAD key default], result "{}", err I0424 15:19:51.412561 3680 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-v5dkc-4tljb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:19:51.412595 3680 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-v5dkc-4tljb I0424 15:19:51.413954 3680 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-v5dkc-4tljb, ips: 10.130.0.37 I0424 15:19:51.413986 3680 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-v5dkc-4tljb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:19:52.287552 3680 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-v5dkc-4tljb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:52.287575 3680 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-v5dkc-4tljb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:56.595255 3680 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-24fx9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:19:56.595803 3680 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-24fx9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:19:57.618476 3680 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-24fx9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:57.618497 3680 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-24fx9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:57.659697 3680 obj_retry.go:492] Detected object tsf/patch-tekton-config-n9hpq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:57.659722 3680 obj_retry.go:492] Detected object tsf/patch-tekton-config-n9hpq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:19:57.667085 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-n9hpq I0424 15:19:57.667108 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-n9hpq W0424 15:19:58.642486 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0424 15:20:01.791404 3680 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:01.791953 3680 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:01.816999 3680 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:01.817018 3680 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:01.828480 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0424 15:20:01.828504 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0424 15:20:03.163322 3680 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-dcrxw] creating logical port tsf-tpa_create-db-dcrxw for pod on switch ip-10-0-1-253.ec2.internal I0424 15:20:03.163415 3680 kube.go:256] Updating pod tsf-tpa/create-db-dcrxw I0424 15:20:03.176664 3680 pod.go:62] [tsf-tpa/create-db-dcrxw] pod update took 13.269523ms I0424 15:20:03.176693 3680 base_network_controller_pods.go:951] [default/tsf-tpa/create-db-dcrxw] addLogicalPort annotation time took 13.301586ms I0424 15:20:03.178166 3680 pods.go:271] [tsf-tpa/create-db-dcrxw] addLogicalPort took 14.862039ms, libovsdb time 871.534µs I0424 15:20:03.521842 3680 cni.go:452] [tsf-tpa/create-db-dcrxw 610d8f65511692305ddf9e485a48f164ae431d80a76ccd3f3a09c49fa6cb21ee network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-db-dcrxw 610d8f65511692305ddf9e485a48f164ae431d80a76ccd3f3a09c49fa6cb21ee network default NAD default NAD key ] I0424 15:20:03.530951 3680 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-db-dcrxw, hostIfaceName: 610d8f655116923, network: default, NAD default, SandboxID: "610d8f65511692305ddf9e485a48f164ae431d80a76ccd3f3a09c49fa6cb21ee", PCI device ID: , UID: "29a56dc9-f9a2-46b5-a1fc-e36e4e9e021c", MAC: 0a:58:0a:82:00:28, IPs: [10.130.0.40/23] I0424 15:20:03.582570 3680 cni.go:473] [tsf-tpa/create-db-dcrxw 610d8f65511692305ddf9e485a48f164ae431d80a76ccd3f3a09c49fa6cb21ee network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-db-dcrxw 610d8f65511692305ddf9e485a48f164ae431d80a76ccd3f3a09c49fa6cb21ee network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:9c:c0:ca:51:0f\",\"name\":\"610d8f655116923\"},{\"mac\":\"0a:58:0a:82:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/78dac139-1089-4bec-baea-485a5f103d32\"}],\"ips\":[{\"address\":\"10.130.0.40/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:20:03.805178 3680 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29617400-k6z5k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:03.805724 3680 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29617400-k6z5k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:04.694128 3680 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29617400-k6z5k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:04.694152 3680 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29617400-k6z5k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:07.352383 3680 cni.go:452] [tsf-tpa/create-db-dcrxw 610d8f65511692305ddf9e485a48f164ae431d80a76ccd3f3a09c49fa6cb21ee network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-db-dcrxw 610d8f65511692305ddf9e485a48f164ae431d80a76ccd3f3a09c49fa6cb21ee network default NAD default NAD key ] I0424 15:20:07.432484 3680 cni.go:473] [tsf-tpa/create-db-dcrxw 610d8f65511692305ddf9e485a48f164ae431d80a76ccd3f3a09c49fa6cb21ee network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-db-dcrxw 610d8f65511692305ddf9e485a48f164ae431d80a76ccd3f3a09c49fa6cb21ee network default NAD default NAD key default], result "{}", err I0424 15:20:07.483823 3680 obj_retry.go:498] Detected object tsf-tpa/create-db-dcrxw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:07.483859 3680 pods.go:176] Deleting pod: tsf-tpa/create-db-dcrxw I0424 15:20:07.485248 3680 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-dcrxw, ips: 10.130.0.40 I0424 15:20:07.485277 3680 obj_retry.go:498] Detected object tsf-tpa/create-db-dcrxw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:08.509605 3680 obj_retry.go:492] Detected object tsf-tpa/create-db-dcrxw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:08.509633 3680 obj_retry.go:492] Detected object tsf-tpa/create-db-dcrxw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:12.856319 3680 obj_retry.go:498] Detected object tsf-tpa/migrate-db-dwff6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:12.856828 3680 obj_retry.go:498] Detected object tsf-tpa/migrate-db-dwff6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:13.882317 3680 obj_retry.go:492] Detected object tsf-tpa/migrate-db-dwff6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:13.882353 3680 obj_retry.go:492] Detected object tsf-tpa/migrate-db-dwff6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:16.015614 3680 obj_retry.go:498] Detected object tsf-tpa/create-importers-7vqnk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:16.016154 3680 obj_retry.go:498] Detected object tsf-tpa/create-importers-7vqnk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:17.043133 3680 obj_retry.go:492] Detected object tsf-tpa/create-importers-7vqnk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:17.043153 3680 obj_retry.go:492] Detected object tsf-tpa/create-importers-7vqnk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:18.410836 3680 cni.go:452] [tsf-tas/rekor-createtree-job-np2zx-lqt7s afd469d700e8f171b3610f13bb9e3d266f56ae764fd327a00797a5807c333bdb network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-np2zx-lqt7s afd469d700e8f171b3610f13bb9e3d266f56ae764fd327a00797a5807c333bdb network default NAD default NAD key ] I0424 15:20:18.503376 3680 cni.go:473] [tsf-tas/rekor-createtree-job-np2zx-lqt7s afd469d700e8f171b3610f13bb9e3d266f56ae764fd327a00797a5807c333bdb network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-np2zx-lqt7s afd469d700e8f171b3610f13bb9e3d266f56ae764fd327a00797a5807c333bdb network default NAD default NAD key default], result "{}", err I0424 15:20:18.555907 3680 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-np2zx-lqt7s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:18.555962 3680 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-np2zx-lqt7s I0424 15:20:18.557264 3680 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-np2zx-lqt7s, ips: 10.130.0.39 I0424 15:20:18.557292 3680 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-np2zx-lqt7s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:19.429050 3680 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-np2zx-lqt7s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:19.429075 3680 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-np2zx-lqt7s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:22.285293 3680 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-545c5dbbf9-5k57p] creating logical port tsf-tas_rekor-search-ui-545c5dbbf9-5k57p for pod on switch ip-10-0-1-253.ec2.internal I0424 15:20:22.285414 3680 kube.go:256] Updating pod tsf-tas/rekor-search-ui-545c5dbbf9-5k57p I0424 15:20:22.299370 3680 pod.go:62] [tsf-tas/rekor-search-ui-545c5dbbf9-5k57p] pod update took 13.975651ms I0424 15:20:22.299403 3680 base_network_controller_pods.go:951] [default/tsf-tas/rekor-search-ui-545c5dbbf9-5k57p] addLogicalPort annotation time took 14.012803ms I0424 15:20:22.300829 3680 pods.go:271] [tsf-tas/rekor-search-ui-545c5dbbf9-5k57p] addLogicalPort took 15.553509ms, libovsdb time 900.3µs I0424 15:20:22.642667 3680 cni.go:452] [tsf-tas/rekor-search-ui-545c5dbbf9-5k57p 30eadeed8c2a92a266169eff9cee2dd0760b2816b429a12cf110bbb8f037ddc6 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-search-ui-545c5dbbf9-5k57p 30eadeed8c2a92a266169eff9cee2dd0760b2816b429a12cf110bbb8f037ddc6 network default NAD default NAD key ] I0424 15:20:22.651874 3680 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-545c5dbbf9-5k57p, hostIfaceName: 30eadeed8c2a92a, network: default, NAD default, SandboxID: "30eadeed8c2a92a266169eff9cee2dd0760b2816b429a12cf110bbb8f037ddc6", PCI device ID: , UID: "af225ee7-f694-4623-a09c-c0d964aa79cf", MAC: 0a:58:0a:82:00:29, IPs: [10.130.0.41/23] I0424 15:20:22.705192 3680 cni.go:473] [tsf-tas/rekor-search-ui-545c5dbbf9-5k57p 30eadeed8c2a92a266169eff9cee2dd0760b2816b429a12cf110bbb8f037ddc6 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-search-ui-545c5dbbf9-5k57p 30eadeed8c2a92a266169eff9cee2dd0760b2816b429a12cf110bbb8f037ddc6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:05:e0:8c:9d:c4\",\"name\":\"30eadeed8c2a92a\"},{\"mac\":\"0a:58:0a:82:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0e8e6409-3c53-4689-8816-c15692906ebf\"}],\"ips\":[{\"address\":\"10.130.0.41/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:20:23.196647 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0424 15:20:24.807975 3680 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-6d4b54676b-b6tpg] creating logical port tsf-tas_rekor-server-6d4b54676b-b6tpg for pod on switch ip-10-0-1-253.ec2.internal I0424 15:20:24.808103 3680 kube.go:256] Updating pod tsf-tas/rekor-server-6d4b54676b-b6tpg I0424 15:20:24.821155 3680 pod.go:62] [tsf-tas/rekor-server-6d4b54676b-b6tpg] pod update took 13.076232ms I0424 15:20:24.821182 3680 base_network_controller_pods.go:951] [default/tsf-tas/rekor-server-6d4b54676b-b6tpg] addLogicalPort annotation time took 13.109396ms I0424 15:20:24.822472 3680 pods.go:271] [tsf-tas/rekor-server-6d4b54676b-b6tpg] addLogicalPort took 14.512674ms, libovsdb time 889.953µs W0424 15:20:25.445107 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0424 15:20:26.816196 3680 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0424 15:20:29.379349 3680 cni.go:452] [tsf-tas/rekor-server-6d4b54676b-b6tpg 8530823373fd740df32ff3f3e880fc2cb04c661fb957e34d0a5652b2713e0940 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-server-6d4b54676b-b6tpg 8530823373fd740df32ff3f3e880fc2cb04c661fb957e34d0a5652b2713e0940 network default NAD default NAD key ] I0424 15:20:29.388795 3680 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-6d4b54676b-b6tpg, hostIfaceName: 8530823373fd740, network: default, NAD default, SandboxID: "8530823373fd740df32ff3f3e880fc2cb04c661fb957e34d0a5652b2713e0940", PCI device ID: , UID: "fcf5fdff-a957-4e1d-ab63-70475a57266a", MAC: 0a:58:0a:82:00:2a, IPs: [10.130.0.42/23] I0424 15:20:29.441546 3680 cni.go:473] [tsf-tas/rekor-server-6d4b54676b-b6tpg 8530823373fd740df32ff3f3e880fc2cb04c661fb957e34d0a5652b2713e0940 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-server-6d4b54676b-b6tpg 8530823373fd740df32ff3f3e880fc2cb04c661fb957e34d0a5652b2713e0940 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:c5:a2:fd:88:ae\",\"name\":\"8530823373fd740\"},{\"mac\":\"0a:58:0a:82:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9fb5b2f6-45ba-4f1d-bd4b-0219d6510f60\"}],\"ips\":[{\"address\":\"10.130.0.42/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:20:30.619986 3680 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0424 15:20:31.481646 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0424 15:20:37.086636 3680 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=120 W0424 15:20:37.434039 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui W0424 15:20:42.491557 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0424 15:20:48.366842 3680 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-sc8jg-vv7th] creating logical port tsf-tas_tuf-repository-init-sc8jg-vv7th for pod on switch ip-10-0-1-253.ec2.internal I0424 15:20:48.366974 3680 kube.go:256] Updating pod tsf-tas/tuf-repository-init-sc8jg-vv7th I0424 15:20:48.379768 3680 pod.go:62] [tsf-tas/tuf-repository-init-sc8jg-vv7th] pod update took 12.814312ms I0424 15:20:48.379791 3680 base_network_controller_pods.go:951] [default/tsf-tas/tuf-repository-init-sc8jg-vv7th] addLogicalPort annotation time took 12.843532ms I0424 15:20:48.381262 3680 pods.go:271] [tsf-tas/tuf-repository-init-sc8jg-vv7th] addLogicalPort took 14.438191ms, libovsdb time 968.047µs I0424 15:20:56.133020 3680 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:56.133660 3680 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:20:56.155548 3680 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:56.155566 3680 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:20:56.163734 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0424 15:20:56.163758 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0424 15:20:57.031639 3680 namespace.go:98] [integration-service] adding namespace I0424 15:20:57.031642 3680 namespace.go:98] [default-tenant] adding namespace I0424 15:20:57.032692 3680 namespace.go:102] [integration-service] adding namespace took 1.032419ms I0424 15:20:57.033322 3680 namespace.go:102] [default-tenant] adding namespace took 1.544476ms I0424 15:20:57.036414 3680 namespace.go:98] [image-controller] adding namespace I0424 15:20:57.036491 3680 namespace.go:98] [build-service] adding namespace I0424 15:20:57.037145 3680 namespace.go:102] [image-controller] adding namespace took 718.478µs I0424 15:20:57.037688 3680 namespace.go:102] [build-service] adding namespace took 1.184124ms I0424 15:20:57.079492 3680 namespace.go:142] [integration-service] updating namespace I0424 15:20:57.106590 3680 namespace.go:142] [integration-service] updating namespace I0424 15:20:57.134715 3680 namespace.go:142] [default-tenant] updating namespace I0424 15:20:57.152954 3680 namespace.go:142] [image-controller] updating namespace I0424 15:20:57.176287 3680 namespace.go:142] [default-tenant] updating namespace I0424 15:20:57.185898 3680 namespace.go:142] [build-service] updating namespace I0424 15:20:57.192608 3680 namespace.go:142] [image-controller] updating namespace I0424 15:20:57.219993 3680 namespace.go:142] [build-service] updating namespace I0424 15:20:57.321069 3680 namespace.go:142] [build-service] updating namespace I0424 15:20:57.351115 3680 namespace.go:142] [integration-service] updating namespace I0424 15:20:57.407757 3680 namespace.go:98] [release-service] adding namespace I0424 15:20:57.408726 3680 namespace.go:102] [release-service] adding namespace took 948.717µs I0424 15:20:57.461863 3680 namespace.go:142] [konflux-ui] updating namespace I0424 15:20:57.461953 3680 namespace.go:142] [release-service] updating namespace I0424 15:20:57.516785 3680 namespace.go:142] [release-service] updating namespace I0424 15:20:57.591579 3680 namespace.go:98] [konflux-info] adding namespace I0424 15:20:57.592376 3680 namespace.go:102] [konflux-info] adding namespace took 779.02µs I0424 15:20:57.700686 3680 namespace.go:98] [namespace-lister] adding namespace I0424 15:20:57.701688 3680 namespace.go:102] [namespace-lister] adding namespace took 978.952µs I0424 15:20:57.744718 3680 namespace.go:142] [konflux-info] updating namespace I0424 15:20:57.905311 3680 namespace.go:142] [konflux-info] updating namespace I0424 15:20:57.981642 3680 namespace.go:98] [enterprise-contract-service] adding namespace I0424 15:20:57.982465 3680 namespace.go:102] [enterprise-contract-service] adding namespace took 799.989µs I0424 15:20:58.011680 3680 namespace.go:142] [namespace-lister] updating namespace I0424 15:20:58.058440 3680 namespace.go:142] [image-controller] updating namespace I0424 15:20:58.168718 3680 namespace.go:142] [namespace-lister] updating namespace I0424 15:20:58.208752 3680 namespace.go:142] [enterprise-contract-service] updating namespace I0424 15:20:58.326865 3680 namespace.go:142] [enterprise-contract-service] updating namespace I0424 15:20:58.327621 3680 cni.go:452] [tsf-tas/tuf-repository-init-sc8jg-vv7th 4ad7b2da8f2ad2a920d77404dd6019c0d8aa927f927223a3cf9cf9dfc82ac33d network default NAD default NAD key ] ADD starting CNI request [tsf-tas/tuf-repository-init-sc8jg-vv7th 4ad7b2da8f2ad2a920d77404dd6019c0d8aa927f927223a3cf9cf9dfc82ac33d network default NAD default NAD key ] I0424 15:20:58.336858 3680 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-sc8jg-vv7th, hostIfaceName: 4ad7b2da8f2ad2a, network: default, NAD default, SandboxID: "4ad7b2da8f2ad2a920d77404dd6019c0d8aa927f927223a3cf9cf9dfc82ac33d", PCI device ID: , UID: "89db95f2-5587-47a9-8d7b-3b2e11130761", MAC: 0a:58:0a:82:00:2b, IPs: [10.130.0.43/23] I0424 15:20:58.388611 3680 cni.go:473] [tsf-tas/tuf-repository-init-sc8jg-vv7th 4ad7b2da8f2ad2a920d77404dd6019c0d8aa927f927223a3cf9cf9dfc82ac33d network default NAD default NAD key default] ADD finished CNI request [tsf-tas/tuf-repository-init-sc8jg-vv7th 4ad7b2da8f2ad2a920d77404dd6019c0d8aa927f927223a3cf9cf9dfc82ac33d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:3c:57:4b:93:a4\",\"name\":\"4ad7b2da8f2ad2a\"},{\"mac\":\"0a:58:0a:82:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/46009a38-2ae3-4f79-b62c-db8eaa6bc631\"}],\"ips\":[{\"address\":\"10.130.0.43/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:20:58.501796 3680 namespace.go:142] [default-tenant] updating namespace I0424 15:20:58.716458 3680 namespace.go:98] [segment-bridge] adding namespace I0424 15:20:58.717348 3680 namespace.go:102] [segment-bridge] adding namespace took 870.315µs I0424 15:20:58.781682 3680 namespace.go:142] [segment-bridge] updating namespace I0424 15:20:58.826733 3680 namespace.go:142] [segment-bridge] updating namespace I0424 15:20:58.849282 3680 namespace.go:98] [konflux-cli] adding namespace I0424 15:20:58.850170 3680 namespace.go:102] [konflux-cli] adding namespace took 867.314µs I0424 15:20:58.925129 3680 namespace.go:142] [konflux-cli] updating namespace I0424 15:20:58.928799 3680 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0424 15:20:58.929429 3680 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0424 15:20:58.929448 3680 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0424 15:20:58.932404 3680 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0424 15:20:58.951766 3680 namespace.go:142] [konflux-cli] updating namespace I0424 15:20:58.985484 3680 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0424 15:20:58.985506 3680 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0424 15:20:58.986256 3680 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0424 15:20:59.267815 3680 base_network_controller_pods.go:487] [default/konflux-ui/dex-b8f5cf58c-wf7g6] creating logical port konflux-ui_dex-b8f5cf58c-wf7g6 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:20:59.267908 3680 kube.go:256] Updating pod konflux-ui/dex-b8f5cf58c-wf7g6 I0424 15:20:59.292132 3680 pod.go:62] [konflux-ui/dex-b8f5cf58c-wf7g6] pod update took 24.249442ms I0424 15:20:59.292155 3680 base_network_controller_pods.go:951] [default/konflux-ui/dex-b8f5cf58c-wf7g6] addLogicalPort annotation time took 24.278458ms I0424 15:20:59.293471 3680 pods.go:271] [konflux-ui/dex-b8f5cf58c-wf7g6] addLogicalPort took 25.672501ms, libovsdb time 823.01µs I0424 15:20:59.312696 3680 namespace.go:142] [build-service] updating namespace I0424 15:20:59.361997 3680 base_network_controller_pods.go:487] [default/konflux-ui/proxy-56f7d9c9d6-2wcn8] creating logical port konflux-ui_proxy-56f7d9c9d6-2wcn8 for pod on switch ip-10-0-1-253.ec2.internal I0424 15:20:59.362137 3680 kube.go:256] Updating pod konflux-ui/proxy-56f7d9c9d6-2wcn8 I0424 15:20:59.407161 3680 pod.go:62] [konflux-ui/proxy-56f7d9c9d6-2wcn8] pod update took 45.067798ms I0424 15:20:59.407182 3680 base_network_controller_pods.go:951] [default/konflux-ui/proxy-56f7d9c9d6-2wcn8] addLogicalPort annotation time took 45.094369ms I0424 15:20:59.408454 3680 pods.go:271] [konflux-ui/proxy-56f7d9c9d6-2wcn8] addLogicalPort took 46.478392ms, libovsdb time 872.37µs I0424 15:20:59.715149 3680 namespace.go:142] [default-tenant] updating namespace I0424 15:21:00.043165 3680 namespace.go:142] [enterprise-contract-service] updating namespace I0424 15:21:00.322015 3680 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7cbf488fcf-hfjbr] creating logical port build-service_build-service-controller-manager-7cbf488fcf-hfjbr for pod on switch ip-10-0-1-253.ec2.internal I0424 15:21:00.322114 3680 kube.go:256] Updating pod build-service/build-service-controller-manager-7cbf488fcf-hfjbr I0424 15:21:00.359910 3680 pod.go:62] [build-service/build-service-controller-manager-7cbf488fcf-hfjbr] pod update took 37.819827ms I0424 15:21:00.359957 3680 base_network_controller_pods.go:951] [default/build-service/build-service-controller-manager-7cbf488fcf-hfjbr] addLogicalPort annotation time took 37.871551ms I0424 15:21:00.361255 3680 pods.go:271] [build-service/build-service-controller-manager-7cbf488fcf-hfjbr] addLogicalPort took 39.254865ms, libovsdb time 878.637µs I0424 15:21:00.695950 3680 cni.go:452] [build-service/build-service-controller-manager-7cbf488fcf-hfjbr 01db0dd8fc8ffa46e4794bcef2d88b5a04dede5947636626159218e529c7426b network default NAD default NAD key ] ADD starting CNI request [build-service/build-service-controller-manager-7cbf488fcf-hfjbr 01db0dd8fc8ffa46e4794bcef2d88b5a04dede5947636626159218e529c7426b network default NAD default NAD key ] I0424 15:21:00.708057 3680 helper_linux.go:556] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7cbf488fcf-hfjbr, hostIfaceName: 01db0dd8fc8ffa4, network: default, NAD default, SandboxID: "01db0dd8fc8ffa46e4794bcef2d88b5a04dede5947636626159218e529c7426b", PCI device ID: , UID: "0620a51b-6a9d-497f-bc00-606fc213acef", MAC: 0a:58:0a:82:00:2e, IPs: [10.130.0.46/23] I0424 15:21:00.767305 3680 cni.go:473] [build-service/build-service-controller-manager-7cbf488fcf-hfjbr 01db0dd8fc8ffa46e4794bcef2d88b5a04dede5947636626159218e529c7426b network default NAD default NAD key default] ADD finished CNI request [build-service/build-service-controller-manager-7cbf488fcf-hfjbr 01db0dd8fc8ffa46e4794bcef2d88b5a04dede5947636626159218e529c7426b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:da:bd:98:e2:03\",\"name\":\"01db0dd8fc8ffa4\"},{\"mac\":\"0a:58:0a:82:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f1aeb349-8d3d-4c90-94c2-25c6f0c66265\"}],\"ips\":[{\"address\":\"10.130.0.46/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:21:00.803038 3680 namespace.go:142] [image-controller] updating namespace I0424 15:21:01.467671 3680 namespace.go:142] [integration-service] updating namespace I0424 15:21:01.610004 3680 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-6bfdb6bcb6-2b6lp] creating logical port release-service_release-service-controller-manager-6bfdb6bcb6-2b6lp for pod on switch ip-10-0-1-253.ec2.internal I0424 15:21:01.610121 3680 kube.go:256] Updating pod release-service/release-service-controller-manager-6bfdb6bcb6-2b6lp I0424 15:21:01.630088 3680 pod.go:62] [release-service/release-service-controller-manager-6bfdb6bcb6-2b6lp] pod update took 19.998509ms I0424 15:21:01.630122 3680 base_network_controller_pods.go:951] [default/release-service/release-service-controller-manager-6bfdb6bcb6-2b6lp] addLogicalPort annotation time took 20.037916ms I0424 15:21:01.631744 3680 pods.go:271] [release-service/release-service-controller-manager-6bfdb6bcb6-2b6lp] addLogicalPort took 21.763936ms, libovsdb time 1.070788ms I0424 15:21:01.954805 3680 namespace.go:142] [konflux-info] updating namespace I0424 15:21:02.298807 3680 namespace.go:142] [namespace-lister] updating namespace I0424 15:21:02.472015 3680 namespace.go:142] [release-service] updating namespace I0424 15:21:02.594696 3680 namespace.go:142] [build-service] updating namespace I0424 15:21:02.614022 3680 namespace.go:142] [default-tenant] updating namespace I0424 15:21:02.636722 3680 namespace.go:142] [enterprise-contract-service] updating namespace I0424 15:21:02.650670 3680 namespace.go:142] [image-controller] updating namespace I0424 15:21:02.673896 3680 namespace.go:142] [integration-service] updating namespace I0424 15:21:02.703670 3680 namespace.go:142] [konflux-info] updating namespace I0424 15:21:02.729260 3680 namespace.go:142] [namespace-lister] updating namespace I0424 15:21:02.741912 3680 namespace.go:142] [release-service] updating namespace I0424 15:21:02.809650 3680 namespace.go:142] [build-service] updating namespace I0424 15:21:02.883367 3680 namespace.go:142] [default-tenant] updating namespace I0424 15:21:02.967350 3680 namespace.go:142] [enterprise-contract-service] updating namespace I0424 15:21:03.044396 3680 namespace.go:142] [image-controller] updating namespace W0424 15:21:03.068417 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0424 15:21:03.107041 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0424 15:21:03.126291 3680 namespace.go:142] [integration-service] updating namespace I0424 15:21:03.195244 3680 namespace.go:142] [konflux-info] updating namespace I0424 15:21:03.299432 3680 namespace.go:142] [namespace-lister] updating namespace I0424 15:21:03.393993 3680 namespace.go:142] [release-service] updating namespace I0424 15:21:04.663915 3680 cni.go:452] [tsf-tas/tuf-repository-init-sc8jg-vv7th 4ad7b2da8f2ad2a920d77404dd6019c0d8aa927f927223a3cf9cf9dfc82ac33d network default NAD default NAD key ] DEL starting CNI request [tsf-tas/tuf-repository-init-sc8jg-vv7th 4ad7b2da8f2ad2a920d77404dd6019c0d8aa927f927223a3cf9cf9dfc82ac33d network default NAD default NAD key ] I0424 15:21:04.668166 3680 namespace.go:142] [konflux-cli] updating namespace I0424 15:21:04.747174 3680 cni.go:473] [tsf-tas/tuf-repository-init-sc8jg-vv7th 4ad7b2da8f2ad2a920d77404dd6019c0d8aa927f927223a3cf9cf9dfc82ac33d network default NAD default NAD key default] DEL finished CNI request [tsf-tas/tuf-repository-init-sc8jg-vv7th 4ad7b2da8f2ad2a920d77404dd6019c0d8aa927f927223a3cf9cf9dfc82ac33d network default NAD default NAD key default], result "{}", err I0424 15:21:04.816415 3680 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-sc8jg-vv7th of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:21:04.816449 3680 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-sc8jg-vv7th I0424 15:21:04.818051 3680 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-sc8jg-vv7th, ips: 10.130.0.43 I0424 15:21:04.818085 3680 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-sc8jg-vv7th of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:21:05.003447 3680 namespace.go:142] [segment-bridge] updating namespace I0424 15:21:05.133602 3680 namespace.go:142] [konflux-cli] updating namespace I0424 15:21:05.152757 3680 namespace.go:142] [segment-bridge] updating namespace I0424 15:21:05.254752 3680 namespace.go:142] [konflux-cli] updating namespace I0424 15:21:05.355410 3680 namespace.go:142] [segment-bridge] updating namespace W0424 15:21:05.666511 3680 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0424 15:21:05.841982 3680 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-sc8jg-vv7th of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:21:05.842010 3680 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-sc8jg-vv7th of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:21:07.133036 3680 cni.go:452] [konflux-ui/dex-b8f5cf58c-wf7g6 689b78d586b301296947a9a7f8e0a51181305ac70b4cf13c1aeff8409991101e network default NAD default NAD key ] ADD starting CNI request [konflux-ui/dex-b8f5cf58c-wf7g6 689b78d586b301296947a9a7f8e0a51181305ac70b4cf13c1aeff8409991101e network default NAD default NAD key ] I0424 15:21:07.142445 3680 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: dex-b8f5cf58c-wf7g6, hostIfaceName: 689b78d586b3012, network: default, NAD default, SandboxID: "689b78d586b301296947a9a7f8e0a51181305ac70b4cf13c1aeff8409991101e", PCI device ID: , UID: "5ebb4619-f8ad-42e5-8cf7-7cd2a31734b8", MAC: 0a:58:0a:82:00:2c, IPs: [10.130.0.44/23] I0424 15:21:07.194989 3680 cni.go:473] [konflux-ui/dex-b8f5cf58c-wf7g6 689b78d586b301296947a9a7f8e0a51181305ac70b4cf13c1aeff8409991101e network default NAD default NAD key default] ADD finished CNI request [konflux-ui/dex-b8f5cf58c-wf7g6 689b78d586b301296947a9a7f8e0a51181305ac70b4cf13c1aeff8409991101e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:a0:09:3d:ea:d6\",\"name\":\"689b78d586b3012\"},{\"mac\":\"0a:58:0a:82:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d9a61230-4fb8-4e20-8de6-074a5726b1ae\"}],\"ips\":[{\"address\":\"10.130.0.44/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:21:07.231393 3680 cni.go:452] [konflux-ui/proxy-56f7d9c9d6-2wcn8 2f10e5abcc792b49e8b8a4a93b0db78f59b58919d5bca056a00437c90a0d291e network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-56f7d9c9d6-2wcn8 2f10e5abcc792b49e8b8a4a93b0db78f59b58919d5bca056a00437c90a0d291e network default NAD default NAD key ] I0424 15:21:07.240873 3680 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-56f7d9c9d6-2wcn8, hostIfaceName: 2f10e5abcc792b4, network: default, NAD default, SandboxID: "2f10e5abcc792b49e8b8a4a93b0db78f59b58919d5bca056a00437c90a0d291e", PCI device ID: , UID: "8b849157-cbfc-4b2a-a677-46ff6b0f0b2f", MAC: 0a:58:0a:82:00:2d, IPs: [10.130.0.45/23] I0424 15:21:07.292402 3680 cni.go:473] [konflux-ui/proxy-56f7d9c9d6-2wcn8 2f10e5abcc792b49e8b8a4a93b0db78f59b58919d5bca056a00437c90a0d291e network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-56f7d9c9d6-2wcn8 2f10e5abcc792b49e8b8a4a93b0db78f59b58919d5bca056a00437c90a0d291e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:4d:05:24:5f:8a\",\"name\":\"2f10e5abcc792b4\"},{\"mac\":\"0a:58:0a:82:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b24d633e-fe20-4472-b796-90c1d9c3ae90\"}],\"ips\":[{\"address\":\"10.130.0.45/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:21:08.081814 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0424 15:21:08.081857 3680 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0424 15:21:08.112983 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0424 15:21:08.113345 3680 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0424 15:21:09.692304 3680 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0424 15:21:10.561983 3680 base_network_controller_pods.go:487] [default/tsf-tas/tuf-97fc8ff76-gjwdk] creating logical port tsf-tas_tuf-97fc8ff76-gjwdk for pod on switch ip-10-0-1-253.ec2.internal I0424 15:21:10.562144 3680 kube.go:256] Updating pod tsf-tas/tuf-97fc8ff76-gjwdk I0424 15:21:10.577666 3680 pod.go:62] [tsf-tas/tuf-97fc8ff76-gjwdk] pod update took 15.559507ms I0424 15:21:10.577699 3680 base_network_controller_pods.go:951] [default/tsf-tas/tuf-97fc8ff76-gjwdk] addLogicalPort annotation time took 15.596541ms I0424 15:21:10.579216 3680 pods.go:271] [tsf-tas/tuf-97fc8ff76-gjwdk] addLogicalPort took 17.254022ms, libovsdb time 940.87µs I0424 15:21:11.213123 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0424 15:21:11.978892 3680 cni.go:452] [release-service/release-service-controller-manager-6bfdb6bcb6-2b6lp 27062b99d17458aa6022c844dc9dd32dc83740ce625ea643646f285c6e3d9896 network default NAD default NAD key ] ADD starting CNI request [release-service/release-service-controller-manager-6bfdb6bcb6-2b6lp 27062b99d17458aa6022c844dc9dd32dc83740ce625ea643646f285c6e3d9896 network default NAD default NAD key ] I0424 15:21:11.989488 3680 helper_linux.go:556] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-6bfdb6bcb6-2b6lp, hostIfaceName: 27062b99d17458a, network: default, NAD default, SandboxID: "27062b99d17458aa6022c844dc9dd32dc83740ce625ea643646f285c6e3d9896", PCI device ID: , UID: "dcc9594c-4fac-4257-be1a-5add4de751d9", MAC: 0a:58:0a:82:00:2f, IPs: [10.130.0.47/23] I0424 15:21:12.043030 3680 cni.go:473] [release-service/release-service-controller-manager-6bfdb6bcb6-2b6lp 27062b99d17458aa6022c844dc9dd32dc83740ce625ea643646f285c6e3d9896 network default NAD default NAD key default] ADD finished CNI request [release-service/release-service-controller-manager-6bfdb6bcb6-2b6lp 27062b99d17458aa6022c844dc9dd32dc83740ce625ea643646f285c6e3d9896 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:52:f7:8f:99:d8\",\"name\":\"27062b99d17458a\"},{\"mac\":\"0a:58:0a:82:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f5b0eb40-a785-4557-8187-81bf68b76d1d\"}],\"ips\":[{\"address\":\"10.130.0.47/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0424 15:21:12.718893 3680 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0424 15:21:12.718913 3680 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0424 15:21:13.784367 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0424 15:21:13.786905 3680 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0424 15:21:14.140126 3680 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0424 15:21:14.140147 3680 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0424 15:21:16.680365 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0424 15:21:18.911483 3680 cni.go:452] [tsf-tas/tuf-97fc8ff76-gjwdk fb3e2793a62491945e6e277c91ccee9561e4ec95b11dfa8fde9b35f3cabc840d network default NAD default NAD key ] ADD starting CNI request [tsf-tas/tuf-97fc8ff76-gjwdk fb3e2793a62491945e6e277c91ccee9561e4ec95b11dfa8fde9b35f3cabc840d network default NAD default NAD key ] I0424 15:21:18.926564 3680 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: tuf-97fc8ff76-gjwdk, hostIfaceName: fb3e2793a624919, network: default, NAD default, SandboxID: "fb3e2793a62491945e6e277c91ccee9561e4ec95b11dfa8fde9b35f3cabc840d", PCI device ID: , UID: "fb2a936d-4a7c-4022-9f74-194a9b12d3e9", MAC: 0a:58:0a:82:00:30, IPs: [10.130.0.48/23] I0424 15:21:18.991674 3680 cni.go:473] [tsf-tas/tuf-97fc8ff76-gjwdk fb3e2793a62491945e6e277c91ccee9561e4ec95b11dfa8fde9b35f3cabc840d network default NAD default NAD key default] ADD finished CNI request [tsf-tas/tuf-97fc8ff76-gjwdk fb3e2793a62491945e6e277c91ccee9561e4ec95b11dfa8fde9b35f3cabc840d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:3e:d5:cd:51:4a\",\"name\":\"fb3e2793a624919\"},{\"mac\":\"0a:58:0a:82:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ccf97db4-f071-446e-bd08-5787cbe4f4ff\"}],\"ips\":[{\"address\":\"10.130.0.48/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0424 15:21:19.442478 3680 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-6p9vf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:21:19.443072 3680 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-6p9vf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:21:20.157273 3680 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-6p9vf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:21:20.157302 3680 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-6p9vf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:21:20.162282 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-6p9vf I0424 15:21:20.162312 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-6p9vf W0424 15:21:23.733840 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex W0424 15:21:24.737528 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0424 15:21:24.744272 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service W0424 15:21:24.802705 3680 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0424 15:21:35.823056 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf I0424 15:21:36.185775 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 W0424 15:21:50.184054 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0424 15:21:50.184075 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0424 15:21:50.184087 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0424 15:21:50.184092 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0424 15:21:51.798817 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0424 15:21:51.798836 3680 util.go:959] Failed to build node endpoints for node ip-10-0-1-253.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy I0424 15:22:19.089163 3680 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=87 I0424 15:22:20.203823 3680 cni.go:452] [konflux-ui/proxy-56f7d9c9d6-2wcn8 2f10e5abcc792b49e8b8a4a93b0db78f59b58919d5bca056a00437c90a0d291e network default NAD default NAD key ] DEL starting CNI request [konflux-ui/proxy-56f7d9c9d6-2wcn8 2f10e5abcc792b49e8b8a4a93b0db78f59b58919d5bca056a00437c90a0d291e network default NAD default NAD key ] I0424 15:22:20.298600 3680 cni.go:473] [konflux-ui/proxy-56f7d9c9d6-2wcn8 2f10e5abcc792b49e8b8a4a93b0db78f59b58919d5bca056a00437c90a0d291e network default NAD default NAD key default] DEL finished CNI request [konflux-ui/proxy-56f7d9c9d6-2wcn8 2f10e5abcc792b49e8b8a4a93b0db78f59b58919d5bca056a00437c90a0d291e network default NAD default NAD key default], result "{}", err I0424 15:22:20.365188 3680 obj_retry.go:498] Detected object konflux-ui/proxy-56f7d9c9d6-2wcn8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:22:20.365234 3680 pods.go:176] Deleting pod: konflux-ui/proxy-56f7d9c9d6-2wcn8 I0424 15:22:20.366654 3680 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-56f7d9c9d6-2wcn8, ips: 10.130.0.45 I0424 15:22:20.366676 3680 obj_retry.go:498] Detected object konflux-ui/proxy-56f7d9c9d6-2wcn8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:22:21.116760 3680 obj_retry.go:492] Detected object konflux-ui/proxy-56f7d9c9d6-2wcn8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:22:21.116801 3680 obj_retry.go:492] Detected object konflux-ui/proxy-56f7d9c9d6-2wcn8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:22:21.124436 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-56f7d9c9d6-2wcn8 I0424 15:22:21.124462 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-56f7d9c9d6-2wcn8 I0424 15:22:27.686607 3680 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 15:22:27.687225 3680 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 15:22:27.728249 3680 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:22:27.728267 3680 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 15:22:27.736117 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0424 15:22:27.736140 3680 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0424 15:22:32.175547 3680 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0424 15:22:37.180588 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0424 15:22:37.213959 3680 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0424 15:22:52.330039 3680 namespace.go:98] [default-managed-tenant-j5lz8] adding namespace I0424 15:22:52.331496 3680 namespace.go:102] [default-managed-tenant-j5lz8] adding namespace took 1.368454ms I0424 15:22:52.355707 3680 namespace.go:142] [default-managed-tenant-j5lz8] updating namespace I0424 15:22:52.372238 3680 namespace.go:142] [default-managed-tenant-j5lz8] updating namespace I0424 15:22:53.387139 3680 namespace.go:142] [default-managed-tenant-j5lz8] updating namespace I0424 15:22:53.515876 3680 namespace.go:142] [default-managed-tenant-j5lz8] updating namespace I0424 15:22:53.617206 3680 namespace.go:142] [default-managed-tenant-j5lz8] updating namespace