+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-108.ec2.internal ++ [[ -n ip-10-0-1-108.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-108.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0423 19:30:36.744412904 - disable conntrack on geneve port' I0423 19:30:36.744412904 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0423 19:30:36.763243783 - starting ovnkube-node' I0423 19:30:36.763243783 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-108.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-108.ec2.internal --init-node ip-10-0-1-108.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-108.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0423 19:30:36.805733 3663 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0423 19:30:36.805791 3663 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-350f7e2753.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0423 19:30:36.807214 3663 kube.go:431] Waiting for certificate I0423 19:30:36.807268 3663 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0423 19:30:36.807317 3663 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0423 19:30:36.808122 3663 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0423 19:30:36.835743 3663 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 19:30:36.835763 3663 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 19:30:36.838751 3663 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 19:30:36.842818 3663 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-rbk9l" I0423 19:30:36.852248 3663 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-rbk9l" I0423 19:30:36.852340 3663 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 19:30:37.808086 3663 kube.go:438] Certificate found I0423 19:30:37.808678 3663 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0423 19:30:37.809986 3663 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0423 19:30:37.866123 3663 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-24 19:25:36 +0000 UTC" deadline="2026-04-24 15:43:20.501026688 +0000 UTC" I0423 19:30:37.866163 3663 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h12m42.63487442s" I0423 19:30:37.886596 3663 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0423 19:30:37.895642 3663 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0423 19:30:37.895703 3663 server.go:101] MetricServer registers OVS metrics I0423 19:30:37.896051 3663 server.go:105] MetricServer registers OVN DB metrics I0423 19:30:37.897016 3663 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0423 19:30:37.897067 3663 factory.go:561] Starting watch factory I0423 19:30:37.897155 3663 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.897160 3663 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.897170 3663 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.897176 3663 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.897184 3663 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.897196 3663 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.897158 3663 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.897764 3663 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.897969 3663 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.898005 3663 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.897160 3663 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.898247 3663 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.900581 3663 controller_manager.go:370] Starting the ovnkube controller I0423 19:30:37.900591 3663 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-108.ec2.internal I0423 19:30:37.900641 3663 controller_manager.go:395] NBDB zone sync took: 43.261µs I0423 19:30:37.900650 3663 factory.go:561] Starting watch factory I0423 19:30:37.901223 3663 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0423 19:30:37.901250 3663 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0423 19:30:37.915744 3663 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0423 19:30:37.915766 3663 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0423 19:30:37.918972 3663 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.919119 3663 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.919656 3663 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.920593 3663 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0423 19:30:37.920629 3663 server.go:109] MetricServer registers OVN Controller metrics I0423 19:30:37.921201 3663 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.922172 3663 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:37.926579 3663 server.go:113] MetricServer registers OVN Northd metrics I0423 19:30:37.931395 3663 metrics.go:519] OVN Metrics Server starts to run ... I0423 19:30:37.933011 3663 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:30:38.002722 3663 factory.go:1977] *v1.Pod informer cache synced successfully I0423 19:30:38.002747 3663 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0423 19:30:38.002753 3663 factory.go:1977] *v1.Namespace informer cache synced successfully I0423 19:30:38.002756 3663 factory.go:1977] *v1.Node informer cache synced successfully I0423 19:30:38.002760 3663 factory.go:1977] *v1.Service informer cache synced successfully I0423 19:30:38.002764 3663 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0423 19:30:38.002858 3663 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:30:38.002880 3663 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:30:38.002941 3663 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:30:38.002958 3663 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:30:38.005900 3663 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0423 19:30:38.005920 3663 factory.go:1977] *v1.Namespace informer cache synced successfully I0423 19:30:38.005924 3663 factory.go:1977] *v1.Node informer cache synced successfully I0423 19:30:38.005927 3663 factory.go:1977] *v1.Service informer cache synced successfully I0423 19:30:38.005930 3663 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0423 19:30:38.005933 3663 factory.go:1977] *v1.Pod informer cache synced successfully I0423 19:30:38.006894 3663 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:30:38.006905 3663 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:30:38.008689 3663 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0423 19:30:38.008702 3663 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0423 19:30:38.008766 3663 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.008780 3663 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.011460 3663 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0423 19:30:38.011471 3663 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0423 19:30:38.012242 3663 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.014247 3663 factory.go:1977] *v1.EgressIP informer cache synced successfully I0423 19:30:38.014310 3663 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.014322 3663 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.016969 3663 factory.go:1977] *v1.EgressIP informer cache synced successfully I0423 19:30:38.017992 3663 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.019601 3663 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0423 19:30:38.019654 3663 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.019665 3663 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.022096 3663 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0423 19:30:38.022956 3663 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.024967 3663 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0423 19:30:38.025023 3663 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.025034 3663 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.028107 3663 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0423 19:30:38.028306 3663 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.030304 3663 factory.go:1977] *v1.EgressService informer cache synced successfully I0423 19:30:38.030361 3663 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.030375 3663 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.033471 3663 factory.go:1977] *v1.EgressService informer cache synced successfully I0423 19:30:38.033720 3663 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.035691 3663 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0423 19:30:38.035848 3663 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 19:30:38.035865 3663 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 19:30:38.038890 3663 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0423 19:30:38.039242 3663 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 19:30:38.041230 3663 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0423 19:30:38.041291 3663 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.041301 3663 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.041347 3663 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.041361 3663 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.044368 3663 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0423 19:30:38.044710 3663 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.044743 3663 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:30:38.046735 3663 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0423 19:30:38.046745 3663 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0423 19:30:38.046753 3663 factory.go:680] Watch Factory start up complete, took: 149.687419ms I0423 19:30:38.046817 3663 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0423 19:30:38.049436 3663 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0423 19:30:38.049446 3663 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0423 19:30:38.049453 3663 factory.go:680] Watch Factory start up complete, took: 148.801678ms I0423 19:30:38.049462 3663 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-1-108.ec2.internal" zone I0423 19:30:38.088220 3663 default_node_network_controller.go:710] Initializing the default node network controller I0423 19:30:38.167938 3663 udn_isolation.go:101] Starting UDN host isolation manager I0423 19:30:38.174124 3663 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0423 19:30:38.228525 3663 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0423 19:30:38.228631 3663 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0423 19:30:38.228657 3663 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0423 19:30:38.264143 3663 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0423 19:30:38.264205 3663 default_node_network_controller.go:809] Node ip-10-0-1-108.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0423 19:30:38.269948 3663 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.108"] k8s.ovn.org/zone-name:ip-10-0-1-108.ec2.internal] on node ip-10-0-1-108.ec2.internal I0423 19:30:38.291464 3663 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0423 19:30:38.297866 3663 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0423 19:30:38.297973 3663 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0423 19:30:38.298116 3663 gateway_init.go:263] Preparing Gateway I0423 19:30:38.298129 3663 gateway_shared_intf.go:1687] Creating new gateway I0423 19:30:38.300410 3663 controller_manager.go:429] Waiting for node in zone sync took: 250.926079ms I0423 19:30:38.307472 3663 controller_manager.go:611] Switching to transit router for layer2 networks I0423 19:30:38.307493 3663 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-108.ec2.internal I0423 19:30:38.324452 3663 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0423 19:30:38.331060 3663 controller_manager.go:290] SCTP support detected in OVN I0423 19:30:38.331699 3663 services_controller.go:69] Creating services controller for network=default I0423 19:30:38.331842 3663 udn_enabled_svc.go:80] Waiting for service informer to sync I0423 19:30:38.331949 3663 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0423 19:30:38.332021 3663 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0423 19:30:38.332046 3663 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0423 19:30:38.332095 3663 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0423 19:30:38.332283 3663 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0423 19:30:38.332322 3663 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0423 19:30:38.332378 3663 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 62.236µs I0423 19:30:38.332404 3663 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 73.378µs I0423 19:30:38.332425 3663 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 89.14µs I0423 19:30:38.332437 3663 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0423 19:30:38.332444 3663 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0423 19:30:38.332453 3663 default_network_controller.go:325] Starting the default network controller I0423 19:30:38.332426 3663 udn_enabled_svc.go:95] Performing full resync I0423 19:30:38.332582 3663 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 12.894µs I0423 19:30:38.332904 3663 udn_enabled_svc.go:99] Waiting for handler to sync I0423 19:30:38.332917 3663 udn_enabled_svc.go:104] Starting worker I0423 19:30:38.341728 3663 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0423 19:30:38.341782 3663 repair.go:33] Syncing exgw routes took 41.719µs I0423 19:30:38.341807 3663 default_network_controller.go:414] Starting all the Watchers... I0423 19:30:38.341961 3663 namespace.go:98] [openshift-apiserver-operator] adding namespace I0423 19:30:38.341963 3663 namespace.go:98] [openshift-config-managed] adding namespace I0423 19:30:38.341991 3663 namespace.go:98] [openshift-network-diagnostics] adding namespace I0423 19:30:38.342003 3663 namespace.go:98] [openshift-dns] adding namespace I0423 19:30:38.342006 3663 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0423 19:30:38.342009 3663 namespace.go:98] [openshift-backplane-cse] adding namespace I0423 19:30:38.342016 3663 namespace.go:98] [openshift-console-user-settings] adding namespace I0423 19:30:38.342019 3663 namespace.go:98] [openshift-must-gather-operator] adding namespace I0423 19:30:38.342024 3663 namespace.go:98] [openshift-infra] adding namespace I0423 19:30:38.342029 3663 namespace.go:98] [kube-system] adding namespace I0423 19:30:38.342032 3663 namespace.go:98] [openshift-host-network] adding namespace I0423 19:30:38.342034 3663 namespace.go:98] [open-cluster-management-2ps23krgrug0inkj9hc8sh6m5b2n43rp] adding namespace I0423 19:30:38.342045 3663 namespace.go:98] [openshift-network-console] adding namespace I0423 19:30:38.342049 3663 namespace.go:98] [openshift-console-operator] adding namespace I0423 19:30:38.342098 3663 namespace.go:98] [openshift-etcd] adding namespace I0423 19:30:38.342550 3663 namespace.go:102] [openshift-apiserver-operator] adding namespace took 577.534µs I0423 19:30:38.342563 3663 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0423 19:30:38.342979 3663 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 411.212µs I0423 19:30:38.342992 3663 namespace.go:98] [openshift-backplane-csm] adding namespace I0423 19:30:38.343487 3663 namespace.go:102] [openshift-backplane-csm] adding namespace took 489.14µs I0423 19:30:38.343503 3663 namespace.go:98] [openshift-operators-redhat] adding namespace W0423 19:30:38.343903 3663 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-2m7fm: pod openshift-network-diagnostics/network-check-target-2m7fm: no pod IPs found W0423 19:30:38.343927 3663 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-m89h2: pod openshift-network-diagnostics/network-check-target-m89h2: no pod IPs found W0423 19:30:38.343935 3663 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-nwqv2: pod openshift-network-diagnostics/network-check-target-nwqv2: no pod IPs found I0423 19:30:38.343934 3663 namespace.go:102] [openshift-config-managed] adding namespace took 1.957172ms I0423 19:30:38.343947 3663 namespace.go:98] [openshift] adding namespace I0423 19:30:38.344467 3663 namespace.go:102] [openshift-network-diagnostics] adding namespace took 2.467236ms I0423 19:30:38.344478 3663 namespace.go:98] [openshift-kube-apiserver] adding namespace I0423 19:30:38.344923 3663 namespace.go:102] [openshift-dns] adding namespace took 2.913276ms I0423 19:30:38.344939 3663 namespace.go:98] [openshift-backplane-mobb] adding namespace I0423 19:30:38.345783 3663 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 3.767611ms I0423 19:30:38.345793 3663 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0423 19:30:38.346249 3663 namespace.go:102] [openshift-console-user-settings] adding namespace took 4.226861ms I0423 19:30:38.346264 3663 namespace.go:98] [openshift-logging] adding namespace I0423 19:30:38.346724 3663 namespace.go:102] [openshift-backplane-cse] adding namespace took 4.705571ms I0423 19:30:38.346734 3663 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0423 19:30:38.347131 3663 namespace.go:102] [openshift-must-gather-operator] adding namespace took 5.100701ms I0423 19:30:38.347153 3663 namespace.go:98] [openshift-backplane-srep] adding namespace I0423 19:30:38.347636 3663 namespace.go:102] [openshift-infra] adding namespace took 5.601373ms I0423 19:30:38.347656 3663 namespace.go:98] [openshift-controller-manager] adding namespace I0423 19:30:38.348125 3663 namespace.go:102] [kube-system] adding namespace took 6.087919ms I0423 19:30:38.348143 3663 namespace.go:98] [openshift-authentication] adding namespace I0423 19:30:38.348691 3663 namespace.go:102] [openshift-host-network] adding namespace took 6.649711ms I0423 19:30:38.348709 3663 namespace.go:98] [openshift-ingress-canary] adding namespace I0423 19:30:38.349332 3663 namespace.go:102] [open-cluster-management-2ps23krgrug0inkj9hc8sh6m5b2n43rp] adding namespace took 7.288307ms I0423 19:30:38.349351 3663 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0423 19:30:38.349837 3663 namespace.go:102] [openshift-network-console] adding namespace took 7.779011ms I0423 19:30:38.349855 3663 namespace.go:98] [openshift-package-operator] adding namespace I0423 19:30:38.350371 3663 namespace.go:102] [openshift-console-operator] adding namespace took 8.310559ms I0423 19:30:38.350384 3663 namespace.go:98] [openshift-marketplace] adding namespace I0423 19:30:38.350792 3663 namespace.go:102] [openshift-etcd] adding namespace took 8.682941ms I0423 19:30:38.350810 3663 namespace.go:98] [dedicated-admin] adding namespace I0423 19:30:38.351376 3663 namespace.go:102] [openshift-operators-redhat] adding namespace took 7.868506ms I0423 19:30:38.351387 3663 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0423 19:30:38.352114 3663 namespace.go:102] [openshift] adding namespace took 8.156977ms I0423 19:30:38.352130 3663 namespace.go:98] [openshift-machine-api] adding namespace I0423 19:30:38.352607 3663 namespace.go:102] [openshift-kube-apiserver] adding namespace took 8.123734ms I0423 19:30:38.352621 3663 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0423 19:30:38.353098 3663 namespace.go:102] [openshift-backplane-mobb] adding namespace took 8.153338ms I0423 19:30:38.353116 3663 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0423 19:30:38.353558 3663 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 7.759282ms I0423 19:30:38.353574 3663 namespace.go:98] [openshift-network-node-identity] adding namespace I0423 19:30:38.354102 3663 namespace.go:102] [openshift-logging] adding namespace took 7.830846ms I0423 19:30:38.354121 3663 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0423 19:30:38.354600 3663 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 7.860669ms I0423 19:30:38.354615 3663 namespace.go:98] [openshift-node] adding namespace I0423 19:30:38.355441 3663 namespace.go:102] [openshift-backplane-srep] adding namespace took 8.280766ms I0423 19:30:38.355457 3663 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0423 19:30:38.355986 3663 namespace.go:102] [openshift-controller-manager] adding namespace took 8.324609ms I0423 19:30:38.355999 3663 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0423 19:30:38.356489 3663 namespace.go:102] [openshift-authentication] adding namespace took 8.340236ms I0423 19:30:38.356504 3663 namespace.go:98] [kube-node-lease] adding namespace I0423 19:30:38.357010 3663 namespace.go:102] [openshift-ingress-canary] adding namespace took 8.291051ms I0423 19:30:38.357024 3663 namespace.go:98] [openshift-route-controller-manager] adding namespace I0423 19:30:38.357452 3663 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 8.095504ms I0423 19:30:38.357466 3663 namespace.go:98] [openshift-machine-config-operator] adding namespace I0423 19:30:38.357835 3663 namespace.go:102] [openshift-package-operator] adding namespace took 7.973922ms I0423 19:30:38.357847 3663 namespace.go:98] [openshift-backplane] adding namespace I0423 19:30:38.358248 3663 namespace.go:102] [openshift-marketplace] adding namespace took 7.860007ms I0423 19:30:38.358261 3663 namespace.go:98] [openshift-network-operator] adding namespace I0423 19:30:38.358626 3663 namespace.go:102] [dedicated-admin] adding namespace took 7.807582ms I0423 19:30:38.358638 3663 namespace.go:98] [openshift-ingress-operator] adding namespace I0423 19:30:38.358999 3663 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 7.607429ms I0423 19:30:38.359404 3663 namespace.go:102] [openshift-machine-api] adding namespace took 7.269299ms I0423 19:30:38.359416 3663 namespace.go:98] [openshift-dns-operator] adding namespace I0423 19:30:38.359782 3663 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.156125ms I0423 19:30:38.359794 3663 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0423 19:30:38.360161 3663 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.040273ms I0423 19:30:38.360174 3663 namespace.go:98] [openshift-config] adding namespace I0423 19:30:38.360589 3663 namespace.go:102] [openshift-network-node-identity] adding namespace took 7.009709ms I0423 19:30:38.360602 3663 namespace.go:98] [openshift-ingress] adding namespace I0423 19:30:38.360997 3663 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.870115ms I0423 19:30:38.361014 3663 namespace.go:98] [openshift-customer-monitoring] adding namespace I0423 19:30:38.361410 3663 namespace.go:102] [openshift-node] adding namespace took 6.788893ms I0423 19:30:38.361428 3663 namespace.go:98] [default] adding namespace I0423 19:30:38.361812 3663 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.348841ms I0423 19:30:38.361828 3663 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0423 19:30:38.362218 3663 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.21264ms I0423 19:30:38.362233 3663 namespace.go:98] [openshift-apiserver] adding namespace I0423 19:30:38.362675 3663 namespace.go:102] [kube-node-lease] adding namespace took 6.166212ms I0423 19:30:38.362692 3663 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0423 19:30:38.363069 3663 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.040424ms I0423 19:30:38.363099 3663 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0423 19:30:38.363500 3663 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.029211ms I0423 19:30:38.363515 3663 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0423 19:30:38.363883 3663 namespace.go:102] [openshift-backplane] adding namespace took 6.031079ms I0423 19:30:38.363897 3663 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0423 19:30:38.364365 3663 namespace.go:102] [openshift-network-operator] adding namespace took 6.098511ms I0423 19:30:38.364379 3663 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0423 19:30:38.364765 3663 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.121399ms I0423 19:30:38.364780 3663 namespace.go:98] [kube-public] adding namespace I0423 19:30:38.365247 3663 namespace.go:102] [openshift-dns-operator] adding namespace took 5.825172ms I0423 19:30:38.365263 3663 namespace.go:98] [openshift-cluster-version] adding namespace I0423 19:30:38.365706 3663 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 5.906377ms I0423 19:30:38.365720 3663 namespace.go:98] [openshift-image-registry] adding namespace I0423 19:30:38.366097 3663 namespace.go:102] [openshift-config] adding namespace took 5.916147ms I0423 19:30:38.366111 3663 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0423 19:30:38.366470 3663 namespace.go:102] [openshift-ingress] adding namespace took 5.863486ms I0423 19:30:38.366481 3663 namespace.go:98] [openshift-console] adding namespace I0423 19:30:38.366893 3663 namespace.go:102] [openshift-customer-monitoring] adding namespace took 5.871913ms I0423 19:30:38.366904 3663 namespace.go:98] [openshift-operators] adding namespace I0423 19:30:38.367364 3663 namespace.go:102] [default] adding namespace took 5.928667ms I0423 19:30:38.367376 3663 namespace.go:98] [openshift-authentication-operator] adding namespace I0423 19:30:38.367801 3663 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 5.965288ms I0423 19:30:38.367816 3663 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0423 19:30:38.368244 3663 namespace.go:102] [openshift-apiserver] adding namespace took 6.004692ms I0423 19:30:38.368261 3663 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0423 19:30:38.368727 3663 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.028923ms I0423 19:30:38.368741 3663 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0423 19:30:38.369159 3663 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.054345ms I0423 19:30:38.369175 3663 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0423 19:30:38.369902 3663 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.380916ms I0423 19:30:38.369913 3663 namespace.go:98] [openshift-insights] adding namespace I0423 19:30:38.370379 3663 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.476004ms I0423 19:30:38.370393 3663 namespace.go:98] [openshift-service-ca-operator] adding namespace I0423 19:30:38.370788 3663 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.402541ms I0423 19:30:38.370804 3663 namespace.go:98] [openshift-backplane-tam] adding namespace I0423 19:30:38.371196 3663 namespace.go:102] [kube-public] adding namespace took 6.408922ms I0423 19:30:38.371223 3663 namespace.go:98] [openshift-kube-scheduler] adding namespace I0423 19:30:38.371665 3663 namespace.go:102] [openshift-cluster-version] adding namespace took 6.395464ms I0423 19:30:38.371683 3663 namespace.go:98] [openshift-config-operator] adding namespace I0423 19:30:38.372274 3663 namespace.go:102] [openshift-image-registry] adding namespace took 6.548436ms I0423 19:30:38.372838 3663 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.721326ms I0423 19:30:38.373405 3663 namespace.go:102] [openshift-console] adding namespace took 6.919123ms I0423 19:30:38.373883 3663 namespace.go:102] [openshift-operators] adding namespace took 6.973693ms I0423 19:30:38.374314 3663 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.933342ms I0423 19:30:38.374691 3663 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.870211ms I0423 19:30:38.375061 3663 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.793702ms I0423 19:30:38.375091 3663 namespace.go:98] [openshift-backplane-cee] adding namespace I0423 19:30:38.375499 3663 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.75306ms I0423 19:30:38.375512 3663 namespace.go:98] [openshift-monitoring] adding namespace I0423 19:30:38.375912 3663 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.730347ms I0423 19:30:38.376797 3663 namespace.go:102] [openshift-insights] adding namespace took 6.879038ms I0423 19:30:38.377202 3663 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.804027ms I0423 19:30:38.377214 3663 namespace.go:98] [openshift-multus] adding namespace I0423 19:30:38.377603 3663 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.793568ms I0423 19:30:38.377984 3663 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.755512ms I0423 19:30:38.378381 3663 namespace.go:102] [openshift-config-operator] adding namespace took 6.693793ms I0423 19:30:38.378747 3663 namespace.go:102] [openshift-backplane-cee] adding namespace took 3.650938ms I0423 19:30:38.379098 3663 namespace.go:102] [openshift-monitoring] adding namespace took 3.581865ms W0423 19:30:38.379122 3663 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-tct7t: pod openshift-multus/network-metrics-daemon-tct7t: no pod IPs found W0423 19:30:38.379136 3663 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-87d46: pod openshift-multus/network-metrics-daemon-87d46: no pod IPs found W0423 19:30:38.379141 3663 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-s6lhr: pod openshift-multus/network-metrics-daemon-s6lhr: no pod IPs found I0423 19:30:38.379546 3663 namespace.go:102] [openshift-multus] adding namespace took 2.325018ms W0423 19:30:38.379627 3663 master.go:458] Unable to parse local node ip-10-0-1-108.ec2.internal chassis-id annotation. Chassis may be removed during sync W0423 19:30:38.379635 3663 master.go:471] Unable to parse remote node ip-10-0-0-245.ec2.internal chassis-id annotation. Chassis may be removed during sync W0423 19:30:38.379640 3663 master.go:471] Unable to parse remote node ip-10-0-2-122.ec2.internal chassis-id annotation. Chassis may be removed during sync I0423 19:30:38.379644 3663 master.go:481] Removing stale chassis with ID/Name: bb8acef1-6114-401a-a4d2-dc7deb7cebe3, hostname: ip-10-0-1-108.ec2.internal I0423 19:30:38.380067 3663 master.go:548] Adding or Updating local node "ip-10-0-1-108.ec2.internal" for network "default" I0423 19:30:38.380140 3663 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-245.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-245.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-245.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-245.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-245.ec2.internal I0423 19:30:38.380139 3663 obj_retry.go:578] Failed to create *v1.Node ip-10-0-2-122.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-122.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-122.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-122.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-122.ec2.internal I0423 19:30:38.383153 3663 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-108.ec2.internal for the network default I0423 19:30:38.383429 3663 zone_ic_handler.go:195] Time taken to create transit switch: 263.82µs I0423 19:30:38.383990 3663 default_network_controller.go:823] Node add failed for ip-10-0-1-108.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-108.ec2.internal, error getting gateway config for node ip-10-0-1-108.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-108.ec2.internal", failed to update chassis to local for local node ip-10-0-1-108.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-108.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-108.ec2.internal] E0423 19:30:38.384027 3663 obj_retry.go:575] Failed to create *v1.Node ip-10-0-1-108.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-108.ec2.internal, error getting gateway config for node ip-10-0-1-108.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-108.ec2.internal", failed to update chassis to local for local node ip-10-0-1-108.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-108.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-108.ec2.internal] I0423 19:30:38.384066 3663 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0423 19:30:38.384105 3663 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-1-108.ec2.internal", UID:"2257927a-018e-43d3-b287-f15e3cc9b827", APIVersion:"v1", ResourceVersion:"8884", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-108.ec2.internal, error getting gateway config for node ip-10-0-1-108.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-108.ec2.internal", failed to update chassis to local for local node ip-10-0-1-108.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-108.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-108.ec2.internal] I0423 19:30:38.384143 3663 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0423 19:30:38.384155 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-122.ec2.internal in network "default" I0423 19:30:38.384157 3663 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0423 19:30:38.384180 3663 node_tracker.go:235] Node ip-10-0-2-122.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-122.ec2.internal" I0423 19:30:38.384193 3663 node_tracker.go:169] Node ip-10-0-2-122.ec2.internal switch + router changed, syncing services in network "default" I0423 19:30:38.384206 3663 services_controller.go:574] Full service sync requested for network=default I0423 19:30:38.384214 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-245.ec2.internal in network "default" I0423 19:30:38.384222 3663 node_tracker.go:235] Node ip-10-0-0-245.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-245.ec2.internal" I0423 19:30:38.384227 3663 node_tracker.go:169] Node ip-10-0-0-245.ec2.internal switch + router changed, syncing services in network "default" I0423 19:30:38.384231 3663 services_controller.go:574] Full service sync requested for network=default I0423 19:30:38.384239 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-108.ec2.internal in network "default" I0423 19:30:38.384246 3663 node_tracker.go:235] Node ip-10-0-1-108.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-108.ec2.internal" I0423 19:30:38.384252 3663 node_tracker.go:169] Node ip-10-0-1-108.ec2.internal switch + router changed, syncing services in network "default" I0423 19:30:38.384256 3663 services_controller.go:574] Full service sync requested for network=default I0423 19:30:38.433784 3663 iptables.go:108] Creating table: filter chain: FORWARD I0423 19:30:38.440519 3663 iptables.go:108] Creating table: filter chain: OUTPUT I0423 19:30:38.447195 3663 gateway_shared_intf.go:1816] Gateway Creation Complete I0423 19:30:38.447218 3663 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-108.ec2.internal","mac-address":"0a:ff:d0:99:43:55","ip-addresses":["10.0.1.108/24"],"ip-address":"10.0.1.108/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:bb8acef1-6114-401a-a4d2-dc7deb7cebe3 k8s.ovn.org/node-encap-ips:["10.0.1.108"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.108/24"} k8s.ovn.org/zone-name:ip-10-0-1-108.ec2.internal] on node ip-10-0-1-108.ec2.internal I0423 19:30:38.462420 3663 master.go:548] Adding or Updating local node "ip-10-0-1-108.ec2.internal" for network "default" I0423 19:30:38.462450 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-108.ec2.internal in network "default" I0423 19:30:38.462533 3663 node_tracker.go:169] Node ip-10-0-1-108.ec2.internal switch + router changed, syncing services in network "default" I0423 19:30:38.462553 3663 services_controller.go:574] Full service sync requested for network=default I0423 19:30:38.466502 3663 default_node_network_controller.go:519] Node connection status = connected W0423 19:30:38.466986 3663 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-s6lhr: pod openshift-multus/network-metrics-daemon-s6lhr: no pod IPs found W0423 19:30:38.467002 3663 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-m89h2: pod openshift-network-diagnostics/network-check-target-m89h2: no pod IPs found I0423 19:30:38.467988 3663 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-108.ec2.internal for the network default I0423 19:30:38.468162 3663 zone_ic_handler.go:195] Time taken to create transit switch: 160.832µs I0423 19:30:38.484620 3663 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0423 19:30:38.484638 3663 services_controller.go:208] Setting up event handlers for services for network=default I0423 19:30:38.484731 3663 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0423 19:30:38.484766 3663 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0423 19:30:38.484776 3663 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0423 19:30:38.484864 3663 default_network_controller.go:1025] Node "ip-10-0-0-245.ec2.internal" in remote zone "ip-10-0-0-245.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:30:38.484894 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-245.ec2.internal in network "default" I0423 19:30:38.484918 3663 node_tracker.go:235] Node ip-10-0-0-245.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-245.ec2.internal" I0423 19:30:38.484936 3663 node_tracker.go:169] Node ip-10-0-0-245.ec2.internal switch + router changed, syncing services in network "default" I0423 19:30:38.484950 3663 services_controller.go:574] Full service sync requested for network=default I0423 19:30:38.535824 3663 default_network_controller.go:1025] Node "ip-10-0-0-245.ec2.internal" in remote zone "ip-10-0-0-245.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:30:38.584889 3663 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0423 19:30:38.584915 3663 repair.go:60] Starting repairing loop for services I0423 19:30:38.585019 3663 repair.go:131] Deleted 0 stale service LBs I0423 19:30:38.585038 3663 repair.go:137] Deleted 0 stale Chassis Template Vars I0423 19:30:38.585054 3663 repair.go:62] Finished repairing loop for services: 141.012µs I0423 19:30:38.585094 3663 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0423 19:30:38.585104 3663 services_controller.go:254] Starting workers for network=default I0423 19:30:38.585491 3663 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-m89h2] creating logical port openshift-network-diagnostics_network-check-target-m89h2 for pod on switch ip-10-0-1-108.ec2.internal I0423 19:30:38.585532 3663 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-nwqv2, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-nwqv2: suppressed error logged: pod openshift-network-diagnostics/network-check-target-nwqv2: no pod IPs found I0423 19:30:38.585559 3663 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-s6lhr] creating logical port openshift-multus_network-metrics-daemon-s6lhr for pod on switch ip-10-0-1-108.ec2.internal I0423 19:30:38.585532 3663 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-87d46, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-87d46: suppressed error logged: pod openshift-multus/network-metrics-daemon-87d46: no pod IPs found I0423 19:30:38.585485 3663 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-2m7fm, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-2m7fm: suppressed error logged: pod openshift-network-diagnostics/network-check-target-2m7fm: no pod IPs found I0423 19:30:38.585644 3663 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-tct7t, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-tct7t: suppressed error logged: pod openshift-multus/network-metrics-daemon-tct7t: no pod IPs found I0423 19:30:38.585837 3663 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-m89h2 I0423 19:30:38.585909 3663 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-s6lhr I0423 19:30:38.603037 3663 pod.go:62] [openshift-network-diagnostics/network-check-target-m89h2] pod update took 17.325639ms I0423 19:30:38.603060 3663 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-m89h2] addLogicalPort annotation time took 17.359315ms I0423 19:30:38.603967 3663 pods.go:271] [openshift-network-diagnostics/network-check-target-m89h2] addLogicalPort took 18.48503ms, libovsdb time 650.1µs I0423 19:30:38.605757 3663 pod.go:62] [openshift-multus/network-metrics-daemon-s6lhr] pod update took 20.059245ms I0423 19:30:38.605774 3663 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-s6lhr] addLogicalPort annotation time took 20.078902ms I0423 19:30:38.606889 3663 pods.go:271] [openshift-multus/network-metrics-daemon-s6lhr] addLogicalPort took 21.34031ms, libovsdb time 699.358µs I0423 19:30:38.607395 3663 admin_network_policy_controller.go:227] Starting controller default-network-controller I0423 19:30:38.607414 3663 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0423 19:30:38.607427 3663 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0423 19:30:38.607431 3663 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0423 19:30:38.607613 3663 repair.go:29] Repairing admin network policies took 176.477µs I0423 19:30:38.607711 3663 repair.go:92] Repairing baseline admin network policies took 89.067µs I0423 19:30:38.608106 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0423 19:30:38.608128 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0423 19:30:38.609738 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0423 19:30:38.609757 3663 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0423 19:30:38.609768 3663 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0423 19:30:38.611825 3663 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0423 19:30:38.611843 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0423 19:30:38.611851 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0423 19:30:38.612407 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0423 19:30:38.612424 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0423 19:30:38.612430 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0423 19:30:38.613606 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0423 19:30:38.613623 3663 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0423 19:30:38.613631 3663 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0423 19:30:38.615685 3663 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0423 19:30:38.615707 3663 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0423 19:30:38.615716 3663 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0423 19:30:38.617835 3663 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0423 19:30:38.617853 3663 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0423 19:30:38.617862 3663 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0423 19:30:38.618940 3663 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0423 19:30:38.618958 3663 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0423 19:30:38.618966 3663 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0423 19:30:38.620697 3663 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0423 19:30:38.620716 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0423 19:30:38.620725 3663 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0423 19:30:38.622448 3663 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0423 19:30:38.622471 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0423 19:30:38.622480 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0423 19:30:38.624166 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0423 19:30:38.624189 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0423 19:30:38.624583 3663 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0423 19:30:38.624596 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0423 19:30:38.625428 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0423 19:30:38.625449 3663 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0423 19:30:38.625458 3663 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0423 19:30:38.626710 3663 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0423 19:30:38.626729 3663 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0423 19:30:38.626736 3663 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0423 19:30:38.627094 3663 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0423 19:30:38.627114 3663 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0423 19:30:38.627124 3663 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0423 19:30:38.628757 3663 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0423 19:30:38.628776 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0423 19:30:38.628785 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0423 19:30:38.629319 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0423 19:30:38.629339 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0423 19:30:38.629345 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0423 19:30:38.629634 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0423 19:30:38.629655 3663 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0423 19:30:38.629666 3663 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0423 19:30:38.630308 3663 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0423 19:30:38.630327 3663 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0423 19:30:38.630346 3663 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0423 19:30:38.631416 3663 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0423 19:30:38.631437 3663 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0423 19:30:38.631457 3663 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0423 19:30:38.631767 3663 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0423 19:30:38.631788 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0423 19:30:38.631800 3663 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0423 19:30:38.632146 3663 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0423 19:30:38.632167 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0423 19:30:38.632176 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0423 19:30:38.632683 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0423 19:30:38.632701 3663 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0423 19:30:38.632710 3663 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0423 19:30:38.633231 3663 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0423 19:30:38.633250 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0423 19:30:38.633258 3663 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0423 19:30:38.633707 3663 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0423 19:30:38.633724 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0423 19:30:38.633732 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0423 19:30:38.634207 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0423 19:30:38.634225 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0423 19:30:38.634233 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0423 19:30:38.634688 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0423 19:30:38.634706 3663 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0423 19:30:38.634714 3663 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0423 19:30:38.635526 3663 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0423 19:30:38.635544 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0423 19:30:38.635551 3663 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0423 19:30:38.636009 3663 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0423 19:30:38.636045 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0423 19:30:38.636099 3663 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0423 19:30:38.636273 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-245.ec2.internal in network "default" I0423 19:30:38.636346 3663 node_tracker.go:169] Node ip-10-0-0-245.ec2.internal switch + router changed, syncing services in network "default" I0423 19:30:38.636377 3663 services_controller.go:574] Full service sync requested for network=default I0423 19:30:38.636384 3663 default_network_controller.go:1025] Node "ip-10-0-0-245.ec2.internal" in remote zone "ip-10-0-0-245.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:30:38.637137 3663 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0423 19:30:38.637164 3663 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0423 19:30:38.637175 3663 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0423 19:30:38.637274 3663 zone_ic_handler.go:195] Time taken to create transit switch: 250.608µs I0423 19:30:38.637288 3663 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-245.ec2.internal for the network default I0423 19:30:38.638118 3663 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0423 19:30:38.638879 3663 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-245.ec2.internal" on network "default" took: 1.868184ms I0423 19:30:38.642921 3663 egressfirewall.go:374] Starting EgressFirewall controller I0423 19:30:38.642972 3663 controller.go:133] Adding controller egress-firewall-controller event handlers I0423 19:30:38.642973 3663 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0423 19:30:38.643026 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0423 19:30:38.643032 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0423 19:30:38.643038 3663 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0423 19:30:38.643043 3663 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0423 19:30:38.643366 3663 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0423 19:30:38.643381 3663 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0423 19:30:38.643388 3663 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0423 19:30:38.643415 3663 egressfirewall.go:723] Syncing node "ip-10-0-0-245.ec2.internal" for egress firewall I0423 19:30:38.643431 3663 egressqos.go:193] Setting up event handlers for EgressQoS I0423 19:30:38.643432 3663 egressfirewall.go:723] Syncing node "ip-10-0-1-108.ec2.internal" for egress firewall I0423 19:30:38.643440 3663 egressfirewall.go:723] Syncing node "ip-10-0-2-122.ec2.internal" for egress firewall I0423 19:30:38.643595 3663 egressqos.go:245] Starting EgressQoS Controller I0423 19:30:38.643606 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0423 19:30:38.643611 3663 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0423 19:30:38.643616 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0423 19:30:38.643620 3663 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0423 19:30:38.643625 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0423 19:30:38.643629 3663 shared_informer.go:356] "Caches are synced" controller="egressqos" I0423 19:30:38.643634 3663 egressqos.go:259] Repairing EgressQoSes I0423 19:30:38.643639 3663 egressqos.go:399] Starting repairing loop for egressqos I0423 19:30:38.643738 3663 egressqos.go:401] Finished repairing loop for egressqos: 89.677µs I0423 19:30:38.643757 3663 egressservice_zone.go:132] Setting up event handlers for Egress Services I0423 19:30:38.643801 3663 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-108.ec2.internal I0423 19:30:38.643888 3663 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-108.ec2.internal : 88.082µs I0423 19:30:38.644123 3663 egressservice_zone.go:210] Starting Egress Services Controller I0423 19:30:38.644138 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0423 19:30:38.644145 3663 shared_informer.go:356] "Caches are synced" controller="egressservices" I0423 19:30:38.644150 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0423 19:30:38.644159 3663 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0423 19:30:38.644170 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0423 19:30:38.644174 3663 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0423 19:30:38.644179 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0423 19:30:38.644183 3663 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0423 19:30:38.644188 3663 egressservice_zone.go:228] Repairing Egress Services I0423 19:30:38.644504 3663 master_controller.go:88] Starting Admin Policy Based Route Controller I0423 19:30:38.644513 3663 external_controller.go:278] Starting Admin Policy Based Route Controller I0423 19:30:38.644629 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-108.ec2.internal I0423 19:30:38.644641 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-108.ec2.internal: 13.703µs I0423 19:30:38.644647 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-122.ec2.internal I0423 19:30:38.644651 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-122.ec2.internal: 3.804µs I0423 19:30:38.644655 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-245.ec2.internal I0423 19:30:38.644658 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-245.ec2.internal: 3.891µs I0423 19:30:38.644730 3663 default_network_controller.go:585] Completing all the Watchers took 302.913932ms I0423 19:30:38.644745 3663 default_network_controller.go:589] Starting unidling controllers I0423 19:30:38.644760 3663 unidle.go:46] Registering OVN SB ControllerEvent handler I0423 19:30:38.644771 3663 unidle.go:63] Populating Initial ContollerEvent events I0423 19:30:38.644782 3663 unidle.go:79] Setting up event handlers for services I0423 19:30:38.644873 3663 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0423 19:30:38.967670 3663 default_node_network_controller.go:519] Node connection status = connected I0423 19:30:39.467003 3663 default_node_network_controller.go:519] Node connection status = connected I0423 19:30:39.921114 3663 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0423 19:30:39.967882 3663 default_node_network_controller.go:519] Node connection status = connected I0423 19:30:39.986731 3663 bridgeconfig.go:537] Gateway is ready I0423 19:30:39.986751 3663 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0423 19:30:39.999247 3663 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.108. Updating OVN encap IP. I0423 19:30:40.004823 3663 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.108 - it is already configured I0423 19:30:40.004843 3663 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.108/24:{} 172.20.0.1/32:{}]. Updating annotations. I0423 19:30:40.005173 3663 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.108/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-108.ec2.internal","mac-address":"0a:ff:d0:99:43:55","ip-addresses":["10.0.1.108/24"],"ip-address":"10.0.1.108/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:bb8acef1-6114-401a-a4d2-dc7deb7cebe3 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.108/24"}] on node ip-10-0-1-108.ec2.internal I0423 19:30:40.021393 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-108.ec2.internal in network "default" I0423 19:30:40.021448 3663 egressfirewall.go:723] Syncing node "ip-10-0-1-108.ec2.internal" for egress firewall I0423 19:30:40.021464 3663 node_tracker.go:169] Node ip-10-0-1-108.ec2.internal switch + router changed, syncing services in network "default" I0423 19:30:40.021474 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-108.ec2.internal I0423 19:30:40.021490 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-108.ec2.internal: 18.596µs I0423 19:30:40.021489 3663 master.go:548] Adding or Updating local node "ip-10-0-1-108.ec2.internal" for network "default" I0423 19:30:40.021515 3663 services_controller.go:574] Full service sync requested for network=default I0423 19:30:40.021671 3663 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.1.108 I0423 19:30:40.021834 3663 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-108.ec2.internal I0423 19:30:40.026707 3663 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-108.ec2.internal. Updating no re-route policies I0423 19:30:40.026811 3663 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-gcltj I0423 19:30:40.026836 3663 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-108.ec2.internal I0423 19:30:40.026845 3663 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-gcltj I0423 19:30:40.026845 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-2fkvk I0423 19:30:40.026854 3663 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-108.ec2.internal I0423 19:30:40.026854 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-108.ec2.internal I0423 19:30:40.026863 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-2fkvk I0423 19:30:40.026867 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-108.ec2.internal I0423 19:30:40.026874 3663 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-gcltj after 0 failed attempt(s) I0423 19:30:40.026881 3663 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-108.ec2.internal after 0 failed attempt(s) I0423 19:30:40.026812 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-kwddf I0423 19:30:40.026855 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-w7fzq I0423 19:30:40.026888 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-108.ec2.internal after 0 failed attempt(s) I0423 19:30:40.026897 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-jhhsl I0423 19:30:40.026891 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-m89h2 I0423 19:30:40.026900 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-s6lhr I0423 19:30:40.026906 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-w7fzq I0423 19:30:40.026909 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-m89h2 I0423 19:30:40.026905 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-jhhsl I0423 19:30:40.026915 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-s6lhr I0423 19:30:40.026913 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-hnjpc I0423 19:30:40.026929 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-jhhsl after 0 failed attempt(s) I0423 19:30:40.026929 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-hnjpc I0423 19:30:40.026932 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-w7fzq after 0 failed attempt(s) I0423 19:30:40.026842 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-87d46 I0423 19:30:40.026980 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-87d46 I0423 19:30:40.026999 3663 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-m89h2] creating logical port openshift-network-diagnostics_network-check-target-m89h2 for pod on switch ip-10-0-1-108.ec2.internal I0423 19:30:40.027029 3663 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-87d46, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-87d46: suppressed error logged: pod openshift-multus/network-metrics-daemon-87d46: no pod IPs found I0423 19:30:40.026821 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-8qnqx I0423 19:30:40.027046 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-8qnqx I0423 19:30:40.027062 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-8qnqx after 0 failed attempt(s) I0423 19:30:40.026826 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-nwqv2 I0423 19:30:40.027107 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-nwqv2 I0423 19:30:40.026832 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-v9m8h I0423 19:30:40.027133 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-v9m8h I0423 19:30:40.027143 3663 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-nwqv2, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-nwqv2: suppressed error logged: pod openshift-network-diagnostics/network-check-target-nwqv2: no pod IPs found I0423 19:30:40.027162 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-v9m8h after 0 failed attempt(s) I0423 19:30:40.026885 3663 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-kr7qv I0423 19:30:40.027182 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-kr7qv I0423 19:30:40.027214 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-kr7qv after 0 failed attempt(s) I0423 19:30:40.026886 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-2fkvk after 0 failed attempt(s) I0423 19:30:40.026895 3663 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-kwddf I0423 19:30:40.027247 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-kwddf after 0 failed attempt(s) I0423 19:30:40.026951 3663 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-s6lhr] creating logical port openshift-multus_network-metrics-daemon-s6lhr for pod on switch ip-10-0-1-108.ec2.internal I0423 19:30:40.026960 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-hnjpc after 0 failed attempt(s) I0423 19:30:40.028669 3663 pods.go:271] [openshift-multus/network-metrics-daemon-s6lhr] addLogicalPort took 1.726583ms, libovsdb time 836.099µs I0423 19:30:40.028687 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-s6lhr after 0 failed attempt(s) I0423 19:30:40.028712 3663 pods.go:271] [openshift-network-diagnostics/network-check-target-m89h2] addLogicalPort took 1.77714ms, libovsdb time 1.073327ms I0423 19:30:40.028729 3663 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-m89h2 after 0 failed attempt(s) I0423 19:30:40.038845 3663 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0423 19:30:40.053774 3663 iptables.go:108] Creating table: mangle chain: OUTPUT I0423 19:30:40.057668 3663 iptables.go:108] Creating table: nat chain: OUTPUT I0423 19:30:40.091995 3663 iptables.go:108] Creating table: nat chain: PREROUTING I0423 19:30:40.158917 3663 iptables.go:108] Creating table: filter chain: FORWARD I0423 19:30:40.177612 3663 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0423 19:30:40.177753 3663 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0423 19:30:40.177804 3663 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0423 19:30:40.177888 3663 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0423 19:30:40.177936 3663 gateway_init.go:325] Gateway and management port readiness took 1.715582824s I0423 19:30:40.177964 3663 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0423 19:30:40.178001 3663 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0423 19:30:40.178014 3663 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0423 19:30:40.178022 3663 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0423 19:30:40.178113 3663 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0423 19:30:40.178202 3663 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 57.807µs I0423 19:30:40.178477 3663 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0423 19:30:40.178505 3663 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 19.322µs I0423 19:30:40.178514 3663 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 38.309µs I0423 19:30:40.178519 3663 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0423 19:30:40.178529 3663 nad_controller.go:379] [node-nad-controller NAD controller]: started I0423 19:30:40.178533 3663 default_node_network_controller.go:899] Starting the default node network controller I0423 19:30:40.178538 3663 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0423 19:30:40.178552 3663 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 12.482µs I0423 19:30:40.178697 3663 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0423 19:30:40.178705 3663 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0423 19:30:40.178729 3663 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bf8810 0xc0002c64d0 0xc0002c6540 0xc0002c65b0 0xc0002c6620 0xc0002c6690 0xc0002c6700 0xc000be1270 0xc0002c6770 0xc0002c6850 0xc000be0410 0xc00645f340 0xc0002c68c0 map[0x2ff9080:0xc003123380 0x2ff95c0:0xc003123500 0x2ffa2e0:0xc0031232c0 0x2ffa580:0xc003123320 0x2ffad60:0xc0031233e0 0x2ffb000:0xc003123440 0x2ffb2a0:0xc0031234a0 0x2ffba80:0xc003123560 0x2ffbd20:0xc0031238c0 0x3026980:0xc001211ce0 0x3026d20:0xc001211d40 0x3027800:0xc000c0f560 0x3029160:0xc001be6000 0x30298a0:0xc0022900c0 0x302a720:0xc001211c80] 0xc0008a90a0 0} I0423 19:30:40.178796 3663 gateway.go:151] Starting gateway service sync I0423 19:30:40.179233 3663 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0423 19:30:40.181648 3663 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0423 19:30:40.198689 3663 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0423 19:30:40.199969 3663 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0423 19:30:40.201262 3663 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0423 19:30:40.235730 3663 gateway.go:168] Gateway service sync done. Time taken: 56.918826ms I0423 19:30:40.236220 3663 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32150 I0423 19:30:40.236560 3663 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bf8810 0xc0002c64d0 0xc0002c6540 0xc0002c65b0 0xc0002c6620 0xc0002c6690 0xc0002c6700 0xc000be1270 0xc0002c6770 0xc0002c6850 0xc000be0410 0xc00645f340 0xc0002c68c0 map[0x2ff9080:0xc003123380 0x2ff95c0:0xc003123500 0x2ffa2e0:0xc0031232c0 0x2ffa580:0xc003123320 0x2ffad60:0xc0031233e0 0x2ffb000:0xc003123440 0x2ffb2a0:0xc0031234a0 0x2ffba80:0xc003123560 0x2ffbd20:0xc0031238c0 0x3026980:0xc001211ce0 0x3026d20:0xc001211d40 0x3027800:0xc000c0f560 0x3029160:0xc001be6000 0x30298a0:0xc0022900c0 0x302a720:0xc001211c80] 0xc0008a90a0 0} I0423 19:30:40.237028 3663 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bf8810 0xc0002c64d0 0xc0002c6540 0xc0002c65b0 0xc0002c6620 0xc0002c6690 0xc0002c6700 0xc000be1270 0xc0002c6770 0xc0002c6850 0xc000be0410 0xc00645f340 0xc0002c68c0 map[0x2ff9080:0xc003123380 0x2ff95c0:0xc003123500 0x2ffa2e0:0xc0031232c0 0x2ffa580:0xc003123320 0x2ffad60:0xc0031233e0 0x2ffb000:0xc003123440 0x2ffb2a0:0xc0031234a0 0x2ffba80:0xc003123560 0x2ffbd20:0xc0031238c0 0x3026980:0xc001211ce0 0x3026d20:0xc001211d40 0x3027800:0xc000c0f560 0x3029160:0xc001be6000 0x30298a0:0xc0022900c0 0x302a720:0xc001211c80] 0xc0008a90a0 0} I0423 19:30:40.237091 3663 gateway.go:532] Reconciling gateway with updates I0423 19:30:40.242950 3663 gateway.go:339] Spawning Conntrack Rule Check Thread I0423 19:30:40.243045 3663 default_node_network_controller.go:944] Gateway and management port readiness took 64.333029ms I0423 19:30:40.243060 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 19:30:40.243096 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 19:30:40.243100 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 19:30:40.243106 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 19:30:40.243112 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 19:30:40.243119 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 19:30:40.243127 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0423 19:30:40.243138 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 19:30:40.243143 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 19:30:40.243138 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 19:30:40.243153 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 19:30:40.243166 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 19:30:40.243165 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0423 19:30:40.243171 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0423 19:30:40.243176 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 19:30:40.243185 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0423 19:30:40.243187 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0423 19:30:40.243191 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 19:30:40.243187 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 19:30:40.243200 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0423 19:30:40.243211 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 19:30:40.243220 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 19:30:40.243215 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0423 19:30:40.243227 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0423 19:30:40.243233 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0423 19:30:40.243234 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 19:30:40.243229 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 19:30:40.243245 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 19:30:40.243243 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0423 19:30:40.243184 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 19:30:40.243252 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0423 19:30:40.243258 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 19:30:40.243263 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 19:30:40.243239 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 19:30:40.243275 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 19:30:40.243278 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 19:30:40.243280 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 19:30:40.243283 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0423 19:30:40.243284 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 19:30:40.243292 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 19:30:40.243176 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 19:30:40.243309 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0423 19:30:40.243317 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 19:30:40.243321 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 19:30:40.243322 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 19:30:40.243326 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0423 19:30:40.243330 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 19:30:40.243332 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 19:30:40.243318 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0423 19:30:40.243248 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0423 19:30:40.243339 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 19:30:40.243342 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:30:40.243354 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 19:30:40.243353 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 19:30:40.243362 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 19:30:40.243364 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:30:40.243370 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0423 19:30:40.243279 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 19:30:40.243195 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0423 19:30:40.243442 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0423 19:30:40.243490 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0423 19:30:40.243341 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0423 19:30:40.243509 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0423 19:30:40.243209 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 19:30:40.243494 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0423 19:30:40.243517 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 19:30:40.243527 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0423 19:30:40.243537 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0423 19:30:40.243276 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 19:30:40.243159 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:30:40.243588 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:30:40.243238 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0423 19:30:40.243161 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0423 19:30:40.243188 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 19:30:40.243643 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 19:30:40.243222 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0423 19:30:40.243669 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0423 19:30:40.243162 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 19:30:40.243248 3663 port_claim.go:71] Svc openshift-ingress/router-default: port 30739 is already open, no action needed I0423 19:30:40.243820 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0423 19:30:40.243825 3663 port_claim.go:71] Svc openshift-ingress/router-default: port 30326 is already open, no action needed I0423 19:30:40.243259 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0423 19:30:40.243175 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 19:30:40.243287 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0423 19:30:40.243290 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 19:30:40.243263 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 19:30:40.243296 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 19:30:40.244007 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 19:30:40.243302 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 19:30:40.243304 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 19:30:40.243303 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 19:30:40.244066 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 19:30:40.243303 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0423 19:30:40.243305 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 19:30:40.244106 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 19:30:40.243322 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 19:30:40.244131 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 19:30:40.244135 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0423 19:30:40.243333 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 19:30:40.243352 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0423 19:30:40.244161 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0423 19:30:40.243433 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 19:30:40.243209 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0423 19:30:40.244203 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0423 19:30:40.244208 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0423 19:30:40.243149 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0423 19:30:40.244330 3663 node_ip_handler_linux.go:160] Node IP manager is running W0423 19:30:40.280995 3663 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0423 19:30:40.281179 3663 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0423 19:30:40.281330 3663 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0423 19:30:40.291275 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0423 19:30:40.291309 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0423 19:30:40.331492 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0423 19:30:40.331521 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0423 19:30:40.331529 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0423 19:30:40.331535 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0423 19:30:40.331540 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0423 19:30:40.331561 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0423 19:30:40.331569 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0423 19:30:40.331573 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0423 19:30:40.331581 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0423 19:30:40.331586 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0423 19:30:40.331591 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0423 19:30:40.331597 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0423 19:30:40.331602 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0423 19:30:40.371135 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0423 19:30:40.371146 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0423 19:30:40.371167 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0423 19:30:40.371180 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0423 19:30:40.371187 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0423 19:30:40.371196 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0423 19:30:40.371212 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0423 19:30:40.371220 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0423 19:30:40.371227 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0423 19:30:40.371231 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0423 19:30:40.371240 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0423 19:30:40.371246 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0423 19:30:40.371252 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0423 19:30:41.194053 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-245.ec2.internal in network "default" I0423 19:30:41.194094 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-245.ec2.internal I0423 19:30:41.194111 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-245.ec2.internal: 41.741µs I0423 19:30:41.194134 3663 egressfirewall.go:723] Syncing node "ip-10-0-0-245.ec2.internal" for egress firewall I0423 19:30:41.194157 3663 node_tracker.go:169] Node ip-10-0-0-245.ec2.internal switch + router changed, syncing services in network "default" I0423 19:30:41.194162 3663 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-245.ec2.internal. Updating no re-route policies I0423 19:30:41.194176 3663 services_controller.go:574] Full service sync requested for network=default I0423 19:30:41.564814 3663 default_node_network_controller.go:1380] Starting node controller node sync W0423 19:30:41.564880 3663 egressip_healthcheck.go:76] Health checking using insecure connection I0423 19:30:41.564950 3663 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0423 19:30:41.577007 3663 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 12.171035ms I0423 19:30:41.577146 3663 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-245.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0423 19:30:41.577158 3663 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-245.ec2.internal", IP: 10.0.0.245 to PMTUD blocking rules I0423 19:30:41.583217 3663 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0423 19:30:41.583507 3663 egressservice_node.go:93] Setting up event handlers for Egress Services I0423 19:30:41.583674 3663 egressservice_node.go:183] Starting Egress Services Controller I0423 19:30:41.583694 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0423 19:30:41.583709 3663 shared_informer.go:356] "Caches are synced" controller="egressservices" I0423 19:30:41.583721 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0423 19:30:41.583726 3663 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0423 19:30:41.583731 3663 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0423 19:30:41.583735 3663 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0423 19:30:41.583740 3663 egressservice_node.go:197] Repairing Egress Services I0423 19:30:41.594269 3663 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0423 19:30:41.594281 3663 external_controller.go:278] Starting Admin Policy Based Route Controller I0423 19:30:41.594502 3663 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0423 19:30:41.594513 3663 link_network_manager.go:119] Link manager is running I0423 19:30:41.594521 3663 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0423 19:30:41.594780 3663 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0423 19:30:41.594893 3663 vrf_manager.go:128] VRF manager is running I0423 19:30:41.594973 3663 node_controller_manager.go:437] Removing flows to drop GARP I0423 19:30:41.594982 3663 gateway.go:532] Reconciling gateway with updates I0423 19:30:41.621105 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 19:30:41.621120 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0423 19:30:41.621125 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0423 19:30:41.621131 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 19:30:41.621134 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 19:30:41.621141 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0423 19:30:41.621143 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 19:30:41.621106 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 19:30:41.621156 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 19:30:41.621157 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 19:30:41.621159 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:30:41.621165 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 19:30:41.621165 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 19:30:41.621170 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:30:41.621170 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 19:30:41.621181 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:30:41.621190 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:30:41.621192 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 19:30:41.621121 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 19:30:41.621198 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 19:30:41.621202 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 19:30:41.621201 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 19:30:41.621211 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 19:30:41.621210 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0423 19:30:41.621213 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0423 19:30:41.621168 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 19:30:41.621219 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 19:30:41.621222 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0423 19:30:41.621219 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 19:30:41.621132 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0423 19:30:41.621226 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 19:30:41.621233 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0423 19:30:41.621236 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0423 19:30:41.621236 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 19:30:41.621238 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 19:30:41.621228 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 19:30:41.621239 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 19:30:41.621251 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 19:30:41.621253 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 19:30:41.621256 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 19:30:41.621259 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 19:30:41.621262 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 19:30:41.621261 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0423 19:30:41.621265 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 19:30:41.621264 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0423 19:30:41.621268 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 19:30:41.621272 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0423 19:30:41.621279 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0423 19:30:41.621280 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0423 19:30:41.621282 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 19:30:41.621137 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0423 19:30:41.621290 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0423 19:30:41.621297 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0423 19:30:41.621270 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0423 19:30:41.621315 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0423 19:30:41.621203 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 19:30:41.621323 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0423 19:30:41.621187 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0423 19:30:41.621335 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0423 19:30:41.621344 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0423 19:30:41.621357 3663 port_claim.go:71] Svc openshift-ingress/router-default: port 30739 is already open, no action needed I0423 19:30:41.621362 3663 port_claim.go:71] Svc openshift-ingress/router-default: port 30326 is already open, no action needed I0423 19:30:41.621183 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 19:30:41.621378 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 19:30:41.621214 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 19:30:41.621397 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 19:30:41.621234 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 19:30:41.621460 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0423 19:30:41.621258 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0423 19:30:41.621148 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 19:30:41.621475 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 19:30:41.621149 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0423 19:30:41.621147 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0423 19:30:41.621506 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0423 19:30:41.621160 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 19:30:41.621543 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 19:30:41.621205 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0423 19:30:41.621566 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0423 19:30:41.621205 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 19:30:41.621592 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 19:30:41.621215 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0423 19:30:41.621215 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0423 19:30:41.621226 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 19:30:41.621227 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0423 19:30:41.621664 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0423 19:30:41.621226 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 19:30:41.621695 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 19:30:41.621228 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 19:30:41.621238 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 19:30:41.621738 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 19:30:41.621236 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 19:30:41.621761 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 19:30:41.621245 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0423 19:30:41.621787 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0423 19:30:41.621247 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 19:30:41.621248 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 19:30:41.621261 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 19:30:41.621260 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0423 19:30:41.621254 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0423 19:30:41.621876 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0423 19:30:41.621255 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 19:30:41.621901 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 19:30:41.621149 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 19:30:41.621258 3663 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 19:30:41.621945 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 19:30:41.621950 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0423 19:30:41.621282 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0423 19:30:41.621289 3663 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0423 19:30:41.621962 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0423 19:30:41.667870 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0423 19:30:41.667899 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0423 19:30:41.667909 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0423 19:30:41.667917 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0423 19:30:41.709215 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0423 19:30:41.709227 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0423 19:30:41.709214 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0423 19:30:41.709239 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0423 19:30:41.709247 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0423 19:30:41.709257 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0423 19:30:41.750407 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0423 19:30:41.750441 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0423 19:30:41.750449 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0423 19:30:41.750456 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0423 19:30:41.750461 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0423 19:30:41.750468 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0423 19:30:41.750473 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0423 19:30:41.750487 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0423 19:30:41.750506 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0423 19:30:41.750513 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0423 19:30:41.750523 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0423 19:30:41.750529 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0423 19:30:41.750534 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0423 19:30:41.750542 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0423 19:30:41.750546 3663 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0423 19:30:42.100924 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-122.ec2.internal in network "default" I0423 19:30:42.100928 3663 default_network_controller.go:1025] Node "ip-10-0-2-122.ec2.internal" in remote zone "ip-10-0-2-122.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:30:42.100952 3663 node_tracker.go:235] Node ip-10-0-2-122.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-122.ec2.internal" I0423 19:30:42.100970 3663 node_tracker.go:169] Node ip-10-0-2-122.ec2.internal switch + router changed, syncing services in network "default" I0423 19:30:42.100984 3663 services_controller.go:574] Full service sync requested for network=default I0423 19:30:42.133153 3663 default_network_controller.go:1025] Node "ip-10-0-2-122.ec2.internal" in remote zone "ip-10-0-2-122.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:30:42.351793 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-122.ec2.internal in network "default" I0423 19:30:42.351816 3663 default_network_controller.go:1025] Node "ip-10-0-2-122.ec2.internal" in remote zone "ip-10-0-2-122.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:30:42.351860 3663 node_tracker.go:169] Node ip-10-0-2-122.ec2.internal switch + router changed, syncing services in network "default" I0423 19:30:42.351873 3663 services_controller.go:574] Full service sync requested for network=default I0423 19:30:42.352902 3663 zone_ic_handler.go:195] Time taken to create transit switch: 294.036µs I0423 19:30:42.352917 3663 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-122.ec2.internal for the network default I0423 19:30:42.354017 3663 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-122.ec2.internal" on network "default" took: 1.420889ms I0423 19:30:42.909021 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-122.ec2.internal I0423 19:30:42.909049 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-122.ec2.internal: 40.084µs I0423 19:30:42.909111 3663 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-122.ec2.internal in network "default" I0423 19:30:42.909127 3663 egressfirewall.go:723] Syncing node "ip-10-0-2-122.ec2.internal" for egress firewall I0423 19:30:42.909151 3663 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-122.ec2.internal", IP: 10.0.2.122 to PMTUD blocking rules I0423 19:30:42.909161 3663 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-122.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0423 19:30:42.909175 3663 node_tracker.go:169] Node ip-10-0-2-122.ec2.internal switch + router changed, syncing services in network "default" I0423 19:30:42.909194 3663 services_controller.go:574] Full service sync requested for network=default I0423 19:30:42.915718 3663 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-122.ec2.internal. Updating no re-route policies I0423 19:30:42.925811 3663 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-122.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0423 19:30:42.925826 3663 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-122.ec2.internal", IP: 10.0.2.122 to PMTUD blocking rules I0423 19:30:43.862926 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-108.ec2.internal I0423 19:30:43.862953 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-108.ec2.internal: 34.123µs I0423 19:30:43.890588 3663 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-57ff957978-f8qz2] creating logical port openshift-image-registry_image-registry-57ff957978-f8qz2 for pod on switch ip-10-0-1-108.ec2.internal I0423 19:30:43.890794 3663 kube.go:256] Updating pod openshift-image-registry/image-registry-57ff957978-f8qz2 I0423 19:30:43.908740 3663 pod.go:62] [openshift-image-registry/image-registry-57ff957978-f8qz2] pod update took 17.977677ms I0423 19:30:43.908768 3663 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-57ff957978-f8qz2] addLogicalPort annotation time took 18.006422ms I0423 19:30:43.909920 3663 pods.go:271] [openshift-image-registry/image-registry-57ff957978-f8qz2] addLogicalPort took 19.356548ms, libovsdb time 885.374µs I0423 19:30:43.936479 3663 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-wghf4] creating logical port openshift-dns_dns-default-wghf4 for pod on switch ip-10-0-1-108.ec2.internal I0423 19:30:43.936666 3663 kube.go:256] Updating pod openshift-dns/dns-default-wghf4 I0423 19:30:43.936854 3663 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-pg2ns] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-pg2ns for pod on switch ip-10-0-1-108.ec2.internal I0423 19:30:43.937019 3663 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-pg2ns I0423 19:30:43.945359 3663 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-dr2mb] creating logical port openshift-ingress-canary_ingress-canary-dr2mb for pod on switch ip-10-0-1-108.ec2.internal I0423 19:30:43.945438 3663 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29616210-gpczz for pod on switch ip-10-0-1-108.ec2.internal I0423 19:30:43.945505 3663 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-dr2mb I0423 19:30:43.945534 3663 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz I0423 19:30:43.952903 3663 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-pg2ns] pod update took 15.902349ms I0423 19:30:43.952929 3663 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-pg2ns] addLogicalPort annotation time took 15.930497ms I0423 19:30:43.953264 3663 pod.go:62] [openshift-dns/dns-default-wghf4] pod update took 16.629866ms I0423 19:30:43.953281 3663 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-wghf4] addLogicalPort annotation time took 16.653448ms I0423 19:30:43.954052 3663 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-pg2ns] addLogicalPort took 17.212127ms, libovsdb time 843.887µs I0423 19:30:43.954459 3663 pods.go:271] [openshift-dns/dns-default-wghf4] addLogicalPort took 18.00425ms, libovsdb time 964.733µs I0423 19:30:43.962628 3663 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz] pod update took 17.106963ms I0423 19:30:43.962649 3663 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz] addLogicalPort annotation time took 17.130933ms I0423 19:30:43.963762 3663 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz] addLogicalPort took 18.325114ms, libovsdb time 836.219µs I0423 19:30:43.964979 3663 pod.go:62] [openshift-ingress-canary/ingress-canary-dr2mb] pod update took 19.512085ms I0423 19:30:43.965001 3663 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-dr2mb] addLogicalPort annotation time took 19.537695ms I0423 19:30:43.966065 3663 pods.go:271] [openshift-ingress-canary/ingress-canary-dr2mb] addLogicalPort took 20.723477ms, libovsdb time 788.912µs I0423 19:30:44.834311 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-245.ec2.internal I0423 19:30:44.834576 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-245.ec2.internal: 274.551µs I0423 19:30:44.915707 3663 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-pg2ns 7ebfc7c272406d371b1e7ef9ceb3e51f4edfd0e3694d5f99d71579d86814d148 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-pg2ns 7ebfc7c272406d371b1e7ef9ceb3e51f4edfd0e3694d5f99d71579d86814d148 network default NAD default NAD key ] I0423 19:30:44.915725 3663 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz bcfd27bee01b0cb9e9d3a79dcfc2790aea4ac48f29f15c6fe7d03500627ff5d6 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz bcfd27bee01b0cb9e9d3a79dcfc2790aea4ac48f29f15c6fe7d03500627ff5d6 network default NAD default NAD key ] I0423 19:30:44.915707 3663 cni.go:452] [openshift-ingress-canary/ingress-canary-dr2mb cf52e61ade413f1577157907f6b0e05717bd551f65450c235c6ad532af914bed network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-dr2mb cf52e61ade413f1577157907f6b0e05717bd551f65450c235c6ad532af914bed network default NAD default NAD key ] I0423 19:30:44.966370 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-pg2ns, hostIfaceName: 7ebfc7c272406d3, network: default, NAD default, SandboxID: "7ebfc7c272406d371b1e7ef9ceb3e51f4edfd0e3694d5f99d71579d86814d148", PCI device ID: , UID: "1504679d-d74d-45e5-865e-be5cee4a2f15", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0423 19:30:44.966667 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-dr2mb, hostIfaceName: cf52e61ade413f1, network: default, NAD default, SandboxID: "cf52e61ade413f1577157907f6b0e05717bd551f65450c235c6ad532af914bed", PCI device ID: , UID: "0448832e-2fcf-4c63-92e1-fefd4503cfa2", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0423 19:30:44.967062 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29616210-gpczz, hostIfaceName: bcfd27bee01b0cb, network: default, NAD default, SandboxID: "bcfd27bee01b0cb9e9d3a79dcfc2790aea4ac48f29f15c6fe7d03500627ff5d6", PCI device ID: , UID: "f455410c-263b-4d7d-bad6-d7c6f10cec54", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0423 19:30:45.019751 3663 cni.go:473] [openshift-ingress-canary/ingress-canary-dr2mb cf52e61ade413f1577157907f6b0e05717bd551f65450c235c6ad532af914bed network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-dr2mb cf52e61ade413f1577157907f6b0e05717bd551f65450c235c6ad532af914bed network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:7f:26:a0:92:58\",\"name\":\"cf52e61ade413f1\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1d556ba5-e40e-4914-8b2e-38636b954243\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:30:45.024207 3663 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-pg2ns 7ebfc7c272406d371b1e7ef9ceb3e51f4edfd0e3694d5f99d71579d86814d148 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-pg2ns 7ebfc7c272406d371b1e7ef9ceb3e51f4edfd0e3694d5f99d71579d86814d148 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:cb:d0:b9:f3:8b\",\"name\":\"7ebfc7c272406d3\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3aa68aec-843e-4947-9829-463d94f054d6\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:30:45.024509 3663 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz bcfd27bee01b0cb9e9d3a79dcfc2790aea4ac48f29f15c6fe7d03500627ff5d6 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz bcfd27bee01b0cb9e9d3a79dcfc2790aea4ac48f29f15c6fe7d03500627ff5d6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:57:28:3c:cb:29\",\"name\":\"bcfd27bee01b0cb\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f5d5254e-c736-4a46-83c6-e7b84b5e9c27\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:30:47.263802 3663 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-122.ec2.internal I0423 19:30:47.263839 3663 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-122.ec2.internal: 46.509µs I0423 19:30:47.985759 3663 cni.go:452] [openshift-network-diagnostics/network-check-target-m89h2 8123e032da0dafb41d09f4a56d44e846a19872de3a7cceee890abc0f01ce1750 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-m89h2 8123e032da0dafb41d09f4a56d44e846a19872de3a7cceee890abc0f01ce1750 network default NAD default NAD key ] I0423 19:30:47.997354 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-m89h2, hostIfaceName: 8123e032da0dafb, network: default, NAD default, SandboxID: "8123e032da0dafb41d09f4a56d44e846a19872de3a7cceee890abc0f01ce1750", PCI device ID: , UID: "594299b2-2599-42c2-9f38-2c8f3337562d", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0423 19:30:48.045983 3663 cni.go:473] [openshift-network-diagnostics/network-check-target-m89h2 8123e032da0dafb41d09f4a56d44e846a19872de3a7cceee890abc0f01ce1750 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-m89h2 8123e032da0dafb41d09f4a56d44e846a19872de3a7cceee890abc0f01ce1750 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:61:7e:4d:b9:24\",\"name\":\"8123e032da0dafb\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b6aa8f13-1ba3-491d-8b07-0625caf6e650\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:30:53.056358 3663 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:30:53.164535 3663 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:30:53.187744 3663 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0423 19:31:06.199396 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0423 19:31:20.145379 3663 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:31:24.174420 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0423 19:31:41.307991 3663 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390sh4kw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:31:41.308562 3663 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390sh4kw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:31:41.308591 3663 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390sh4kw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:31:41.308601 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0423 19:31:41.308615 3663 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390sh4kw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 19:31:41.308620 3663 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0423 19:31:42.333985 3663 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390sh4kw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:31:42.334009 3663 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390sh4kw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:31:42.334024 3663 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390sh4kw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:31:42.334034 3663 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390sh4kw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:31:43.451257 3663 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0423 19:31:43.462313 3663 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0423 19:31:49.181032 3663 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:32:42.339612 3663 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5549b9c849-rjng6] creating logical port openshift-ingress_router-default-5549b9c849-rjng6 for pod on switch ip-10-0-1-108.ec2.internal I0423 19:32:42.339706 3663 kube.go:256] Updating pod openshift-ingress/router-default-5549b9c849-rjng6 I0423 19:32:42.355982 3663 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-dbcccc67f-9st5g] creating logical port openshift-ingress_router-default-dbcccc67f-9st5g for pod on switch ip-10-0-1-108.ec2.internal I0423 19:32:42.356153 3663 kube.go:256] Updating pod openshift-ingress/router-default-dbcccc67f-9st5g I0423 19:32:42.372027 3663 pod.go:62] [openshift-ingress/router-default-5549b9c849-rjng6] pod update took 32.349046ms I0423 19:32:42.372048 3663 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-5549b9c849-rjng6] addLogicalPort annotation time took 32.375111ms I0423 19:32:42.373134 3663 pods.go:271] [openshift-ingress/router-default-5549b9c849-rjng6] addLogicalPort took 33.536028ms, libovsdb time 814.847µs I0423 19:32:42.385485 3663 pod.go:62] [openshift-ingress/router-default-dbcccc67f-9st5g] pod update took 29.366477ms I0423 19:32:42.385506 3663 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-dbcccc67f-9st5g] addLogicalPort annotation time took 29.395777ms I0423 19:32:42.386608 3663 pods.go:271] [openshift-ingress/router-default-dbcccc67f-9st5g] addLogicalPort took 30.64401ms, libovsdb time 838.076µs W0423 19:32:47.377939 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0423 19:32:47.646621 3663 namespace.go:98] [openshift-service-ca] adding namespace I0423 19:32:47.647471 3663 namespace.go:102] [openshift-service-ca] adding namespace took 826.579µs I0423 19:32:47.654799 3663 namespace.go:142] [openshift-service-ca] updating namespace I0423 19:32:48.062178 3663 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0423 19:32:48.062980 3663 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 776.035µs I0423 19:32:48.096779 3663 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0423 19:32:48.196956 3663 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-gnwdn] creating logical port openshift-kube-storage-version-migrator_migrator-846d9b8bdc-gnwdn for pod on switch ip-10-0-1-108.ec2.internal I0423 19:32:48.197047 3663 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-846d9b8bdc-gnwdn I0423 19:32:48.213320 3663 pod.go:62] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-gnwdn] pod update took 16.295117ms I0423 19:32:48.213342 3663 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-gnwdn] addLogicalPort annotation time took 16.323312ms I0423 19:32:48.214536 3663 pods.go:271] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-gnwdn] addLogicalPort took 17.602313ms, libovsdb time 921.707µs I0423 19:32:48.553463 3663 cni.go:452] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-gnwdn 855d42271e7ecb028af9253393d06985dcfa764bee992b9885de61fb8fad25fd network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-gnwdn 855d42271e7ecb028af9253393d06985dcfa764bee992b9885de61fb8fad25fd network default NAD default NAD key ] I0423 19:32:48.562764 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-846d9b8bdc-gnwdn, hostIfaceName: 855d42271e7ecb0, network: default, NAD default, SandboxID: "855d42271e7ecb028af9253393d06985dcfa764bee992b9885de61fb8fad25fd", PCI device ID: , UID: "f8a36ad6-0f36-40b5-9875-1f8f5536c1fe", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0423 19:32:48.611908 3663 cni.go:473] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-gnwdn 855d42271e7ecb028af9253393d06985dcfa764bee992b9885de61fb8fad25fd network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-gnwdn 855d42271e7ecb028af9253393d06985dcfa764bee992b9885de61fb8fad25fd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:2d:79:a7:70:d3\",\"name\":\"855d42271e7ecb0\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7afd22f4-2903-4195-813c-09277761a483\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:32:56.420122 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0423 19:33:13.579347 3663 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-9j28j] creating logical port openshift-insights_insights-runtime-extractor-9j28j for pod on switch ip-10-0-1-108.ec2.internal I0423 19:33:13.579486 3663 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-9j28j I0423 19:33:13.601727 3663 pod.go:62] [openshift-insights/insights-runtime-extractor-9j28j] pod update took 22.268415ms I0423 19:33:13.601752 3663 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-9j28j] addLogicalPort annotation time took 22.297663ms I0423 19:33:13.602778 3663 pods.go:271] [openshift-insights/insights-runtime-extractor-9j28j] addLogicalPort took 23.45247ms, libovsdb time 725.055µs I0423 19:33:13.943600 3663 cni.go:452] [openshift-insights/insights-runtime-extractor-9j28j 80ece93b05d86c5aff558438a5ab77c96507a0265b6fb58371392e7fc7780315 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-9j28j 80ece93b05d86c5aff558438a5ab77c96507a0265b6fb58371392e7fc7780315 network default NAD default NAD key ] I0423 19:33:13.953059 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-9j28j, hostIfaceName: 80ece93b05d86c5, network: default, NAD default, SandboxID: "80ece93b05d86c5aff558438a5ab77c96507a0265b6fb58371392e7fc7780315", PCI device ID: , UID: "8e57558d-c7f6-4e37-a148-1b441f961010", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0423 19:33:14.006042 3663 cni.go:473] [openshift-insights/insights-runtime-extractor-9j28j 80ece93b05d86c5aff558438a5ab77c96507a0265b6fb58371392e7fc7780315 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-9j28j 80ece93b05d86c5aff558438a5ab77c96507a0265b6fb58371392e7fc7780315 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:77:21:a0:c6:1f\",\"name\":\"80ece93b05d86c5\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/da46f3d1-4a6d-4653-998c-703ae7ccb493\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:33:14.503261 3663 obj_retry.go:498] Detected object openshift-image-registry/image-registry-57ff957978-f8qz2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:33:14.503309 3663 pods.go:176] Deleting pod: openshift-image-registry/image-registry-57ff957978-f8qz2 I0423 19:33:14.504716 3663 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-57ff957978-f8qz2, ips: 10.129.0.5 I0423 19:33:14.504755 3663 obj_retry.go:498] Detected object openshift-image-registry/image-registry-57ff957978-f8qz2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:33:14.512473 3663 cni.go:452] [openshift-ingress/router-default-dbcccc67f-9st5g 5c4039bbe81ee0843f3c7ca822de83d90426d394ec5f90eb4a282cb84a4303c7 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-dbcccc67f-9st5g 5c4039bbe81ee0843f3c7ca822de83d90426d394ec5f90eb4a282cb84a4303c7 network default NAD default NAD key ] I0423 19:33:14.514519 3663 cni.go:452] [openshift-ingress/router-default-5549b9c849-rjng6 fb1d849ae980429eef39933157692601f225dc595a7ed4870f4c09f5c93dc3f7 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-5549b9c849-rjng6 fb1d849ae980429eef39933157692601f225dc595a7ed4870f4c09f5c93dc3f7 network default NAD default NAD key ] I0423 19:33:14.514911 3663 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7fb79b599b-5pdmw] creating logical port openshift-image-registry_image-registry-7fb79b599b-5pdmw for pod on switch ip-10-0-1-108.ec2.internal I0423 19:33:14.515029 3663 kube.go:256] Updating pod openshift-image-registry/image-registry-7fb79b599b-5pdmw I0423 19:33:14.522495 3663 obj_retry.go:492] Detected object openshift-image-registry/image-registry-57ff957978-f8qz2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:33:14.522526 3663 obj_retry.go:492] Detected object openshift-image-registry/image-registry-57ff957978-f8qz2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:33:14.524580 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-dbcccc67f-9st5g, hostIfaceName: 5c4039bbe81ee08, network: default, NAD default, SandboxID: "5c4039bbe81ee0843f3c7ca822de83d90426d394ec5f90eb4a282cb84a4303c7", PCI device ID: , UID: "6bef6477-88ab-4580-a932-dae739e89101", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0423 19:33:14.524949 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5549b9c849-rjng6, hostIfaceName: fb1d849ae980429, network: default, NAD default, SandboxID: "fb1d849ae980429eef39933157692601f225dc595a7ed4870f4c09f5c93dc3f7", PCI device ID: , UID: "09bc0b2c-bcd2-407b-8ccc-c884c0043f55", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0423 19:33:14.529541 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-57ff957978-f8qz2 I0423 19:33:14.529634 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-57ff957978-f8qz2 I0423 19:33:14.538781 3663 pod.go:62] [openshift-image-registry/image-registry-7fb79b599b-5pdmw] pod update took 23.806123ms I0423 19:33:14.538802 3663 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-7fb79b599b-5pdmw] addLogicalPort annotation time took 23.830725ms I0423 19:33:14.540182 3663 pods.go:271] [openshift-image-registry/image-registry-7fb79b599b-5pdmw] addLogicalPort took 25.286474ms, libovsdb time 1.021972ms I0423 19:33:14.574272 3663 cni.go:473] [openshift-ingress/router-default-dbcccc67f-9st5g 5c4039bbe81ee0843f3c7ca822de83d90426d394ec5f90eb4a282cb84a4303c7 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-dbcccc67f-9st5g 5c4039bbe81ee0843f3c7ca822de83d90426d394ec5f90eb4a282cb84a4303c7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:d9:7b:31:03:75\",\"name\":\"5c4039bbe81ee08\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dd0cffec-b6aa-4c44-a835-0a838b892000\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:33:14.589619 3663 cni.go:473] [openshift-ingress/router-default-5549b9c849-rjng6 fb1d849ae980429eef39933157692601f225dc595a7ed4870f4c09f5c93dc3f7 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-5549b9c849-rjng6 fb1d849ae980429eef39933157692601f225dc595a7ed4870f4c09f5c93dc3f7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:31:2b:83:2b:97\",\"name\":\"fb1d849ae980429\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/33dd7896-b76e-4997-ba97-f0e5f4ee8e12\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:33:14.875027 3663 cni.go:452] [openshift-image-registry/image-registry-7fb79b599b-5pdmw 42dfce7d257765dc0cd15eba4febce81d32f7349718232a6c51686fbe4e680b7 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-7fb79b599b-5pdmw 42dfce7d257765dc0cd15eba4febce81d32f7349718232a6c51686fbe4e680b7 network default NAD default NAD key ] I0423 19:33:14.884725 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7fb79b599b-5pdmw, hostIfaceName: 42dfce7d257765d, network: default, NAD default, SandboxID: "42dfce7d257765dc0cd15eba4febce81d32f7349718232a6c51686fbe4e680b7", PCI device ID: , UID: "de6bb869-b9d3-4727-97a7-4cc40b583d02", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0423 19:33:14.933017 3663 cni.go:473] [openshift-image-registry/image-registry-7fb79b599b-5pdmw 42dfce7d257765dc0cd15eba4febce81d32f7349718232a6c51686fbe4e680b7 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-7fb79b599b-5pdmw 42dfce7d257765dc0cd15eba4febce81d32f7349718232a6c51686fbe4e680b7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:7b:05:4c:c4:ef\",\"name\":\"42dfce7d257765d\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67b28af7-d30d-466e-9661-051ded1787c0\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:33:15.391001 3663 obj_retry.go:498] Detected object openshift-image-registry/image-registry-dbb9d5bb5-nmrgk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:33:15.391695 3663 obj_retry.go:498] Detected object openshift-image-registry/image-registry-dbb9d5bb5-nmrgk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:33:15.406034 3663 obj_retry.go:492] Detected object openshift-image-registry/image-registry-dbb9d5bb5-nmrgk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:33:15.406057 3663 obj_retry.go:492] Detected object openshift-image-registry/image-registry-dbb9d5bb5-nmrgk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:33:15.412158 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-dbb9d5bb5-nmrgk I0423 19:33:15.412178 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-dbb9d5bb5-nmrgk W0423 19:33:15.474210 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:15.474256 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:15.477020 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:15.477038 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:15.477043 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:15.513451 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:15.513479 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:15.513617 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:15.513629 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:15.513636 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:15.540995 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:15.541015 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:15.541020 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:15.542025 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:15.542042 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:15.567910 3663 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:33:16.511869 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:16.511892 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:16.511902 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:16.511906 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:16.511919 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:16.511928 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:16.515166 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:16.515182 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:16.515188 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:16.515203 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:16.515208 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:16.515213 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:16.536355 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:16.536376 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:16.536383 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:33:16.552675 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0423 19:33:16.552702 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0423 19:33:16.636950 3663 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0423 19:33:16.752989 3663 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0423 19:33:17.438195 3663 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0423 19:33:17.638500 3663 base_network_controller_pods.go:487] [default/openshift-console/console-7bc8874b7d-66d84] creating logical port openshift-console_console-7bc8874b7d-66d84 for pod on switch ip-10-0-1-108.ec2.internal I0423 19:33:17.638608 3663 kube.go:256] Updating pod openshift-console/console-7bc8874b7d-66d84 I0423 19:33:17.660647 3663 pod.go:62] [openshift-console/console-7bc8874b7d-66d84] pod update took 22.068946ms I0423 19:33:17.660672 3663 base_network_controller_pods.go:951] [default/openshift-console/console-7bc8874b7d-66d84] addLogicalPort annotation time took 22.101927ms I0423 19:33:17.661835 3663 pods.go:271] [openshift-console/console-7bc8874b7d-66d84] addLogicalPort took 23.403299ms, libovsdb time 833.928µs I0423 19:33:17.716722 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-zp4pm] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-zp4pm for pod on switch ip-10-0-1-108.ec2.internal I0423 19:33:17.716840 3663 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-zp4pm I0423 19:33:17.736697 3663 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-zp4pm] pod update took 19.878094ms I0423 19:33:17.736717 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-zp4pm] addLogicalPort annotation time took 19.902971ms I0423 19:33:17.737723 3663 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-zp4pm] addLogicalPort took 21.015247ms, libovsdb time 725.385µs I0423 19:33:17.987762 3663 cni.go:452] [openshift-console/console-7bc8874b7d-66d84 76aae054e091204b88d262974daa2b7c96cf91e40b754fcc4f6be53981441db2 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7bc8874b7d-66d84 76aae054e091204b88d262974daa2b7c96cf91e40b754fcc4f6be53981441db2 network default NAD default NAD key ] I0423 19:33:17.999066 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7bc8874b7d-66d84, hostIfaceName: 76aae054e091204, network: default, NAD default, SandboxID: "76aae054e091204b88d262974daa2b7c96cf91e40b754fcc4f6be53981441db2", PCI device ID: , UID: "8a734b7f-9e4e-4d69-96e5-3c642e944de9", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0423 19:33:18.049689 3663 cni.go:473] [openshift-console/console-7bc8874b7d-66d84 76aae054e091204b88d262974daa2b7c96cf91e40b754fcc4f6be53981441db2 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7bc8874b7d-66d84 76aae054e091204b88d262974daa2b7c96cf91e40b754fcc4f6be53981441db2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:0b:3d:10:e8:a2\",\"name\":\"76aae054e091204\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/41ee5153-1b8c-4ce6-a587-25286a2f90db\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:33:18.071536 3663 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-zp4pm c957c610b2f698934b005132ae44235d95e6f74967d4526d3c283d27257bb649 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-zp4pm c957c610b2f698934b005132ae44235d95e6f74967d4526d3c283d27257bb649 network default NAD default NAD key ] I0423 19:33:18.083729 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-zp4pm, hostIfaceName: c957c610b2f6989, network: default, NAD default, SandboxID: "c957c610b2f698934b005132ae44235d95e6f74967d4526d3c283d27257bb649", PCI device ID: , UID: "fb791c2e-70f4-4289-b921-5e3c405185d4", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0423 19:33:18.138630 3663 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-zp4pm c957c610b2f698934b005132ae44235d95e6f74967d4526d3c283d27257bb649 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-zp4pm c957c610b2f698934b005132ae44235d95e6f74967d4526d3c283d27257bb649 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:15:8a:12:4f:06\",\"name\":\"c957c610b2f6989\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/925a0fe8-5cfe-4474-9fb6-b59c76f7636f\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:33:20.413638 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0423 19:33:20.555204 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0423 19:33:22.423238 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:33:22.504536 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:33:22.543879 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0423 19:33:27.265772 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-b8587444b-s4gvw] creating logical port openshift-monitoring_openshift-state-metrics-b8587444b-s4gvw for pod on switch ip-10-0-1-108.ec2.internal I0423 19:33:27.265885 3663 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-b8587444b-s4gvw I0423 19:33:27.295126 3663 pod.go:62] [openshift-monitoring/openshift-state-metrics-b8587444b-s4gvw] pod update took 29.273603ms I0423 19:33:27.295155 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/openshift-state-metrics-b8587444b-s4gvw] addLogicalPort annotation time took 29.312883ms I0423 19:33:27.296359 3663 pods.go:271] [openshift-monitoring/openshift-state-metrics-b8587444b-s4gvw] addLogicalPort took 30.608658ms, libovsdb time 884.36µs I0423 19:33:28.227301 3663 cni.go:452] [openshift-monitoring/openshift-state-metrics-b8587444b-s4gvw ecab23cdca37007f0a6dc9d6a55e3f1b1650e38d617bc9731b3f8848021f8f23 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-s4gvw ecab23cdca37007f0a6dc9d6a55e3f1b1650e38d617bc9731b3f8848021f8f23 network default NAD default NAD key ] I0423 19:33:28.237209 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-b8587444b-s4gvw, hostIfaceName: ecab23cdca37007, network: default, NAD default, SandboxID: "ecab23cdca37007f0a6dc9d6a55e3f1b1650e38d617bc9731b3f8848021f8f23", PCI device ID: , UID: "5f53361f-5834-4a3a-a729-08885907ffea", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0423 19:33:28.289597 3663 cni.go:473] [openshift-monitoring/openshift-state-metrics-b8587444b-s4gvw ecab23cdca37007f0a6dc9d6a55e3f1b1650e38d617bc9731b3f8848021f8f23 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-s4gvw ecab23cdca37007f0a6dc9d6a55e3f1b1650e38d617bc9731b3f8848021f8f23 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:68:5f:90:4c:1e\",\"name\":\"ecab23cdca37007\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dcbc1845-c194-431d-a008-c787a7eaea96\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:33:28.474284 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-108.ec2.internal I0423 19:33:28.474435 3663 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0423 19:33:28.515217 3663 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 40.825045ms I0423 19:33:28.515242 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 40.854667ms I0423 19:33:28.516478 3663 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 42.213945ms, libovsdb time 918.577µs I0423 19:33:29.441840 3663 cni.go:452] [openshift-monitoring/alertmanager-main-1 5e86af0d87596e180005c953f90e81747d38130ee5d605f8d11f3b6398c9e548 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 5e86af0d87596e180005c953f90e81747d38130ee5d605f8d11f3b6398c9e548 network default NAD default NAD key ] I0423 19:33:29.451693 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 5e86af0d87596e1, network: default, NAD default, SandboxID: "5e86af0d87596e180005c953f90e81747d38130ee5d605f8d11f3b6398c9e548", PCI device ID: , UID: "692619b9-0e09-46e8-98cf-f82a59a18b0a", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0423 19:33:29.499110 3663 cni.go:473] [openshift-monitoring/alertmanager-main-1 5e86af0d87596e180005c953f90e81747d38130ee5d605f8d11f3b6398c9e548 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 5e86af0d87596e180005c953f90e81747d38130ee5d605f8d11f3b6398c9e548 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:00:d2:5e:b8:60\",\"name\":\"5e86af0d87596e1\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f61e9c88-60be-493d-8209-62d593de3d2f\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:33:31.600569 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:33:31.600593 3663 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:33:31.600600 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0423 19:33:31.805706 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-59f855df4c-kbxkw] creating logical port openshift-monitoring_metrics-server-59f855df4c-kbxkw for pod on switch ip-10-0-1-108.ec2.internal I0423 19:33:31.805806 3663 kube.go:256] Updating pod openshift-monitoring/metrics-server-59f855df4c-kbxkw I0423 19:33:31.827845 3663 pod.go:62] [openshift-monitoring/metrics-server-59f855df4c-kbxkw] pod update took 22.065197ms I0423 19:33:31.827870 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-59f855df4c-kbxkw] addLogicalPort annotation time took 22.096257ms I0423 19:33:31.829006 3663 pods.go:271] [openshift-monitoring/metrics-server-59f855df4c-kbxkw] addLogicalPort took 23.319136ms, libovsdb time 826.598µs W0423 19:33:32.021467 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:33:32.032124 3663 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-6d5b8d6f87-sjnjv IP address from the namespace address-set, err: pod openshift-console/console-6d5b8d6f87-sjnjv: no pod IPs found I0423 19:33:32.165272 3663 cni.go:452] [openshift-monitoring/metrics-server-59f855df4c-kbxkw bd3999df2d4bb9f0aa01490c43c470e681d337225c23300f15cfdccd68d64ad1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-59f855df4c-kbxkw bd3999df2d4bb9f0aa01490c43c470e681d337225c23300f15cfdccd68d64ad1 network default NAD default NAD key ] I0423 19:33:32.174729 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-59f855df4c-kbxkw, hostIfaceName: bd3999df2d4bb9f, network: default, NAD default, SandboxID: "bd3999df2d4bb9f0aa01490c43c470e681d337225c23300f15cfdccd68d64ad1", PCI device ID: , UID: "51126ded-a0f8-4783-a4f0-391d712f9473", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0423 19:33:32.224528 3663 cni.go:473] [openshift-monitoring/metrics-server-59f855df4c-kbxkw bd3999df2d4bb9f0aa01490c43c470e681d337225c23300f15cfdccd68d64ad1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-59f855df4c-kbxkw bd3999df2d4bb9f0aa01490c43c470e681d337225c23300f15cfdccd68d64ad1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:16:3a:75:71:d5\",\"name\":\"bd3999df2d4bb9f\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/363e8186-025a-4552-ac9d-27bfdfd398ed\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:33:33.639390 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-1-108.ec2.internal I0423 19:33:33.639527 3663 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0423 19:33:33.679677 3663 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 40.202114ms I0423 19:33:33.679701 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 40.236634ms I0423 19:33:33.680907 3663 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 41.546607ms, libovsdb time 900.171µs I0423 19:33:34.005226 3663 cni.go:452] [openshift-monitoring/prometheus-k8s-1 ea1c327f11e2bbcec77bde259161b4c3b70b42b898131e00b2e703ffbb04df90 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 ea1c327f11e2bbcec77bde259161b4c3b70b42b898131e00b2e703ffbb04df90 network default NAD default NAD key ] I0423 19:33:34.018121 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: ea1c327f11e2bbc, network: default, NAD default, SandboxID: "ea1c327f11e2bbcec77bde259161b4c3b70b42b898131e00b2e703ffbb04df90", PCI device ID: , UID: "2fb4e22e-eeff-458a-95ce-7aaef598445b", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0423 19:33:34.071482 3663 cni.go:473] [openshift-monitoring/prometheus-k8s-1 ea1c327f11e2bbcec77bde259161b4c3b70b42b898131e00b2e703ffbb04df90 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 ea1c327f11e2bbcec77bde259161b4c3b70b42b898131e00b2e703ffbb04df90 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:01:49:e0:ee:ca\",\"name\":\"ea1c327f11e2bbc\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c3a724df-8286-4eae-81fe-040b8a277366\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:33:34.619205 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:33:34.619230 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:33:34.636902 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 19:33:35.473832 3663 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0423 19:33:36.583086 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:33:36.583112 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:33:37.578004 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0423 19:33:39.688740 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 19:33:39.748257 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:33:39.748282 3663 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:33:39.748290 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:33:39.792297 3663 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-6685b4cb5f-2bjfz IP address from the namespace address-set, err: pod openshift-console/console-6685b4cb5f-2bjfz: no pod IPs found W0423 19:33:39.793712 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:33:40.533557 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:33:40.533580 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:33:40.533587 3663 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:33:40.533593 3663 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:33:40.648044 3663 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:33:40.648068 3663 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:33:40.648091 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:33:40.648095 3663 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier I0423 19:33:46.208921 3663 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0423 19:33:51.585132 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0423 19:33:55.621920 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 19:33:59.508641 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:33:59.508665 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:33:59.508671 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0423 19:34:02.614317 3663 obj_retry.go:498] Detected object openshift-console/console-7bc8874b7d-4llp6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:34:02.614900 3663 obj_retry.go:498] Detected object openshift-console/console-7bc8874b7d-4llp6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0423 19:34:02.622347 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:34:03.626654 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0423 19:34:03.641997 3663 obj_retry.go:492] Detected object openshift-console/console-7bc8874b7d-4llp6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:34:03.642025 3663 obj_retry.go:492] Detected object openshift-console/console-7bc8874b7d-4llp6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:34:03.647771 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7bc8874b7d-4llp6 I0423 19:34:03.647792 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7bc8874b7d-4llp6 W0423 19:34:07.710909 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0423 19:34:12.577337 3663 cni.go:452] [openshift-console/console-7bc8874b7d-66d84 76aae054e091204b88d262974daa2b7c96cf91e40b754fcc4f6be53981441db2 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7bc8874b7d-66d84 76aae054e091204b88d262974daa2b7c96cf91e40b754fcc4f6be53981441db2 network default NAD default NAD key ] I0423 19:34:12.656150 3663 cni.go:473] [openshift-console/console-7bc8874b7d-66d84 76aae054e091204b88d262974daa2b7c96cf91e40b754fcc4f6be53981441db2 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7bc8874b7d-66d84 76aae054e091204b88d262974daa2b7c96cf91e40b754fcc4f6be53981441db2 network default NAD default NAD key default], result "{}", err I0423 19:34:12.701482 3663 obj_retry.go:498] Detected object openshift-console/console-7bc8874b7d-66d84 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:34:12.701515 3663 pods.go:176] Deleting pod: openshift-console/console-7bc8874b7d-66d84 I0423 19:34:12.702939 3663 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7bc8874b7d-66d84, ips: 10.129.0.15 I0423 19:34:12.702968 3663 obj_retry.go:498] Detected object openshift-console/console-7bc8874b7d-66d84 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:34:12.709608 3663 base_network_controller_pods.go:487] [default/openshift-console/console-6457b7df48-bskgw] creating logical port openshift-console_console-6457b7df48-bskgw for pod on switch ip-10-0-1-108.ec2.internal I0423 19:34:12.709748 3663 kube.go:256] Updating pod openshift-console/console-6457b7df48-bskgw W0423 19:34:12.710207 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0423 19:34:12.725230 3663 pod.go:62] [openshift-console/console-6457b7df48-bskgw] pod update took 15.526333ms I0423 19:34:12.725259 3663 base_network_controller_pods.go:951] [default/openshift-console/console-6457b7df48-bskgw] addLogicalPort annotation time took 15.563547ms I0423 19:34:12.726696 3663 pods.go:271] [openshift-console/console-6457b7df48-bskgw] addLogicalPort took 17.100202ms, libovsdb time 1.0193ms I0423 19:34:12.760997 3663 obj_retry.go:492] Detected object openshift-console/console-7bc8874b7d-66d84 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:34:12.761042 3663 obj_retry.go:492] Detected object openshift-console/console-7bc8874b7d-66d84 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:34:12.766847 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7bc8874b7d-66d84 I0423 19:34:12.766869 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7bc8874b7d-66d84 I0423 19:34:13.075740 3663 cni.go:452] [openshift-console/console-6457b7df48-bskgw 703cf74e2ce14a344a712b579e6d444edcf2761ab26663beb9c898396e390c2b network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-6457b7df48-bskgw 703cf74e2ce14a344a712b579e6d444edcf2761ab26663beb9c898396e390c2b network default NAD default NAD key ] I0423 19:34:13.085504 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-6457b7df48-bskgw, hostIfaceName: 703cf74e2ce14a3, network: default, NAD default, SandboxID: "703cf74e2ce14a344a712b579e6d444edcf2761ab26663beb9c898396e390c2b", PCI device ID: , UID: "7d32aefc-0bda-4cd8-ac8c-6b7c6f55b866", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0423 19:34:13.134935 3663 cni.go:473] [openshift-console/console-6457b7df48-bskgw 703cf74e2ce14a344a712b579e6d444edcf2761ab26663beb9c898396e390c2b network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-6457b7df48-bskgw 703cf74e2ce14a344a712b579e6d444edcf2761ab26663beb9c898396e390c2b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:0d:66:7b:8b:e7\",\"name\":\"703cf74e2ce14a3\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/477eec90-a41c-4d1a-ae94-e51da0ec5e66\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:34:13.764274 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:34:17.756518 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0423 19:34:19.989876 3663 obj_retry.go:498] Detected object openshift-console/console-6d5b8d6f87-ctvxq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:34:19.990520 3663 obj_retry.go:498] Detected object openshift-console/console-6d5b8d6f87-ctvxq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0423 19:34:20.002019 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:34:20.777989 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0423 19:34:20.791391 3663 obj_retry.go:492] Detected object openshift-console/console-6d5b8d6f87-ctvxq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:34:20.791412 3663 obj_retry.go:492] Detected object openshift-console/console-6d5b8d6f87-ctvxq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:34:20.800464 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6d5b8d6f87-ctvxq I0423 19:34:20.800486 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6d5b8d6f87-ctvxq I0423 19:34:26.038141 3663 cni.go:452] [openshift-multus/network-metrics-daemon-s6lhr 7ac5f03e261bc853ec5c2903b2f98380b09ec1a78d9097fcfc961e0a103630ad network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-s6lhr 7ac5f03e261bc853ec5c2903b2f98380b09ec1a78d9097fcfc961e0a103630ad network default NAD default NAD key ] I0423 19:34:26.047754 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-s6lhr, hostIfaceName: 7ac5f03e261bc85, network: default, NAD default, SandboxID: "7ac5f03e261bc853ec5c2903b2f98380b09ec1a78d9097fcfc961e0a103630ad", PCI device ID: , UID: "2668a993-b63b-443f-bf7a-402ef2da8a55", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0423 19:34:26.098473 3663 cni.go:473] [openshift-multus/network-metrics-daemon-s6lhr 7ac5f03e261bc853ec5c2903b2f98380b09ec1a78d9097fcfc961e0a103630ad network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-s6lhr 7ac5f03e261bc853ec5c2903b2f98380b09ec1a78d9097fcfc961e0a103630ad network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:2a:9c:20:a7:50\",\"name\":\"7ac5f03e261bc85\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/99cc0701-602d-4261-bd58-48857e490f39\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:34:32.218259 3663 cni.go:452] [openshift-ingress/router-default-dbcccc67f-9st5g 5c4039bbe81ee0843f3c7ca822de83d90426d394ec5f90eb4a282cb84a4303c7 network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-dbcccc67f-9st5g 5c4039bbe81ee0843f3c7ca822de83d90426d394ec5f90eb4a282cb84a4303c7 network default NAD default NAD key ] I0423 19:34:32.316758 3663 cni.go:473] [openshift-ingress/router-default-dbcccc67f-9st5g 5c4039bbe81ee0843f3c7ca822de83d90426d394ec5f90eb4a282cb84a4303c7 network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-dbcccc67f-9st5g 5c4039bbe81ee0843f3c7ca822de83d90426d394ec5f90eb4a282cb84a4303c7 network default NAD default NAD key default], result "{}", err I0423 19:34:32.371373 3663 obj_retry.go:498] Detected object openshift-ingress/router-default-dbcccc67f-9st5g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:34:32.371421 3663 pods.go:176] Deleting pod: openshift-ingress/router-default-dbcccc67f-9st5g I0423 19:34:32.373264 3663 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-dbcccc67f-9st5g, ips: 10.129.0.11 I0423 19:34:32.373296 3663 obj_retry.go:498] Detected object openshift-ingress/router-default-dbcccc67f-9st5g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:34:32.838008 3663 obj_retry.go:492] Detected object openshift-ingress/router-default-dbcccc67f-9st5g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:34:32.838032 3663 obj_retry.go:492] Detected object openshift-ingress/router-default-dbcccc67f-9st5g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:34:32.841455 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-dbcccc67f-9st5g I0423 19:34:32.841479 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-dbcccc67f-9st5g W0423 19:34:34.882674 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:34:34.882695 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:34:34.882706 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:34:34.882710 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:34:34.896542 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:34:34.896562 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:34:47.266749 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:34:47.293268 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0423 19:34:49.198128 3663 cni.go:452] [openshift-monitoring/alertmanager-main-1 5e86af0d87596e180005c953f90e81747d38130ee5d605f8d11f3b6398c9e548 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 5e86af0d87596e180005c953f90e81747d38130ee5d605f8d11f3b6398c9e548 network default NAD default NAD key ] I0423 19:34:49.269619 3663 cni.go:473] [openshift-monitoring/alertmanager-main-1 5e86af0d87596e180005c953f90e81747d38130ee5d605f8d11f3b6398c9e548 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 5e86af0d87596e180005c953f90e81747d38130ee5d605f8d11f3b6398c9e548 network default NAD default NAD key default], result "{}", err I0423 19:34:49.326742 3663 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:34:49.326778 3663 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0423 19:34:49.328141 3663 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.129.0.18 I0423 19:34:49.328179 3663 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:34:49.914621 3663 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:34:49.914650 3663 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:34:49.922637 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0423 19:34:49.922675 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0423 19:34:49.960736 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-108.ec2.internal I0423 19:34:49.960855 3663 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0423 19:34:49.984848 3663 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 24.031405ms I0423 19:34:49.984876 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 24.065065ms I0423 19:34:49.986171 3663 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 25.453665ms, libovsdb time 949.79µs I0423 19:34:50.320552 3663 cni.go:452] [openshift-monitoring/alertmanager-main-1 05c08dcafd89ae618830bb4efdad575d6de8ee248dce0ec2357428f1c4c2042f network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 05c08dcafd89ae618830bb4efdad575d6de8ee248dce0ec2357428f1c4c2042f network default NAD default NAD key ] I0423 19:34:50.331270 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 05c08dcafd89ae6, network: default, NAD default, SandboxID: "05c08dcafd89ae618830bb4efdad575d6de8ee248dce0ec2357428f1c4c2042f", PCI device ID: , UID: "c3e0816c-ae6d-4526-b551-54b87f5a905f", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0423 19:34:50.381258 3663 cni.go:473] [openshift-monitoring/alertmanager-main-1 05c08dcafd89ae618830bb4efdad575d6de8ee248dce0ec2357428f1c4c2042f network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 05c08dcafd89ae618830bb4efdad575d6de8ee248dce0ec2357428f1c4c2042f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:12:d6:fb:30:2c\",\"name\":\"05c08dcafd89ae6\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ebc1ba6-d084-4ae7-801f-d72accc2fdd2\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:34:51.399970 3663 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0423 19:34:53.506219 3663 cni.go:452] [openshift-monitoring/prometheus-k8s-1 ea1c327f11e2bbcec77bde259161b4c3b70b42b898131e00b2e703ffbb04df90 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 ea1c327f11e2bbcec77bde259161b4c3b70b42b898131e00b2e703ffbb04df90 network default NAD default NAD key ] I0423 19:34:53.590943 3663 cni.go:473] [openshift-monitoring/prometheus-k8s-1 ea1c327f11e2bbcec77bde259161b4c3b70b42b898131e00b2e703ffbb04df90 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 ea1c327f11e2bbcec77bde259161b4c3b70b42b898131e00b2e703ffbb04df90 network default NAD default NAD key default], result "{}", err I0423 19:34:53.662264 3663 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:34:53.662400 3663 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0423 19:34:53.663715 3663 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.129.0.20 I0423 19:34:53.663745 3663 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:34:53.947952 3663 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:34:53.947986 3663 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:34:53.954621 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0423 19:34:53.954659 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0423 19:34:53.957325 3663 cni.go:452] [openshift-dns/dns-default-wghf4 10de4ae231df106338f9a4c1a4f02afa228903c0efa95406cf16ac70ae43f602 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-wghf4 10de4ae231df106338f9a4c1a4f02afa228903c0efa95406cf16ac70ae43f602 network default NAD default NAD key ] I0423 19:34:53.967675 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-wghf4, hostIfaceName: 10de4ae231df106, network: default, NAD default, SandboxID: "10de4ae231df106338f9a4c1a4f02afa228903c0efa95406cf16ac70ae43f602", PCI device ID: , UID: "59f69824-ef4f-4ee2-a2b8-f579aa0a41fc", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0423 19:34:53.994967 3663 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-1-108.ec2.internal I0423 19:34:53.995129 3663 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0423 19:34:54.022418 3663 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 27.354435ms I0423 19:34:54.022440 3663 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 27.380358ms I0423 19:34:54.023051 3663 cni.go:473] [openshift-dns/dns-default-wghf4 10de4ae231df106338f9a4c1a4f02afa228903c0efa95406cf16ac70ae43f602 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-wghf4 10de4ae231df106338f9a4c1a4f02afa228903c0efa95406cf16ac70ae43f602 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:ed:6d:58:de:cf\",\"name\":\"10de4ae231df106\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/94b9a743-f923-4fee-984b-d40c7f5e159b\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:34:54.023547 3663 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 28.598277ms, libovsdb time 790.712µs I0423 19:34:54.356969 3663 cni.go:452] [openshift-monitoring/prometheus-k8s-1 d34e20c1eec0760ca483cc80f0332495e07785a9fc9f137c115cfd8df842b8e1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 d34e20c1eec0760ca483cc80f0332495e07785a9fc9f137c115cfd8df842b8e1 network default NAD default NAD key ] I0423 19:34:54.367042 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: d34e20c1eec0760, network: default, NAD default, SandboxID: "d34e20c1eec0760ca483cc80f0332495e07785a9fc9f137c115cfd8df842b8e1", PCI device ID: , UID: "3a7eaaed-2850-456a-aab6-8f244530d7f3", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0423 19:34:54.416777 3663 cni.go:473] [openshift-monitoring/prometheus-k8s-1 d34e20c1eec0760ca483cc80f0332495e07785a9fc9f137c115cfd8df842b8e1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 d34e20c1eec0760ca483cc80f0332495e07785a9fc9f137c115cfd8df842b8e1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:d5:d6:15:22:71\",\"name\":\"d34e20c1eec0760\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/84a5f74c-9955-419a-980a-9e9cec28ecf2\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:34:55.933915 3663 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:34:56.935750 3663 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0423 19:34:56.935770 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0423 19:34:56.935775 3663 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0423 19:34:56.968170 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0423 19:34:56.968190 3663 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0423 19:34:56.968198 3663 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0423 19:34:59.859583 3663 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0423 19:34:59.859604 3663 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0423 19:34:59.859609 3663 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0423 19:35:07.013994 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0423 19:35:07.014016 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0423 19:35:07.014026 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0423 19:35:10.381787 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:35:10.381808 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:35:10.381814 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0423 19:35:11.677121 3663 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:35:11.677747 3663 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:35:11.911517 3663 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:35:11.911540 3663 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:35:11.919272 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0423 19:35:11.919296 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0423 19:35:12.454742 3663 cni.go:452] [openshift-console/console-6457b7df48-bskgw 703cf74e2ce14a344a712b579e6d444edcf2761ab26663beb9c898396e390c2b network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-6457b7df48-bskgw 703cf74e2ce14a344a712b579e6d444edcf2761ab26663beb9c898396e390c2b network default NAD default NAD key ] I0423 19:35:12.526493 3663 cni.go:473] [openshift-console/console-6457b7df48-bskgw 703cf74e2ce14a344a712b579e6d444edcf2761ab26663beb9c898396e390c2b network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-6457b7df48-bskgw 703cf74e2ce14a344a712b579e6d444edcf2761ab26663beb9c898396e390c2b network default NAD default NAD key default], result "{}", err I0423 19:35:12.558532 3663 obj_retry.go:498] Detected object openshift-console/console-6685b4cb5f-72bt8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:35:12.559140 3663 obj_retry.go:498] Detected object openshift-console/console-6685b4cb5f-72bt8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0423 19:35:12.567701 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0423 19:35:12.575279 3663 obj_retry.go:498] Detected object openshift-console/console-6457b7df48-bskgw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:35:12.575311 3663 pods.go:176] Deleting pod: openshift-console/console-6457b7df48-bskgw I0423 19:35:12.576472 3663 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6457b7df48-bskgw, ips: 10.129.0.21 I0423 19:35:12.576498 3663 obj_retry.go:498] Detected object openshift-console/console-6457b7df48-bskgw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:35:12.586286 3663 base_network_controller_pods.go:487] [default/openshift-console/console-7bf7d4bd85-ztcsv] creating logical port openshift-console_console-7bf7d4bd85-ztcsv for pod on switch ip-10-0-1-108.ec2.internal I0423 19:35:12.586411 3663 kube.go:256] Updating pod openshift-console/console-7bf7d4bd85-ztcsv W0423 19:35:12.586420 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0423 19:35:12.603013 3663 pod.go:62] [openshift-console/console-7bf7d4bd85-ztcsv] pod update took 16.639477ms I0423 19:35:12.603037 3663 base_network_controller_pods.go:951] [default/openshift-console/console-7bf7d4bd85-ztcsv] addLogicalPort annotation time took 16.667895ms I0423 19:35:12.604406 3663 pods.go:271] [openshift-console/console-7bf7d4bd85-ztcsv] addLogicalPort took 18.132994ms, libovsdb time 1.011581ms I0423 19:35:12.911035 3663 obj_retry.go:492] Detected object openshift-console/console-6685b4cb5f-72bt8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:35:12.911061 3663 obj_retry.go:492] Detected object openshift-console/console-6685b4cb5f-72bt8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:35:12.916782 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6685b4cb5f-72bt8 I0423 19:35:12.916809 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6685b4cb5f-72bt8 I0423 19:35:12.945152 3663 cni.go:452] [openshift-console/console-7bf7d4bd85-ztcsv aaf3b746bdaa3a1ecef51c557a13338e4de51be427b67f7d111695c7ac654640 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7bf7d4bd85-ztcsv aaf3b746bdaa3a1ecef51c557a13338e4de51be427b67f7d111695c7ac654640 network default NAD default NAD key ] I0423 19:35:12.954683 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7bf7d4bd85-ztcsv, hostIfaceName: aaf3b746bdaa3a1, network: default, NAD default, SandboxID: "aaf3b746bdaa3a1ecef51c557a13338e4de51be427b67f7d111695c7ac654640", PCI device ID: , UID: "e6aecf67-8177-44bf-b796-03b95d163f44", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0423 19:35:13.004280 3663 cni.go:473] [openshift-console/console-7bf7d4bd85-ztcsv aaf3b746bdaa3a1ecef51c557a13338e4de51be427b67f7d111695c7ac654640 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7bf7d4bd85-ztcsv aaf3b746bdaa3a1ecef51c557a13338e4de51be427b67f7d111695c7ac654640 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:30:1e:ad:e6:09\",\"name\":\"aaf3b746bdaa3a1\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/95a75314-3084-4cfe-b4eb-057dafa17a72\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:35:13.041949 3663 obj_retry.go:492] Detected object openshift-console/console-6457b7df48-bskgw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:35:13.041970 3663 obj_retry.go:492] Detected object openshift-console/console-6457b7df48-bskgw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:35:13.047886 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6457b7df48-bskgw I0423 19:35:13.047914 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6457b7df48-bskgw W0423 19:35:13.968420 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:35:14.040163 3663 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:35:14.874569 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:20.411632 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:35:20.411655 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:35:21.076117 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:21.076148 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:21.076179 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:21.975385 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:21.975434 3663 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:21.975437 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:21.975575 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:22.991225 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:35:22.991246 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:35:23.028193 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:35:23.028219 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:35:23.087571 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:35:24.945205 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:24.945231 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0423 19:35:41.197404 3663 obj_retry.go:498] Detected object openshift-image-registry/image-registry-57ff957978-zgs2j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:35:41.197932 3663 obj_retry.go:498] Detected object openshift-image-registry/image-registry-57ff957978-zgs2j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0423 19:35:41.211105 3663 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:41.211153 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:41.211186 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:41.211200 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:35:42.086009 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0423 19:35:42.118061 3663 obj_retry.go:492] Detected object openshift-image-registry/image-registry-57ff957978-zgs2j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:35:42.118102 3663 obj_retry.go:492] Detected object openshift-image-registry/image-registry-57ff957978-zgs2j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:35:42.125661 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-57ff957978-zgs2j I0423 19:35:42.125682 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-57ff957978-zgs2j I0423 19:35:48.314490 3663 obj_retry.go:498] Detected object openshift-console/console-6457b7df48-l8627 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:35:48.315047 3663 obj_retry.go:498] Detected object openshift-console/console-6457b7df48-l8627 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:35:49.147842 3663 obj_retry.go:492] Detected object openshift-console/console-6457b7df48-l8627 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:35:49.147863 3663 obj_retry.go:492] Detected object openshift-console/console-6457b7df48-l8627 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:35:49.154428 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6457b7df48-l8627 I0423 19:35:49.154450 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6457b7df48-l8627 W0423 19:35:55.239215 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:35:55.239235 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0423 19:35:56.539435 3663 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:35:56.539971 3663 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:35:57.197914 3663 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:35:57.197941 3663 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:35:57.204373 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0423 19:35:57.204401 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0423 19:36:09.927002 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0423 19:36:23.310577 3663 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz bcfd27bee01b0cb9e9d3a79dcfc2790aea4ac48f29f15c6fe7d03500627ff5d6 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz bcfd27bee01b0cb9e9d3a79dcfc2790aea4ac48f29f15c6fe7d03500627ff5d6 network default NAD default NAD key ] I0423 19:36:23.392882 3663 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz bcfd27bee01b0cb9e9d3a79dcfc2790aea4ac48f29f15c6fe7d03500627ff5d6 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz bcfd27bee01b0cb9e9d3a79dcfc2790aea4ac48f29f15c6fe7d03500627ff5d6 network default NAD default NAD key default], result "{}", err I0423 19:36:23.434746 3663 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:36:23.434782 3663 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz I0423 19:36:23.436052 3663 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz, ips: 10.129.0.9 I0423 19:36:23.436093 3663 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:36:24.455176 3663 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:36:24.455201 3663 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616210-gpczz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:37:12.011234 3663 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0423 19:37:49.925838 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=70 I0423 19:38:04.049160 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0423 19:38:27.037537 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0423 19:38:49.925769 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=147 I0423 19:38:51.016915 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0423 19:39:06.926669 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=16 I0423 19:39:19.627533 3663 cni.go:452] [openshift-console/console-7bf7d4bd85-ztcsv aaf3b746bdaa3a1ecef51c557a13338e4de51be427b67f7d111695c7ac654640 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7bf7d4bd85-ztcsv aaf3b746bdaa3a1ecef51c557a13338e4de51be427b67f7d111695c7ac654640 network default NAD default NAD key ] I0423 19:39:19.713026 3663 cni.go:473] [openshift-console/console-7bf7d4bd85-ztcsv aaf3b746bdaa3a1ecef51c557a13338e4de51be427b67f7d111695c7ac654640 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7bf7d4bd85-ztcsv aaf3b746bdaa3a1ecef51c557a13338e4de51be427b67f7d111695c7ac654640 network default NAD default NAD key default], result "{}", err I0423 19:39:19.760408 3663 obj_retry.go:498] Detected object openshift-console/console-7bf7d4bd85-ztcsv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:39:19.760480 3663 pods.go:176] Deleting pod: openshift-console/console-7bf7d4bd85-ztcsv I0423 19:39:19.762003 3663 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7bf7d4bd85-ztcsv, ips: 10.129.0.24 I0423 19:39:19.762034 3663 obj_retry.go:498] Detected object openshift-console/console-7bf7d4bd85-ztcsv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:39:19.768406 3663 base_network_controller_pods.go:487] [default/openshift-console/console-75fd4676b7-lvx7w] creating logical port openshift-console_console-75fd4676b7-lvx7w for pod on switch ip-10-0-1-108.ec2.internal I0423 19:39:19.768541 3663 kube.go:256] Updating pod openshift-console/console-75fd4676b7-lvx7w I0423 19:39:19.783128 3663 pod.go:62] [openshift-console/console-75fd4676b7-lvx7w] pod update took 14.610886ms I0423 19:39:19.783156 3663 base_network_controller_pods.go:951] [default/openshift-console/console-75fd4676b7-lvx7w] addLogicalPort annotation time took 14.643195ms I0423 19:39:19.784432 3663 pods.go:271] [openshift-console/console-75fd4676b7-lvx7w] addLogicalPort took 16.041526ms, libovsdb time 880.146µs I0423 19:39:19.977794 3663 obj_retry.go:492] Detected object openshift-console/console-7bf7d4bd85-ztcsv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:39:19.977821 3663 obj_retry.go:492] Detected object openshift-console/console-7bf7d4bd85-ztcsv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:39:19.985164 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7bf7d4bd85-ztcsv I0423 19:39:19.985197 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7bf7d4bd85-ztcsv I0423 19:39:20.127410 3663 cni.go:452] [openshift-console/console-75fd4676b7-lvx7w 226851073d18c6db837ad7ef9649150cd78d27c09d2c6e79acd33b0918af8968 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-75fd4676b7-lvx7w 226851073d18c6db837ad7ef9649150cd78d27c09d2c6e79acd33b0918af8968 network default NAD default NAD key ] I0423 19:39:20.137634 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-75fd4676b7-lvx7w, hostIfaceName: 226851073d18c6d, network: default, NAD default, SandboxID: "226851073d18c6db837ad7ef9649150cd78d27c09d2c6e79acd33b0918af8968", PCI device ID: , UID: "85410bec-1b7d-4e5d-b66e-b9b324463ccc", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0423 19:39:20.188232 3663 cni.go:473] [openshift-console/console-75fd4676b7-lvx7w 226851073d18c6db837ad7ef9649150cd78d27c09d2c6e79acd33b0918af8968 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-75fd4676b7-lvx7w 226851073d18c6db837ad7ef9649150cd78d27c09d2c6e79acd33b0918af8968 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:20:91:c1:7d:06\",\"name\":\"226851073d18c6d\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/253ff134-cdf1-4b10-9c95-33d7a5cd2fd6\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:39:22.926215 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=72 I0423 19:39:30.268352 3663 obj_retry.go:498] Detected object openshift-console/console-7bf7d4bd85-xmpk6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:39:30.268950 3663 obj_retry.go:498] Detected object openshift-console/console-7bf7d4bd85-xmpk6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:39:30.962241 3663 obj_retry.go:492] Detected object openshift-console/console-7bf7d4bd85-xmpk6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:39:30.962270 3663 obj_retry.go:492] Detected object openshift-console/console-7bf7d4bd85-xmpk6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:39:30.968181 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7bf7d4bd85-xmpk6 I0423 19:39:30.968207 3663 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7bf7d4bd85-xmpk6 W0423 19:39:31.034953 3663 util.go:959] Failed to build node endpoints for node ip-10-0-1-108.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0423 19:39:46.011163 3663 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=11 I0423 19:40:00.178791 3663 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29616220-dpssk for pod on switch ip-10-0-1-108.ec2.internal I0423 19:40:00.178932 3663 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk I0423 19:40:00.193516 3663 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk] pod update took 14.611406ms I0423 19:40:00.193543 3663 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk] addLogicalPort annotation time took 14.644483ms I0423 19:40:00.194700 3663 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk] addLogicalPort took 15.932269ms, libovsdb time 842.72µs I0423 19:40:00.557634 3663 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk 7f00132e86d12240586bf4d3440c5904add7925ee7e848c1f2af74d4bcb3fcd3 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk 7f00132e86d12240586bf4d3440c5904add7925ee7e848c1f2af74d4bcb3fcd3 network default NAD default NAD key ] I0423 19:40:00.567230 3663 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29616220-dpssk, hostIfaceName: 7f00132e86d1224, network: default, NAD default, SandboxID: "7f00132e86d12240586bf4d3440c5904add7925ee7e848c1f2af74d4bcb3fcd3", PCI device ID: , UID: "043a0ba2-2daf-4835-8987-b8554698eee1", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0423 19:40:00.618247 3663 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk 7f00132e86d12240586bf4d3440c5904add7925ee7e848c1f2af74d4bcb3fcd3 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk 7f00132e86d12240586bf4d3440c5904add7925ee7e848c1f2af74d4bcb3fcd3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:17:65:ae:14:68\",\"name\":\"7f00132e86d1224\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0407fa14-4f2f-4266-bd4c-11182eeab4fd\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:40:02.043966 3663 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0423 19:40:04.158056 3663 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk 7f00132e86d12240586bf4d3440c5904add7925ee7e848c1f2af74d4bcb3fcd3 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk 7f00132e86d12240586bf4d3440c5904add7925ee7e848c1f2af74d4bcb3fcd3 network default NAD default NAD key ] I0423 19:40:04.236300 3663 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk 7f00132e86d12240586bf4d3440c5904add7925ee7e848c1f2af74d4bcb3fcd3 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk 7f00132e86d12240586bf4d3440c5904add7925ee7e848c1f2af74d4bcb3fcd3 network default NAD default NAD key default], result "{}", err I0423 19:40:04.277125 3663 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:40:04.277164 3663 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk I0423 19:40:04.278529 3663 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk, ips: 10.129.0.26 I0423 19:40:04.278552 3663 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:40:05.021930 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=11 I0423 19:40:05.169837 3663 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:40:05.169862 3663 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616220-dpssk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:40:08.031757 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0423 19:40:21.026853 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0423 19:40:34.048948 3663 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11 I0423 19:40:35.937326 3663 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=654 I0423 19:40:59.852267 3663 namespace.go:98] [tsf] adding namespace I0423 19:40:59.853174 3663 namespace.go:102] [tsf] adding namespace took 881.071µs I0423 19:40:59.872527 3663 namespace.go:142] [tsf] updating namespace I0423 19:40:59.880035 3663 namespace.go:142] [tsf] updating namespace