+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-119.ec2.internal ++ [[ -n ip-10-0-1-119.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-119.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0507 15:31:00.313296659 - disable conntrack on geneve port' I0507 15:31:00.313296659 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0507 15:31:00.356485479 - starting ovnkube-node + echo 'I0507 15:31:00.356485479 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-119.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-119.ec2.internal --init-node ip-10-0-1-119.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-119.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0507 15:31:00.411872 3779 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0507 15:31:00.411937 3779 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-51a02430fb.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0507 15:31:00.413598 3779 kube.go:431] Waiting for certificate I0507 15:31:00.413629 3779 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0507 15:31:00.413659 3779 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0507 15:31:00.414365 3779 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0507 15:31:00.453730 3779 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0507 15:31:00.453754 3779 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0507 15:31:00.457100 3779 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0507 15:31:00.457210 3779 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-hf2hl" I0507 15:31:00.464281 3779 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-hf2hl" I0507 15:31:00.464366 3779 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0507 15:31:01.414008 3779 kube.go:438] Certificate found I0507 15:31:01.414723 3779 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0507 15:31:01.415762 3779 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0507 15:31:01.481034 3779 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0507 15:31:01.484485 3779 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-05-08 15:26:00 +0000 UTC" deadline="2026-05-08 08:30:55.195929208 +0000 UTC" I0507 15:31:01.484950 3779 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="16h59m53.711138174s" I0507 15:31:01.503432 3779 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0507 15:31:01.503554 3779 server.go:101] MetricServer registers OVS metrics I0507 15:31:01.503860 3779 server.go:105] MetricServer registers OVN DB metrics I0507 15:31:01.504577 3779 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0507 15:31:01.504633 3779 factory.go:561] Starting watch factory I0507 15:31:01.504703 3779 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.504717 3779 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.504730 3779 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.504750 3779 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.504765 3779 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.504793 3779 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.505313 3779 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.505497 3779 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.506793 3779 controller_manager.go:370] Starting the ovnkube controller I0507 15:31:01.506806 3779 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-119.ec2.internal I0507 15:31:01.506879 3779 controller_manager.go:395] NBDB zone sync took: 65.343µs I0507 15:31:01.506893 3779 factory.go:561] Starting watch factory I0507 15:31:01.508398 3779 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.508418 3779 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.508784 3779 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0507 15:31:01.508805 3779 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0507 15:31:01.509534 3779 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.509622 3779 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.522770 3779 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0507 15:31:01.522795 3779 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0507 15:31:01.525821 3779 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.526034 3779 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.526807 3779 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0507 15:31:01.526836 3779 server.go:109] MetricServer registers OVN Controller metrics I0507 15:31:01.527922 3779 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.528734 3779 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.529211 3779 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.531172 3779 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0507 15:31:01.534384 3779 server.go:113] MetricServer registers OVN Northd metrics I0507 15:31:01.538920 3779 metrics.go:519] OVN Metrics Server starts to run ... I0507 15:31:01.610787 3779 factory.go:1977] *v1.Pod informer cache synced successfully I0507 15:31:01.610809 3779 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0507 15:31:01.610813 3779 factory.go:1977] *v1.Namespace informer cache synced successfully I0507 15:31:01.610836 3779 factory.go:1977] *v1.Node informer cache synced successfully I0507 15:31:01.610839 3779 factory.go:1977] *v1.Service informer cache synced successfully I0507 15:31:01.610842 3779 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0507 15:31:01.610948 3779 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0507 15:31:01.610972 3779 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0507 15:31:01.611027 3779 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0507 15:31:01.611049 3779 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0507 15:31:01.612994 3779 factory.go:1977] *v1.Service informer cache synced successfully I0507 15:31:01.613009 3779 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0507 15:31:01.613013 3779 factory.go:1977] *v1.Pod informer cache synced successfully I0507 15:31:01.613016 3779 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0507 15:31:01.613019 3779 factory.go:1977] *v1.Namespace informer cache synced successfully I0507 15:31:01.613022 3779 factory.go:1977] *v1.Node informer cache synced successfully I0507 15:31:01.614586 3779 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0507 15:31:01.614602 3779 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0507 15:31:01.616355 3779 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0507 15:31:01.616364 3779 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0507 15:31:01.616412 3779 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.616423 3779 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.618173 3779 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0507 15:31:01.618184 3779 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0507 15:31:01.619233 3779 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.622059 3779 factory.go:1977] *v1.EgressIP informer cache synced successfully I0507 15:31:01.622144 3779 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.622165 3779 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.623475 3779 factory.go:1977] *v1.EgressIP informer cache synced successfully I0507 15:31:01.624854 3779 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.627320 3779 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0507 15:31:01.627423 3779 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.627444 3779 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.628758 3779 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0507 15:31:01.630139 3779 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.632566 3779 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0507 15:31:01.632628 3779 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.632640 3779 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.633977 3779 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0507 15:31:01.635166 3779 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.638576 3779 factory.go:1977] *v1.EgressService informer cache synced successfully I0507 15:31:01.638631 3779 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.638643 3779 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.639728 3779 factory.go:1977] *v1.EgressService informer cache synced successfully I0507 15:31:01.641293 3779 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.643649 3779 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0507 15:31:01.643812 3779 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0507 15:31:01.643828 3779 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0507 15:31:01.645075 3779 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0507 15:31:01.646615 3779 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0507 15:31:01.648971 3779 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0507 15:31:01.649037 3779 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.649055 3779 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.649082 3779 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.649098 3779 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.650356 3779 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0507 15:31:01.651921 3779 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.651945 3779 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0507 15:31:01.654429 3779 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0507 15:31:01.654441 3779 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0507 15:31:01.654447 3779 factory.go:680] Watch Factory start up complete, took: 149.817682ms I0507 15:31:01.654525 3779 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0507 15:31:01.656075 3779 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0507 15:31:01.656085 3779 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0507 15:31:01.656094 3779 factory.go:680] Watch Factory start up complete, took: 149.20008ms I0507 15:31:01.656103 3779 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-1-119.ec2.internal" zone I0507 15:31:01.703175 3779 default_node_network_controller.go:710] Initializing the default node network controller I0507 15:31:01.794751 3779 udn_isolation.go:101] Starting UDN host isolation manager I0507 15:31:01.800828 3779 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0507 15:31:01.850807 3779 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0507 15:31:01.850902 3779 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0507 15:31:01.850924 3779 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0507 15:31:01.885243 3779 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0507 15:31:01.885308 3779 default_node_network_controller.go:809] Node ip-10-0-1-119.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0507 15:31:01.891176 3779 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.119"] k8s.ovn.org/zone-name:ip-10-0-1-119.ec2.internal] on node ip-10-0-1-119.ec2.internal I0507 15:31:01.913850 3779 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0507 15:31:01.920215 3779 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0507 15:31:01.920432 3779 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0507 15:31:01.920628 3779 gateway_init.go:263] Preparing Gateway I0507 15:31:01.920643 3779 gateway_shared_intf.go:1687] Creating new gateway I0507 15:31:02.034352 3779 iptables.go:108] Creating table: filter chain: FORWARD I0507 15:31:02.041203 3779 iptables.go:108] Creating table: filter chain: OUTPUT I0507 15:31:02.048328 3779 gateway_shared_intf.go:1816] Gateway Creation Complete I0507 15:31:02.048359 3779 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-119.ec2.internal","mac-address":"0a:ff:d1:c8:bc:d5","ip-addresses":["10.0.1.119/24"],"ip-address":"10.0.1.119/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:b2baa5fb-4cf3-43e9-a681-16aa7a77ba28 k8s.ovn.org/node-encap-ips:["10.0.1.119"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.119/24"} k8s.ovn.org/zone-name:ip-10-0-1-119.ec2.internal] on node ip-10-0-1-119.ec2.internal I0507 15:31:02.068297 3779 default_node_network_controller.go:519] Node connection status = connected I0507 15:31:02.157163 3779 controller_manager.go:429] Waiting for node in zone sync took: 501.043667ms I0507 15:31:02.162710 3779 controller_manager.go:611] Switching to transit router for layer2 networks I0507 15:31:02.162725 3779 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-119.ec2.internal I0507 15:31:02.179301 3779 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0507 15:31:02.186566 3779 controller_manager.go:290] SCTP support detected in OVN I0507 15:31:02.187193 3779 services_controller.go:69] Creating services controller for network=default I0507 15:31:02.187296 3779 udn_enabled_svc.go:80] Waiting for service informer to sync I0507 15:31:02.187508 3779 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0507 15:31:02.187590 3779 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0507 15:31:02.187610 3779 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0507 15:31:02.187620 3779 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0507 15:31:02.187864 3779 udn_enabled_svc.go:95] Performing full resync I0507 15:31:02.187932 3779 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0507 15:31:02.188012 3779 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0507 15:31:02.188065 3779 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 73.816µs I0507 15:31:02.188087 3779 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 64.816µs I0507 15:31:02.188093 3779 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 82.449µs I0507 15:31:02.188100 3779 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0507 15:31:02.188106 3779 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0507 15:31:02.188114 3779 default_network_controller.go:325] Starting the default network controller I0507 15:31:02.188171 3779 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 10.129µs I0507 15:31:02.188193 3779 udn_enabled_svc.go:99] Waiting for handler to sync I0507 15:31:02.188209 3779 udn_enabled_svc.go:104] Starting worker I0507 15:31:02.198350 3779 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0507 15:31:02.198405 3779 repair.go:33] Syncing exgw routes took 41.332µs I0507 15:31:02.198429 3779 default_network_controller.go:414] Starting all the Watchers... I0507 15:31:02.198604 3779 namespace.go:98] [openshift-insights] adding namespace I0507 15:31:02.198618 3779 namespace.go:98] [openshift-monitoring] adding namespace I0507 15:31:02.198625 3779 namespace.go:98] [dedicated-admin] adding namespace I0507 15:31:02.198633 3779 namespace.go:98] [openshift-authentication-operator] adding namespace I0507 15:31:02.198636 3779 namespace.go:98] [openshift-backplane-cee] adding namespace I0507 15:31:02.198643 3779 namespace.go:98] [openshift-machine-config-operator] adding namespace I0507 15:31:02.198644 3779 namespace.go:98] [openshift-customer-monitoring] adding namespace I0507 15:31:02.198657 3779 namespace.go:98] [openshift-authentication] adding namespace I0507 15:31:02.198664 3779 namespace.go:98] [openshift-backplane] adding namespace I0507 15:31:02.198664 3779 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0507 15:31:02.198661 3779 namespace.go:98] [openshift-host-network] adding namespace I0507 15:31:02.198670 3779 namespace.go:98] [openshift-dns] adding namespace I0507 15:31:02.198671 3779 namespace.go:98] [openshift-network-diagnostics] adding namespace I0507 15:31:02.198607 3779 namespace.go:98] [openshift-backplane-csm] adding namespace I0507 15:31:02.198752 3779 namespace.go:98] [openshift-dns-operator] adding namespace I0507 15:31:02.199239 3779 namespace.go:102] [openshift-insights] adding namespace took 619.865µs I0507 15:31:02.199255 3779 namespace.go:98] [openshift-backplane-tam] adding namespace I0507 15:31:02.199709 3779 namespace.go:102] [openshift-backplane-tam] adding namespace took 448.465µs I0507 15:31:02.199728 3779 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0507 15:31:02.200222 3779 namespace.go:102] [openshift-monitoring] adding namespace took 1.594786ms I0507 15:31:02.200234 3779 namespace.go:98] [openshift-backplane-cse] adding namespace I0507 15:31:02.200815 3779 namespace.go:102] [openshift-backplane-cse] adding namespace took 573.507µs I0507 15:31:02.200831 3779 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0507 15:31:02.201329 3779 namespace.go:102] [dedicated-admin] adding namespace took 2.692273ms I0507 15:31:02.201372 3779 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0507 15:31:02.201894 3779 namespace.go:102] [openshift-authentication-operator] adding namespace took 3.249698ms I0507 15:31:02.201909 3779 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0507 15:31:02.202434 3779 namespace.go:102] [openshift-backplane-cee] adding namespace took 3.787502ms I0507 15:31:02.202453 3779 namespace.go:98] [openshift-console-operator] adding namespace I0507 15:31:02.202998 3779 namespace.go:102] [openshift-machine-config-operator] adding namespace took 4.349603ms I0507 15:31:02.203027 3779 namespace.go:98] [openshift-cluster-version] adding namespace I0507 15:31:02.203937 3779 namespace.go:102] [openshift-customer-monitoring] adding namespace took 5.285693ms I0507 15:31:02.203952 3779 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0507 15:31:02.204471 3779 namespace.go:102] [openshift-authentication] adding namespace took 5.806088ms I0507 15:31:02.204482 3779 namespace.go:98] [openshift-console] adding namespace I0507 15:31:02.204901 3779 namespace.go:102] [openshift-backplane] adding namespace took 6.231348ms I0507 15:31:02.204913 3779 namespace.go:98] [openshift-etcd] adding namespace I0507 15:31:02.205448 3779 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.77358ms I0507 15:31:02.205463 3779 namespace.go:98] [openshift-multus] adding namespace I0507 15:31:02.205900 3779 namespace.go:102] [openshift-host-network] adding namespace took 7.221618ms I0507 15:31:02.205916 3779 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace W0507 15:31:02.206328 3779 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-445p7: pod openshift-network-diagnostics/network-check-target-445p7: no pod IPs found I0507 15:31:02.206358 3779 namespace.go:102] [openshift-dns] adding namespace took 7.678436ms I0507 15:31:02.206367 3779 namespace.go:98] [openshift-network-console] adding namespace I0507 15:31:02.206923 3779 namespace.go:102] [openshift-network-diagnostics] adding namespace took 8.23707ms I0507 15:31:02.206940 3779 namespace.go:98] [openshift-image-registry] adding namespace I0507 15:31:02.207365 3779 namespace.go:102] [openshift-backplane-csm] adding namespace took 8.638115ms I0507 15:31:02.207375 3779 namespace.go:98] [openshift-apiserver] adding namespace I0507 15:31:02.207822 3779 namespace.go:102] [openshift-dns-operator] adding namespace took 9.059881ms I0507 15:31:02.207836 3779 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0507 15:31:02.208209 3779 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 8.472275ms I0507 15:31:02.208221 3779 namespace.go:98] [openshift-infra] adding namespace I0507 15:31:02.208639 3779 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.801783ms I0507 15:31:02.208653 3779 namespace.go:98] [openshift-ingress-operator] adding namespace I0507 15:31:02.209030 3779 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.65213ms I0507 15:31:02.209043 3779 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0507 15:31:02.209429 3779 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.515969ms I0507 15:31:02.209439 3779 namespace.go:98] [openshift-operators-redhat] adding namespace I0507 15:31:02.209861 3779 namespace.go:102] [openshift-console-operator] adding namespace took 7.402671ms I0507 15:31:02.209872 3779 namespace.go:98] [openshift-kube-scheduler] adding namespace I0507 15:31:02.210261 3779 namespace.go:102] [openshift-cluster-version] adding namespace took 7.227391ms I0507 15:31:02.210275 3779 namespace.go:98] [openshift-network-operator] adding namespace I0507 15:31:02.210622 3779 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.665945ms I0507 15:31:02.210634 3779 namespace.go:98] [openshift-marketplace] adding namespace I0507 15:31:02.211062 3779 namespace.go:102] [openshift-console] adding namespace took 6.574297ms I0507 15:31:02.211075 3779 namespace.go:98] [openshift-service-ca-operator] adding namespace W0507 15:31:02.211413 3779 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-mpbjx: pod openshift-multus/network-metrics-daemon-mpbjx: no pod IPs found I0507 15:31:02.211458 3779 namespace.go:102] [openshift-etcd] adding namespace took 6.537337ms I0507 15:31:02.211471 3779 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0507 15:31:02.211850 3779 namespace.go:102] [openshift-multus] adding namespace took 6.383192ms I0507 15:31:02.211859 3779 namespace.go:98] [openshift-backplane-srep] adding namespace I0507 15:31:02.212242 3779 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.319591ms I0507 15:31:02.212256 3779 namespace.go:98] [openshift] adding namespace I0507 15:31:02.212649 3779 namespace.go:102] [openshift-network-console] adding namespace took 6.278141ms I0507 15:31:02.212659 3779 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0507 15:31:02.213108 3779 namespace.go:102] [openshift-image-registry] adding namespace took 6.162965ms I0507 15:31:02.213121 3779 namespace.go:98] [open-cluster-management-2q55t83gi7r3iblf5ke81p881m72e003] adding namespace I0507 15:31:02.213612 3779 namespace.go:102] [openshift-apiserver] adding namespace took 6.231931ms I0507 15:31:02.213624 3779 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0507 15:31:02.213958 3779 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.116589ms I0507 15:31:02.213976 3779 namespace.go:98] [openshift-ingress] adding namespace I0507 15:31:02.214418 3779 namespace.go:102] [openshift-infra] adding namespace took 6.190552ms I0507 15:31:02.214433 3779 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0507 15:31:02.214874 3779 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.214814ms I0507 15:31:02.214888 3779 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0507 15:31:02.215206 3779 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.157966ms I0507 15:31:02.215217 3779 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0507 15:31:02.215705 3779 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.260129ms I0507 15:31:02.215718 3779 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0507 15:31:02.216019 3779 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.142917ms I0507 15:31:02.216029 3779 namespace.go:98] [openshift-machine-api] adding namespace I0507 15:31:02.216480 3779 namespace.go:102] [openshift-network-operator] adding namespace took 6.199289ms I0507 15:31:02.216493 3779 namespace.go:98] [openshift-config-managed] adding namespace I0507 15:31:02.216878 3779 namespace.go:102] [openshift-marketplace] adding namespace took 6.239352ms I0507 15:31:02.216890 3779 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0507 15:31:02.217262 3779 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.181797ms I0507 15:31:02.217275 3779 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0507 15:31:02.217675 3779 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.2005ms I0507 15:31:02.217686 3779 namespace.go:98] [kube-node-lease] adding namespace I0507 15:31:02.218106 3779 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.242793ms I0507 15:31:02.218115 3779 namespace.go:98] [openshift-route-controller-manager] adding namespace I0507 15:31:02.218488 3779 namespace.go:102] [openshift] adding namespace took 6.228923ms I0507 15:31:02.218498 3779 namespace.go:98] [openshift-operators] adding namespace I0507 15:31:02.218899 3779 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.234845ms I0507 15:31:02.218912 3779 namespace.go:98] [openshift-ingress-canary] adding namespace I0507 15:31:02.219260 3779 namespace.go:102] [open-cluster-management-2q55t83gi7r3iblf5ke81p881m72e003] adding namespace took 6.134207ms I0507 15:31:02.219270 3779 namespace.go:98] [openshift-network-node-identity] adding namespace I0507 15:31:02.220079 3779 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.448987ms I0507 15:31:02.220092 3779 namespace.go:98] [openshift-kube-apiserver] adding namespace I0507 15:31:02.220490 3779 namespace.go:102] [openshift-ingress] adding namespace took 6.50864ms I0507 15:31:02.220502 3779 namespace.go:98] [kube-system] adding namespace I0507 15:31:02.220851 3779 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.414072ms I0507 15:31:02.220862 3779 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0507 15:31:02.221270 3779 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.377582ms I0507 15:31:02.221283 3779 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0507 15:31:02.221673 3779 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.451147ms I0507 15:31:02.221684 3779 namespace.go:98] [openshift-config-operator] adding namespace I0507 15:31:02.222111 3779 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.389169ms I0507 15:31:02.222120 3779 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0507 15:31:02.222529 3779 namespace.go:102] [openshift-machine-api] adding namespace took 6.494773ms I0507 15:31:02.222543 3779 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0507 15:31:02.222922 3779 namespace.go:102] [openshift-config-managed] adding namespace took 6.424634ms I0507 15:31:02.222934 3779 namespace.go:98] [openshift-config] adding namespace I0507 15:31:02.223359 3779 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.463655ms I0507 15:31:02.223372 3779 namespace.go:98] [openshift-package-operator] adding namespace I0507 15:31:02.223859 3779 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.579078ms I0507 15:31:02.223873 3779 namespace.go:98] [openshift-backplane-mobb] adding namespace I0507 15:31:02.224239 3779 namespace.go:102] [kube-node-lease] adding namespace took 6.548168ms I0507 15:31:02.224249 3779 namespace.go:98] [openshift-apiserver-operator] adding namespace I0507 15:31:02.224689 3779 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.570152ms I0507 15:31:02.224700 3779 namespace.go:98] [openshift-node] adding namespace I0507 15:31:02.225082 3779 namespace.go:102] [openshift-operators] adding namespace took 6.580413ms I0507 15:31:02.225091 3779 namespace.go:98] [openshift-console-user-settings] adding namespace I0507 15:31:02.225479 3779 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.561508ms I0507 15:31:02.225492 3779 namespace.go:98] [openshift-controller-manager] adding namespace I0507 15:31:02.225879 3779 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.604504ms I0507 15:31:02.225888 3779 namespace.go:98] [default] adding namespace W0507 15:31:02.226225 3779 base_network_controller_namespace.go:410] Failed to get IPs for pod kube-system/global-pull-secret-syncer-6d6qj: pod kube-system/global-pull-secret-syncer-6d6qj: no pod IPs found I0507 15:31:02.226275 3779 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.17836ms I0507 15:31:02.226286 3779 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0507 15:31:02.226729 3779 namespace.go:102] [kube-system] adding namespace took 6.221893ms I0507 15:31:02.226740 3779 namespace.go:98] [kube-public] adding namespace I0507 15:31:02.227115 3779 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.248031ms I0507 15:31:02.227596 3779 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.308777ms I0507 15:31:02.227975 3779 namespace.go:102] [openshift-config-operator] adding namespace took 6.28638ms I0507 15:31:02.228387 3779 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.263164ms I0507 15:31:02.228790 3779 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.242194ms I0507 15:31:02.229171 3779 namespace.go:102] [openshift-config] adding namespace took 6.233188ms I0507 15:31:02.229594 3779 namespace.go:102] [openshift-package-operator] adding namespace took 6.217036ms I0507 15:31:02.229977 3779 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.100296ms I0507 15:31:02.230370 3779 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.116184ms I0507 15:31:02.230730 3779 namespace.go:102] [openshift-node] adding namespace took 6.023963ms I0507 15:31:02.231119 3779 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.023642ms I0507 15:31:02.231581 3779 namespace.go:102] [openshift-controller-manager] adding namespace took 6.084879ms I0507 15:31:02.231950 3779 namespace.go:102] [default] adding namespace took 6.058658ms I0507 15:31:02.231963 3779 namespace.go:98] [openshift-must-gather-operator] adding namespace I0507 15:31:02.232354 3779 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.061488ms I0507 15:31:02.232739 3779 namespace.go:102] [kube-public] adding namespace took 5.99349ms I0507 15:31:02.232753 3779 namespace.go:98] [openshift-logging] adding namespace I0507 15:31:02.233070 3779 namespace.go:102] [openshift-must-gather-operator] adding namespace took 1.10215ms I0507 15:31:02.233485 3779 namespace.go:102] [openshift-logging] adding namespace took 725.974µs I0507 15:31:02.233679 3779 master.go:548] Adding or Updating local node "ip-10-0-1-119.ec2.internal" for network "default" W0507 15:31:02.240778 3779 gateway.go:207] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-6d6qj: pod kube-system/global-pull-secret-syncer-6d6qj: no pod IPs found W0507 15:31:02.240797 3779 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-mpbjx: pod openshift-multus/network-metrics-daemon-mpbjx: no pod IPs found W0507 15:31:02.240806 3779 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-445p7: pod openshift-network-diagnostics/network-check-target-445p7: no pod IPs found I0507 15:31:02.241739 3779 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-119.ec2.internal for the network default I0507 15:31:02.242043 3779 zone_ic_handler.go:195] Time taken to create transit switch: 290.029µs I0507 15:31:02.242734 3779 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0507 15:31:02.242773 3779 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0507 15:31:02.242787 3779 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0507 15:31:02.242806 3779 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-119.ec2.internal in network "default" I0507 15:31:02.242862 3779 node_tracker.go:169] Node ip-10-0-1-119.ec2.internal switch + router changed, syncing services in network "default" I0507 15:31:02.242880 3779 services_controller.go:574] Full service sync requested for network=default I0507 15:31:02.343120 3779 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0507 15:31:02.343144 3779 services_controller.go:208] Setting up event handlers for services for network=default I0507 15:31:02.343229 3779 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0507 15:31:02.343325 3779 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0507 15:31:02.343381 3779 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0507 15:31:02.443892 3779 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0507 15:31:02.443919 3779 repair.go:60] Starting repairing loop for services I0507 15:31:02.444005 3779 repair.go:131] Deleted 0 stale service LBs I0507 15:31:02.444023 3779 repair.go:137] Deleted 0 stale Chassis Template Vars I0507 15:31:02.444039 3779 repair.go:62] Finished repairing loop for services: 126.239µs I0507 15:31:02.444061 3779 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0507 15:31:02.444070 3779 services_controller.go:254] Starting workers for network=default I0507 15:31:02.445023 3779 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-6d6qj] creating logical port kube-system_global-pull-secret-syncer-6d6qj for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:02.445075 3779 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-mpbjx] creating logical port openshift-multus_network-metrics-daemon-mpbjx for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:02.445091 3779 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-445p7] creating logical port openshift-network-diagnostics_network-check-target-445p7 for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:02.445229 3779 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-6d6qj I0507 15:31:02.445267 3779 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-mpbjx I0507 15:31:02.445359 3779 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-445p7 I0507 15:31:02.461754 3779 pod.go:62] [kube-system/global-pull-secret-syncer-6d6qj] pod update took 16.652409ms I0507 15:31:02.461774 3779 base_network_controller_pods.go:951] [default/kube-system/global-pull-secret-syncer-6d6qj] addLogicalPort annotation time took 16.677533ms I0507 15:31:02.462748 3779 pods.go:271] [kube-system/global-pull-secret-syncer-6d6qj] addLogicalPort took 17.739501ms, libovsdb time 737.207µs I0507 15:31:02.464259 3779 pod.go:62] [openshift-network-diagnostics/network-check-target-445p7] pod update took 19.100841ms I0507 15:31:02.464279 3779 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-445p7] addLogicalPort annotation time took 19.120442ms I0507 15:31:02.465293 3779 pod.go:62] [openshift-multus/network-metrics-daemon-mpbjx] pod update took 20.155438ms I0507 15:31:02.465357 3779 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-mpbjx] addLogicalPort annotation time took 20.182979ms I0507 15:31:02.465403 3779 pods.go:271] [openshift-network-diagnostics/network-check-target-445p7] addLogicalPort took 20.315684ms, libovsdb time 859.068µs I0507 15:31:02.466324 3779 pods.go:271] [openshift-multus/network-metrics-daemon-mpbjx] addLogicalPort took 21.258449ms, libovsdb time 627.235µs I0507 15:31:02.466870 3779 admin_network_policy_controller.go:227] Starting controller default-network-controller I0507 15:31:02.466894 3779 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0507 15:31:02.466906 3779 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0507 15:31:02.466918 3779 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0507 15:31:02.467131 3779 repair.go:29] Repairing admin network policies took 207.481µs I0507 15:31:02.467230 3779 repair.go:92] Repairing baseline admin network policies took 90.068µs I0507 15:31:02.467807 3779 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0507 15:31:02.467830 3779 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0507 15:31:02.471210 3779 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0507 15:31:02.471237 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0507 15:31:02.471250 3779 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0507 15:31:02.473367 3779 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0507 15:31:02.473392 3779 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0507 15:31:02.473405 3779 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0507 15:31:02.475867 3779 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0507 15:31:02.475892 3779 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0507 15:31:02.475907 3779 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0507 15:31:02.477567 3779 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0507 15:31:02.477592 3779 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0507 15:31:02.477606 3779 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0507 15:31:02.478915 3779 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0507 15:31:02.478933 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0507 15:31:02.478941 3779 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0507 15:31:02.480774 3779 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0507 15:31:02.480797 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0507 15:31:02.480809 3779 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0507 15:31:02.482712 3779 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0507 15:31:02.482732 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0507 15:31:02.482740 3779 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0507 15:31:02.483314 3779 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0507 15:31:02.483331 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0507 15:31:02.483375 3779 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0507 15:31:02.484173 3779 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0507 15:31:02.484197 3779 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0507 15:31:02.484212 3779 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0507 15:31:02.487324 3779 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0507 15:31:02.487371 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0507 15:31:02.487384 3779 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0507 15:31:02.488383 3779 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0507 15:31:02.488406 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0507 15:31:02.488415 3779 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0507 15:31:02.489613 3779 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0507 15:31:02.489639 3779 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0507 15:31:02.489652 3779 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0507 15:31:02.491645 3779 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0507 15:31:02.491679 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0507 15:31:02.491688 3779 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0507 15:31:02.492373 3779 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0507 15:31:02.492392 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0507 15:31:02.492986 3779 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0507 15:31:02.493005 3779 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0507 15:31:02.493900 3779 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0507 15:31:02.493939 3779 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0507 15:31:02.493947 3779 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0507 15:31:02.494558 3779 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0507 15:31:02.494574 3779 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0507 15:31:02.494580 3779 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0507 15:31:02.494927 3779 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0507 15:31:02.494942 3779 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0507 15:31:02.494952 3779 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0507 15:31:02.496566 3779 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0507 15:31:02.496589 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0507 15:31:02.496600 3779 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0507 15:31:02.497231 3779 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0507 15:31:02.497254 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0507 15:31:02.497279 3779 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0507 15:31:02.498133 3779 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0507 15:31:02.498157 3779 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0507 15:31:02.498169 3779 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0507 15:31:02.498903 3779 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0507 15:31:02.498921 3779 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0507 15:31:02.498929 3779 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0507 15:31:02.499282 3779 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0507 15:31:02.499307 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0507 15:31:02.499318 3779 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0507 15:31:02.499876 3779 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0507 15:31:02.499896 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0507 15:31:02.499903 3779 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0507 15:31:02.500215 3779 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0507 15:31:02.500239 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0507 15:31:02.500249 3779 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0507 15:31:02.500601 3779 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0507 15:31:02.500617 3779 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0507 15:31:02.500627 3779 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0507 15:31:02.502198 3779 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0507 15:31:02.502220 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0507 15:31:02.502232 3779 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0507 15:31:02.502890 3779 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0507 15:31:02.502910 3779 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0507 15:31:02.502917 3779 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0507 15:31:02.504380 3779 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0507 15:31:02.504401 3779 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0507 15:31:02.504413 3779 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0507 15:31:02.505363 3779 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0507 15:31:02.510570 3779 egressfirewall.go:374] Starting EgressFirewall controller I0507 15:31:02.510596 3779 controller.go:133] Adding controller egress-firewall-controller event handlers I0507 15:31:02.510627 3779 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0507 15:31:02.510643 3779 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0507 15:31:02.510645 3779 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0507 15:31:02.510702 3779 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0507 15:31:02.510714 3779 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0507 15:31:02.510992 3779 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0507 15:31:02.511006 3779 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0507 15:31:02.511011 3779 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0507 15:31:02.511019 3779 egressqos.go:193] Setting up event handlers for EgressQoS I0507 15:31:02.511030 3779 egressfirewall.go:723] Syncing node "ip-10-0-1-119.ec2.internal" for egress firewall I0507 15:31:02.511545 3779 egressqos.go:245] Starting EgressQoS Controller I0507 15:31:02.511569 3779 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0507 15:31:02.511580 3779 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0507 15:31:02.511585 3779 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0507 15:31:02.511589 3779 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0507 15:31:02.511594 3779 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0507 15:31:02.511598 3779 shared_informer.go:356] "Caches are synced" controller="egressqos" I0507 15:31:02.511602 3779 egressqos.go:259] Repairing EgressQoSes I0507 15:31:02.511606 3779 egressqos.go:399] Starting repairing loop for egressqos I0507 15:31:02.511704 3779 egressqos.go:401] Finished repairing loop for egressqos: 97.347µs I0507 15:31:02.511726 3779 egressservice_zone.go:132] Setting up event handlers for Egress Services I0507 15:31:02.511757 3779 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-119.ec2.internal I0507 15:31:02.511839 3779 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-119.ec2.internal : 84.505µs I0507 15:31:02.511927 3779 egressservice_zone.go:210] Starting Egress Services Controller I0507 15:31:02.511942 3779 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0507 15:31:02.511952 3779 shared_informer.go:356] "Caches are synced" controller="egressservices" I0507 15:31:02.511961 3779 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0507 15:31:02.511968 3779 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0507 15:31:02.511976 3779 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0507 15:31:02.511983 3779 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0507 15:31:02.511990 3779 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0507 15:31:02.511997 3779 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0507 15:31:02.512001 3779 egressservice_zone.go:228] Repairing Egress Services I0507 15:31:02.512311 3779 master_controller.go:88] Starting Admin Policy Based Route Controller I0507 15:31:02.512321 3779 external_controller.go:278] Starting Admin Policy Based Route Controller I0507 15:31:02.512353 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-119.ec2.internal I0507 15:31:02.512368 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-119.ec2.internal: 17.453µs I0507 15:31:02.512448 3779 default_network_controller.go:585] Completing all the Watchers took 314.006846ms I0507 15:31:02.512472 3779 default_network_controller.go:589] Starting unidling controllers I0507 15:31:02.512488 3779 unidle.go:46] Registering OVN SB ControllerEvent handler I0507 15:31:02.512494 3779 unidle.go:63] Populating Initial ContollerEvent events I0507 15:31:02.512506 3779 unidle.go:79] Setting up event handlers for services I0507 15:31:02.512566 3779 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0507 15:31:02.568864 3779 default_node_network_controller.go:519] Node connection status = connected I0507 15:31:03.069179 3779 default_node_network_controller.go:519] Node connection status = connected I0507 15:31:03.569502 3779 default_node_network_controller.go:519] Node connection status = connected I0507 15:31:04.069447 3779 default_node_network_controller.go:519] Node connection status = connected I0507 15:31:04.293450 3779 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0507 15:31:04.569460 3779 default_node_network_controller.go:519] Node connection status = connected I0507 15:31:04.588846 3779 bridgeconfig.go:537] Gateway is ready I0507 15:31:04.588868 3779 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0507 15:31:04.602123 3779 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.119. Updating OVN encap IP. I0507 15:31:04.608086 3779 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.119 - it is already configured I0507 15:31:04.608107 3779 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.119/24:{} 172.20.0.1/32:{}]. Updating annotations. I0507 15:31:04.608452 3779 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.119/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-119.ec2.internal","mac-address":"0a:ff:d1:c8:bc:d5","ip-addresses":["10.0.1.119/24"],"ip-address":"10.0.1.119/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:b2baa5fb-4cf3-43e9-a681-16aa7a77ba28 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.119/24"}] on node ip-10-0-1-119.ec2.internal I0507 15:31:04.624927 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-119.ec2.internal I0507 15:31:04.624956 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-119.ec2.internal: 33.376µs I0507 15:31:04.625001 3779 egressfirewall.go:723] Syncing node "ip-10-0-1-119.ec2.internal" for egress firewall I0507 15:31:04.624999 3779 master.go:548] Adding or Updating local node "ip-10-0-1-119.ec2.internal" for network "default" I0507 15:31:04.625017 3779 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-119.ec2.internal in network "default" I0507 15:31:04.625074 3779 node_tracker.go:169] Node ip-10-0-1-119.ec2.internal switch + router changed, syncing services in network "default" I0507 15:31:04.625093 3779 services_controller.go:574] Full service sync requested for network=default I0507 15:31:04.625563 3779 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.1.119 I0507 15:31:04.625716 3779 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-119.ec2.internal I0507 15:31:04.630111 3779 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-119.ec2.internal. Updating no re-route policies I0507 15:31:04.630188 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-4wbmm I0507 15:31:04.630210 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-4wbmm I0507 15:31:04.630211 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-119.ec2.internal I0507 15:31:04.630212 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-445p7 I0507 15:31:04.630219 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-77hvb I0507 15:31:04.630229 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-119.ec2.internal I0507 15:31:04.630232 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-445p7 I0507 15:31:04.630233 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-vjjxw I0507 15:31:04.630236 3779 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-4wbmm after 0 failed attempt(s) I0507 15:31:04.630240 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-77hvb I0507 15:31:04.630255 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-2zxrt I0507 15:31:04.630265 3779 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-77hvb after 0 failed attempt(s) I0507 15:31:04.630259 3779 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-119.ec2.internal after 0 failed attempt(s) I0507 15:31:04.630266 3779 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-6d6qj I0507 15:31:04.630277 3779 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-wfn5k I0507 15:31:04.630280 3779 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-445p7] creating logical port openshift-network-diagnostics_network-check-target-445p7 for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:04.630284 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-w8h8w I0507 15:31:04.630288 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-2zxrt I0507 15:31:04.630290 3779 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-wfn5k I0507 15:31:04.630293 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-vjjxw I0507 15:31:04.630287 3779 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-6d6qj I0507 15:31:04.630296 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-w8h8w I0507 15:31:04.630306 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-bwz9x I0507 15:31:04.630315 3779 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-wfn5k after 0 failed attempt(s) I0507 15:31:04.630314 3779 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-2zxrt after 0 failed attempt(s) I0507 15:31:04.630322 3779 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-w8h8w after 0 failed attempt(s) I0507 15:31:04.630214 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-mpbjx I0507 15:31:04.630223 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-k2mm6 I0507 15:31:04.630357 3779 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-6d6qj] creating logical port kube-system_global-pull-secret-syncer-6d6qj for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:04.630363 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-mpbjx I0507 15:31:04.630382 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-k2mm6 I0507 15:31:04.630390 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-bwz9x I0507 15:31:04.630410 3779 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-k2mm6 after 0 failed attempt(s) I0507 15:31:04.630416 3779 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-bwz9x after 0 failed attempt(s) I0507 15:31:04.630428 3779 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-mpbjx] creating logical port openshift-multus_network-metrics-daemon-mpbjx for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:04.630457 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-fkbkm I0507 15:31:04.630460 3779 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-vjjxw after 0 failed attempt(s) I0507 15:31:04.630464 3779 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-119.ec2.internal I0507 15:31:04.630475 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-fkbkm I0507 15:31:04.630478 3779 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-119.ec2.internal I0507 15:31:04.630491 3779 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-119.ec2.internal after 0 failed attempt(s) I0507 15:31:04.630495 3779 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-fkbkm after 0 failed attempt(s) I0507 15:31:04.631084 3779 pods.go:271] [openshift-network-diagnostics/network-check-target-445p7] addLogicalPort took 819.9µs, libovsdb time 305.099µs I0507 15:31:04.631096 3779 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-445p7 after 0 failed attempt(s) I0507 15:31:04.631188 3779 pods.go:271] [openshift-multus/network-metrics-daemon-mpbjx] addLogicalPort took 795.457µs, libovsdb time 305.906µs I0507 15:31:04.631195 3779 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-mpbjx after 0 failed attempt(s) I0507 15:31:04.631283 3779 pods.go:271] [kube-system/global-pull-secret-syncer-6d6qj] addLogicalPort took 960.94µs, libovsdb time 374.382µs I0507 15:31:04.631290 3779 obj_retry.go:418] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-6d6qj after 0 failed attempt(s) I0507 15:31:04.641989 3779 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0507 15:31:04.657097 3779 iptables.go:108] Creating table: mangle chain: OUTPUT I0507 15:31:04.661066 3779 iptables.go:108] Creating table: nat chain: OUTPUT I0507 15:31:04.703018 3779 iptables.go:108] Creating table: nat chain: PREROUTING I0507 15:31:04.761247 3779 iptables.go:108] Creating table: filter chain: FORWARD I0507 15:31:04.779439 3779 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0507 15:31:04.779579 3779 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0507 15:31:04.779629 3779 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0507 15:31:04.779714 3779 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0507 15:31:04.779785 3779 gateway_init.go:325] Gateway and management port readiness took 2.715712069s I0507 15:31:04.779812 3779 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0507 15:31:04.779881 3779 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0507 15:31:04.779893 3779 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0507 15:31:04.779902 3779 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0507 15:31:04.779980 3779 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0507 15:31:04.780069 3779 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 58.15µs I0507 15:31:04.780320 3779 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0507 15:31:04.780363 3779 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 33.045µs I0507 15:31:04.780372 3779 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 52.228µs I0507 15:31:04.780377 3779 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0507 15:31:04.780386 3779 nad_controller.go:379] [node-nad-controller NAD controller]: started I0507 15:31:04.780391 3779 default_node_network_controller.go:899] Starting the default node network controller I0507 15:31:04.780396 3779 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0507 15:31:04.780471 3779 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 12.318µs I0507 15:31:04.780557 3779 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0507 15:31:04.780565 3779 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0507 15:31:04.780588 3779 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0008f23d0 0xc0002c7ce0 0xc0002c7dc0 0xc0002c7e30 0xc0002c7ea0 0xc0002c7f10 0xc0002ee000 0xc000cc61e0 0xc0002ee070 0xc0002ee0e0 0xc000aac2d0 0xc0064c1340 0xc0002ee150 map[0x2ff9080:0xc00220c840 0x2ff95c0:0xc00220ca80 0x2ffa2e0:0xc00220c780 0x2ffa580:0xc00220c7e0 0x2ffad60:0xc00220c8a0 0x2ffb000:0xc00220c900 0x2ffb2a0:0xc00220c960 0x2ffba80:0xc00220cae0 0x2ffbd20:0xc00220cb40 0x3026980:0xc000e98840 0x3026d20:0xc000e988a0 0x3027800:0xc000cc4180 0x3029160:0xc001c82000 0x30298a0:0xc002a8d200 0x302a720:0xc000e987e0] 0xc0000d2d20 0} I0507 15:31:04.780644 3779 gateway.go:151] Starting gateway service sync I0507 15:31:04.781054 3779 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0507 15:31:04.783453 3779 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0507 15:31:04.804473 3779 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0507 15:31:04.805788 3779 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0507 15:31:04.807094 3779 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0507 15:31:04.841689 3779 gateway.go:168] Gateway service sync done. Time taken: 61.031095ms I0507 15:31:04.842384 3779 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31578 I0507 15:31:04.842531 3779 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0008f23d0 0xc0002c7ce0 0xc0002c7dc0 0xc0002c7e30 0xc0002c7ea0 0xc0002c7f10 0xc0002ee000 0xc000cc61e0 0xc0002ee070 0xc0002ee0e0 0xc000aac2d0 0xc0064c1340 0xc0002ee150 map[0x2ff9080:0xc00220c840 0x2ff95c0:0xc00220ca80 0x2ffa2e0:0xc00220c780 0x2ffa580:0xc00220c7e0 0x2ffad60:0xc00220c8a0 0x2ffb000:0xc00220c900 0x2ffb2a0:0xc00220c960 0x2ffba80:0xc00220cae0 0x2ffbd20:0xc00220cb40 0x3026980:0xc000e98840 0x3026d20:0xc000e988a0 0x3027800:0xc000cc4180 0x3029160:0xc001c82000 0x30298a0:0xc002a8d200 0x302a720:0xc000e987e0] 0xc0000d2d20 0} I0507 15:31:04.842985 3779 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0008f23d0 0xc0002c7ce0 0xc0002c7dc0 0xc0002c7e30 0xc0002c7ea0 0xc0002c7f10 0xc0002ee000 0xc000cc61e0 0xc0002ee070 0xc0002ee0e0 0xc000aac2d0 0xc0064c1340 0xc0002ee150 map[0x2ff9080:0xc00220c840 0x2ff95c0:0xc00220ca80 0x2ffa2e0:0xc00220c780 0x2ffa580:0xc00220c7e0 0x2ffad60:0xc00220c8a0 0x2ffb000:0xc00220c900 0x2ffb2a0:0xc00220c960 0x2ffba80:0xc00220cae0 0x2ffbd20:0xc00220cb40 0x3026980:0xc000e98840 0x3026d20:0xc000e988a0 0x3027800:0xc000cc4180 0x3029160:0xc001c82000 0x30298a0:0xc002a8d200 0x302a720:0xc000e987e0] 0xc0000d2d20 0} I0507 15:31:04.843021 3779 gateway.go:532] Reconciling gateway with updates I0507 15:31:04.848036 3779 gateway.go:339] Spawning Conntrack Rule Check Thread I0507 15:31:04.848114 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0507 15:31:04.848127 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0507 15:31:04.848132 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0507 15:31:04.848139 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0507 15:31:04.848145 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0507 15:31:04.848148 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0507 15:31:04.848155 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0507 15:31:04.848159 3779 default_node_network_controller.go:944] Gateway and management port readiness took 67.583951ms I0507 15:31:04.848161 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0507 15:31:04.848174 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0507 15:31:04.848183 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0507 15:31:04.848191 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0507 15:31:04.848198 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0507 15:31:04.848216 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0507 15:31:04.848216 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0507 15:31:04.848226 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0507 15:31:04.848160 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0507 15:31:04.848227 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0507 15:31:04.848237 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0507 15:31:04.848240 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0507 15:31:04.848248 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0507 15:31:04.848243 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0507 15:31:04.848249 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0507 15:31:04.848256 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0507 15:31:04.848255 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0507 15:31:04.848263 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0507 15:31:04.848251 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0507 15:31:04.848267 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0507 15:31:04.848268 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0507 15:31:04.848273 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0507 15:31:04.848276 3779 port_claim.go:71] Svc openshift-ingress/router-default: port 31599 is already open, no action needed I0507 15:31:04.848288 3779 port_claim.go:71] Svc openshift-ingress/router-default: port 31203 is already open, no action needed I0507 15:31:04.848293 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0507 15:31:04.848294 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0507 15:31:04.848299 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0507 15:31:04.848299 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0507 15:31:04.848300 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0507 15:31:04.848307 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0507 15:31:04.848314 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0507 15:31:04.848328 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0507 15:31:04.848332 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0507 15:31:04.848251 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0507 15:31:04.848363 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0507 15:31:04.848239 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0507 15:31:04.848366 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0507 15:31:04.848374 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0507 15:31:04.848377 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0507 15:31:04.848381 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0507 15:31:04.848386 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0507 15:31:04.848389 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0507 15:31:04.848389 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0507 15:31:04.848395 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0507 15:31:04.848364 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0507 15:31:04.848399 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0507 15:31:04.848411 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0507 15:31:04.848414 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0507 15:31:04.848413 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0507 15:31:04.848265 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0507 15:31:04.848427 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0507 15:31:04.848420 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0507 15:31:04.848434 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0507 15:31:04.848440 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0507 15:31:04.848442 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0507 15:31:04.848449 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0507 15:31:04.848460 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0507 15:31:04.848463 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0507 15:31:04.848467 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0507 15:31:04.848429 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0507 15:31:04.848473 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0507 15:31:04.848481 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0507 15:31:04.848496 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0507 15:31:04.848396 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0507 15:31:04.848514 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0507 15:31:04.848520 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0507 15:31:04.848273 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0507 15:31:04.848582 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0507 15:31:04.848588 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0507 15:31:04.848596 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0507 15:31:04.848600 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0507 15:31:04.848605 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0507 15:31:04.848605 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0507 15:31:04.848615 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0507 15:31:04.848614 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0507 15:31:04.848482 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0507 15:31:04.848625 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0507 15:31:04.848633 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0507 15:31:04.848634 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0507 15:31:04.848634 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0507 15:31:04.848647 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0507 15:31:04.848647 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0507 15:31:04.848452 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0507 15:31:04.848659 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0507 15:31:04.848617 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0507 15:31:04.848667 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0507 15:31:04.848672 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0507 15:31:04.848608 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0507 15:31:04.848475 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0507 15:31:04.848692 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0507 15:31:04.848357 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0507 15:31:04.848791 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0507 15:31:04.849140 3779 node_ip_handler_linux.go:160] Node IP manager is running I0507 15:31:04.868791 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0507 15:31:04.868854 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0507 15:31:04.868870 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0507 15:31:04.868877 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) W0507 15:31:04.886810 3779 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0507 15:31:04.886950 3779 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0507 15:31:04.887091 3779 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0507 15:31:04.916067 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0507 15:31:04.916076 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0507 15:31:04.916093 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0507 15:31:04.956542 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0507 15:31:04.956583 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0507 15:31:04.956599 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0507 15:31:04.956608 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0507 15:31:04.956617 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0507 15:31:04.956626 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0507 15:31:04.956643 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0507 15:31:04.956665 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0507 15:31:04.956672 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0507 15:31:04.956681 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0507 15:31:04.956687 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0507 15:31:04.956693 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0507 15:31:04.956698 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0507 15:31:04.956705 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0507 15:31:04.956712 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0507 15:31:04.956723 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0507 15:31:04.956729 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0507 15:31:04.956735 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0507 15:31:04.956740 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0507 15:31:04.956745 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0507 15:31:04.956750 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0507 15:31:04.956755 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0507 15:31:04.956761 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0507 15:31:04.956766 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0507 15:31:04.956771 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0507 15:31:04.956776 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0507 15:31:04.956781 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0507 15:31:04.956798 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0507 15:31:06.335029 3779 default_node_network_controller.go:1380] Starting node controller node sync W0507 15:31:06.335040 3779 egressip_healthcheck.go:76] Health checking using insecure connection I0507 15:31:06.335097 3779 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0507 15:31:06.346740 3779 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.689993ms I0507 15:31:06.346921 3779 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0507 15:31:06.347191 3779 egressservice_node.go:93] Setting up event handlers for Egress Services I0507 15:31:06.347311 3779 egressservice_node.go:183] Starting Egress Services Controller I0507 15:31:06.347328 3779 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0507 15:31:06.347359 3779 shared_informer.go:356] "Caches are synced" controller="egressservices" I0507 15:31:06.347367 3779 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0507 15:31:06.347374 3779 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0507 15:31:06.347382 3779 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0507 15:31:06.347388 3779 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0507 15:31:06.347394 3779 egressservice_node.go:197] Repairing Egress Services I0507 15:31:06.357325 3779 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0507 15:31:06.357347 3779 external_controller.go:278] Starting Admin Policy Based Route Controller I0507 15:31:06.357560 3779 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0507 15:31:06.357570 3779 link_network_manager.go:119] Link manager is running I0507 15:31:06.357580 3779 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0507 15:31:06.357793 3779 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0507 15:31:06.358148 3779 vrf_manager.go:128] VRF manager is running I0507 15:31:06.358229 3779 node_controller_manager.go:437] Removing flows to drop GARP I0507 15:31:06.358238 3779 gateway.go:532] Reconciling gateway with updates I0507 15:31:06.378446 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0507 15:31:06.378457 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0507 15:31:06.378469 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0507 15:31:06.378481 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0507 15:31:06.378486 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0507 15:31:06.378486 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0507 15:31:06.378496 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0507 15:31:06.378504 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0507 15:31:06.378508 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0507 15:31:06.378505 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0507 15:31:06.378516 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0507 15:31:06.378521 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0507 15:31:06.378523 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0507 15:31:06.378523 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0507 15:31:06.378527 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0507 15:31:06.378536 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0507 15:31:06.378541 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0507 15:31:06.378509 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0507 15:31:06.378550 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0507 15:31:06.378549 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0507 15:31:06.378475 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0507 15:31:06.378567 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0507 15:31:06.378568 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0507 15:31:06.378578 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0507 15:31:06.378582 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0507 15:31:06.378588 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0507 15:31:06.378597 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0507 15:31:06.378597 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0507 15:31:06.378599 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0507 15:31:06.378561 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0507 15:31:06.378618 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0507 15:31:06.378620 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0507 15:31:06.378619 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0507 15:31:06.378627 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0507 15:31:06.378636 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0507 15:31:06.378635 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0507 15:31:06.378644 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0507 15:31:06.378646 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0507 15:31:06.378647 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0507 15:31:06.378653 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0507 15:31:06.378655 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0507 15:31:06.378662 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0507 15:31:06.378664 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0507 15:31:06.378671 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0507 15:31:06.378677 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0507 15:31:06.378493 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0507 15:31:06.378573 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0507 15:31:06.378717 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0507 15:31:06.378624 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0507 15:31:06.378734 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0507 15:31:06.378760 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0507 15:31:06.378784 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0507 15:31:06.378785 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0507 15:31:06.378790 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0507 15:31:06.378791 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0507 15:31:06.378805 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0507 15:31:06.378806 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0507 15:31:06.378807 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0507 15:31:06.378812 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0507 15:31:06.378818 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0507 15:31:06.378817 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0507 15:31:06.378823 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0507 15:31:06.378828 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0507 15:31:06.378828 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0507 15:31:06.378836 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0507 15:31:06.378837 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0507 15:31:06.378839 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0507 15:31:06.378841 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0507 15:31:06.378850 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0507 15:31:06.378852 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0507 15:31:06.378853 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0507 15:31:06.378852 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0507 15:31:06.378858 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0507 15:31:06.378862 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0507 15:31:06.378865 3779 port_claim.go:71] Svc openshift-ingress/router-default: port 31599 is already open, no action needed I0507 15:31:06.378864 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0507 15:31:06.378865 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0507 15:31:06.378870 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0507 15:31:06.378875 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0507 15:31:06.378737 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0507 15:31:06.378879 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0507 15:31:06.378885 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0507 15:31:06.378888 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0507 15:31:06.378813 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0507 15:31:06.378853 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0507 15:31:06.378837 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0507 15:31:06.378869 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0507 15:31:06.378880 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0507 15:31:06.378831 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0507 15:31:06.378839 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0507 15:31:06.378921 3779 port_claim.go:71] Svc openshift-ingress/router-default: port 31203 is already open, no action needed I0507 15:31:06.378637 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0507 15:31:06.378625 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0507 15:31:06.379013 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0507 15:31:06.379021 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0507 15:31:06.378637 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0507 15:31:06.379033 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0507 15:31:06.378642 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0507 15:31:06.378643 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0507 15:31:06.379075 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0507 15:31:06.378620 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0507 15:31:06.379101 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0507 15:31:06.378798 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0507 15:31:06.379120 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0507 15:31:06.378804 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0507 15:31:06.378635 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0507 15:31:06.378824 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0507 15:31:06.378835 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0507 15:31:06.379174 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0507 15:31:06.378846 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0507 15:31:06.378846 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0507 15:31:06.378874 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0507 15:31:06.378880 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0507 15:31:06.379267 3779 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0507 15:31:06.379280 3779 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0507 15:31:06.418857 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0507 15:31:06.418882 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0507 15:31:06.418890 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0507 15:31:06.418896 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0507 15:31:06.418902 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0507 15:31:06.418918 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0507 15:31:06.418934 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0507 15:31:06.418940 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0507 15:31:06.418947 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0507 15:31:06.458237 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0507 15:31:06.458245 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0507 15:31:06.458263 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0507 15:31:06.458269 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0507 15:31:06.458277 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0507 15:31:06.458284 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0507 15:31:06.458293 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0507 15:31:06.458302 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0507 15:31:06.458309 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0507 15:31:06.458315 3779 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0507 15:31:08.211292 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-119.ec2.internal I0507 15:31:08.211322 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-119.ec2.internal: 37.752µs I0507 15:31:08.239110 3779 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-5849cb56c9-wswmc] creating logical port openshift-image-registry_image-registry-5849cb56c9-wswmc for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:08.239240 3779 kube.go:256] Updating pod openshift-image-registry/image-registry-5849cb56c9-wswmc I0507 15:31:08.260162 3779 pod.go:62] [openshift-image-registry/image-registry-5849cb56c9-wswmc] pod update took 20.953355ms I0507 15:31:08.260183 3779 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-5849cb56c9-wswmc] addLogicalPort annotation time took 20.980629ms I0507 15:31:08.261296 3779 pods.go:271] [openshift-image-registry/image-registry-5849cb56c9-wswmc] addLogicalPort took 22.199899ms, libovsdb time 868.527µs I0507 15:31:08.279394 3779 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-8pg7v] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-8pg7v for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:08.279498 3779 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-8pg7v I0507 15:31:08.282724 3779 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-mtm7b] creating logical port openshift-dns_dns-default-mtm7b for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:08.282768 3779 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29636130-79wpf for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:08.282841 3779 kube.go:256] Updating pod openshift-dns/dns-default-mtm7b I0507 15:31:08.282842 3779 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf I0507 15:31:08.287380 3779 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-5v67m] creating logical port openshift-ingress-canary_ingress-canary-5v67m for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:08.287470 3779 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-5v67m I0507 15:31:08.297241 3779 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8pg7v] pod update took 17.767328ms I0507 15:31:08.297266 3779 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-8pg7v] addLogicalPort annotation time took 17.798339ms I0507 15:31:08.298638 3779 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8pg7v] addLogicalPort took 19.321332ms, libovsdb time 1.051623ms I0507 15:31:08.303134 3779 pod.go:62] [openshift-dns/dns-default-mtm7b] pod update took 20.317239ms I0507 15:31:08.303151 3779 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-mtm7b] addLogicalPort annotation time took 20.346639ms I0507 15:31:08.304200 3779 pods.go:271] [openshift-dns/dns-default-mtm7b] addLogicalPort took 21.556498ms, libovsdb time 762.894µs I0507 15:31:08.304828 3779 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf] pod update took 22.00928ms I0507 15:31:08.304847 3779 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf] addLogicalPort annotation time took 22.028025ms I0507 15:31:08.305908 3779 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf] addLogicalPort took 23.176017ms, libovsdb time 779.012µs I0507 15:31:08.307717 3779 pod.go:62] [openshift-ingress-canary/ingress-canary-5v67m] pod update took 20.268074ms I0507 15:31:08.307737 3779 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-5v67m] addLogicalPort annotation time took 20.289303ms I0507 15:31:08.308648 3779 pods.go:271] [openshift-ingress-canary/ingress-canary-5v67m] addLogicalPort took 21.276091ms, libovsdb time 655.054µs I0507 15:31:08.669373 3779 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8pg7v 4a0fe9731102f98c27fff3e98fc15d1822d243f1e61619ee6b1cd2b02ffe8531 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8pg7v 4a0fe9731102f98c27fff3e98fc15d1822d243f1e61619ee6b1cd2b02ffe8531 network default NAD default NAD key ] I0507 15:31:08.703578 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-8pg7v, hostIfaceName: 4a0fe9731102f98, network: default, NAD default, SandboxID: "4a0fe9731102f98c27fff3e98fc15d1822d243f1e61619ee6b1cd2b02ffe8531", PCI device ID: , UID: "62181f96-6f4a-4267-9de6-93e9d00d1f60", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0507 15:31:08.711535 3779 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf 4ed842bccfef35deec86440aa5e34c856e17289401e54adacd710775473bf3e0 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf 4ed842bccfef35deec86440aa5e34c856e17289401e54adacd710775473bf3e0 network default NAD default NAD key ] I0507 15:31:08.735545 3779 cni.go:452] [openshift-ingress-canary/ingress-canary-5v67m 403f87d49c88112667dddc0d7bce2101d9466aa20a990d79c4679c9777edce36 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-5v67m 403f87d49c88112667dddc0d7bce2101d9466aa20a990d79c4679c9777edce36 network default NAD default NAD key ] I0507 15:31:08.737254 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29636130-79wpf, hostIfaceName: 4ed842bccfef35d, network: default, NAD default, SandboxID: "4ed842bccfef35deec86440aa5e34c856e17289401e54adacd710775473bf3e0", PCI device ID: , UID: "598d1cfb-9fbc-493e-85ba-b6a61fa10297", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0507 15:31:08.751086 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-5v67m, hostIfaceName: 403f87d49c88112, network: default, NAD default, SandboxID: "403f87d49c88112667dddc0d7bce2101d9466aa20a990d79c4679c9777edce36", PCI device ID: , UID: "97348c30-f244-41bc-abc1-3373c2ea8674", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0507 15:31:08.769094 3779 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8pg7v 4a0fe9731102f98c27fff3e98fc15d1822d243f1e61619ee6b1cd2b02ffe8531 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-8pg7v 4a0fe9731102f98c27fff3e98fc15d1822d243f1e61619ee6b1cd2b02ffe8531 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:f3:63:14:c0:03\",\"name\":\"4a0fe9731102f98\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/42c785df-b0a7-4b0b-8fb8-faf8d81983b5\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:08.787886 3779 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf 4ed842bccfef35deec86440aa5e34c856e17289401e54adacd710775473bf3e0 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf 4ed842bccfef35deec86440aa5e34c856e17289401e54adacd710775473bf3e0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:03:7e:ef:b7:71\",\"name\":\"4ed842bccfef35d\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/505c9dc4-48f4-4149-b3bf-41749d68beb1\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:08.799455 3779 cni.go:473] [openshift-ingress-canary/ingress-canary-5v67m 403f87d49c88112667dddc0d7bce2101d9466aa20a990d79c4679c9777edce36 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-5v67m 403f87d49c88112667dddc0d7bce2101d9466aa20a990d79c4679c9777edce36 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:c4:84:05:3f:26\",\"name\":\"403f87d49c88112\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4c84b282-f40c-4e56-8308-6c1517d65843\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:09.344149 3779 cni.go:452] [kube-system/global-pull-secret-syncer-6d6qj 69364d7b071ea7ca7b4c16d9fda59b4326ae81b8410ea53d095119dae85165e0 network default NAD default NAD key ] ADD starting CNI request [kube-system/global-pull-secret-syncer-6d6qj 69364d7b071ea7ca7b4c16d9fda59b4326ae81b8410ea53d095119dae85165e0 network default NAD default NAD key ] I0507 15:31:09.354687 3779 helper_linux.go:556] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-6d6qj, hostIfaceName: 69364d7b071ea7c, network: default, NAD default, SandboxID: "69364d7b071ea7ca7b4c16d9fda59b4326ae81b8410ea53d095119dae85165e0", PCI device ID: , UID: "dbbdea1b-56bd-4538-85b3-fd77a1134b93", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0507 15:31:09.366254 3779 cni.go:452] [openshift-network-diagnostics/network-check-target-445p7 595f94dd93d93e3d00d40ecce40f8a317856dc1df28945250ec41cb6ad8a3c1e network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-445p7 595f94dd93d93e3d00d40ecce40f8a317856dc1df28945250ec41cb6ad8a3c1e network default NAD default NAD key ] I0507 15:31:09.375515 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-445p7, hostIfaceName: 595f94dd93d93e3, network: default, NAD default, SandboxID: "595f94dd93d93e3d00d40ecce40f8a317856dc1df28945250ec41cb6ad8a3c1e", PCI device ID: , UID: "6f281fe7-def0-4693-aa98-ca3e3e134cf8", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0507 15:31:09.404960 3779 cni.go:473] [kube-system/global-pull-secret-syncer-6d6qj 69364d7b071ea7ca7b4c16d9fda59b4326ae81b8410ea53d095119dae85165e0 network default NAD default NAD key default] ADD finished CNI request [kube-system/global-pull-secret-syncer-6d6qj 69364d7b071ea7ca7b4c16d9fda59b4326ae81b8410ea53d095119dae85165e0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:a0:3b:ba:10:07\",\"name\":\"69364d7b071ea7c\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bfe00c7f-1183-430c-b079-acfa0f11475c\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:09.422968 3779 cni.go:473] [openshift-network-diagnostics/network-check-target-445p7 595f94dd93d93e3d00d40ecce40f8a317856dc1df28945250ec41cb6ad8a3c1e network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-445p7 595f94dd93d93e3d00d40ecce40f8a317856dc1df28945250ec41cb6ad8a3c1e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:df:24:22:ef:cf\",\"name\":\"595f94dd93d93e3\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5d2fc278-519e-46a6-a458-bf95a19b184f\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0507 15:31:17.583318 3779 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0507 15:31:18.587793 3779 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0507 15:31:18.934975 3779 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-54.ec2.internal in network "default" I0507 15:31:18.934999 3779 node_tracker.go:220] Node ip-10-0-0-54.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0507 15:31:18.935002 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-54.ec2.internal I0507 15:31:18.935021 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-54.ec2.internal: 23.682µs I0507 15:31:18.934997 3779 egressfirewall.go:723] Syncing node "ip-10-0-0-54.ec2.internal" for egress firewall E0507 15:31:18.935106 3779 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-54.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0507 15:31:18.935127 3779 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-54.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-54.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-54.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-54.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-54.ec2.internal, error parsing annotation for node ip-10-0-0-54.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0507 15:31:18.949939 3779 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-54.ec2.internal annotation: unexpected end of JSON input I0507 15:31:18.949994 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0507 15:31:18.950032 3779 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-54.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0507 15:31:18.965657 3779 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-54.ec2.internal annotation: unexpected end of JSON input I0507 15:31:18.965685 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-54.ec2.internal I0507 15:31:18.965697 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-54.ec2.internal: 14.565µs I0507 15:31:18.965732 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:18.965730 3779 egressfirewall.go:723] Syncing node "ip-10-0-0-54.ec2.internal" for egress firewall E0507 15:31:18.965773 3779 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-54.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0507 15:31:19.035370 3779 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-54.ec2.internal annotation: unexpected end of JSON input I0507 15:31:19.035401 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0507 15:31:19.035380 3779 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-54.ec2.internal annotation: unexpected end of JSON input E0507 15:31:19.035469 3779 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-54.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-54.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-54.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-54.ec2.internal" I0507 15:31:19.051028 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0507 15:31:19.133391 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-54.ec2.internal I0507 15:31:19.133409 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:19.133414 3779 egressfirewall.go:723] Syncing node "ip-10-0-0-54.ec2.internal" for egress firewall I0507 15:31:19.133416 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-54.ec2.internal: 31.553µs I0507 15:31:19.152947 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:19.387711 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:22.515137 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:26.577570 3779 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-kc6jg] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67f4d568c7-kc6jg for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:26.577706 3779 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-kc6jg I0507 15:31:26.582521 3779 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-jwvtn] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-jwvtn for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:26.582635 3779 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-jwvtn I0507 15:31:26.593012 3779 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-dd4fb9b7f-gkbhw] creating logical port openshift-ingress_router-default-dd4fb9b7f-gkbhw for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:26.593129 3779 kube.go:256] Updating pod openshift-ingress/router-default-dd4fb9b7f-gkbhw I0507 15:31:26.597068 3779 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-x8t2h] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-6944588dcc-x8t2h for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:26.597097 3779 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-kc6jg] pod update took 19.414203ms I0507 15:31:26.597113 3779 base_network_controller_pods.go:951] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-kc6jg] addLogicalPort annotation time took 19.435355ms I0507 15:31:26.597172 3779 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-x8t2h I0507 15:31:26.598305 3779 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-kc6jg] addLogicalPort took 20.757596ms, libovsdb time 870.058µs I0507 15:31:26.600888 3779 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-jwvtn] pod update took 18.273703ms I0507 15:31:26.600914 3779 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-jwvtn] addLogicalPort annotation time took 18.302661ms I0507 15:31:26.602077 3779 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-jwvtn] addLogicalPort took 19.569357ms, libovsdb time 831.279µs I0507 15:31:26.614942 3779 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-x8t2h] pod update took 17.787703ms I0507 15:31:26.614963 3779 base_network_controller_pods.go:951] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-x8t2h] addLogicalPort annotation time took 17.813109ms I0507 15:31:26.615028 3779 pod.go:62] [openshift-ingress/router-default-dd4fb9b7f-gkbhw] pod update took 21.92646ms I0507 15:31:26.615045 3779 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-dd4fb9b7f-gkbhw] addLogicalPort annotation time took 21.947546ms I0507 15:31:26.616056 3779 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-x8t2h] addLogicalPort took 19.000958ms, libovsdb time 782.157µs I0507 15:31:26.616325 3779 pods.go:271] [openshift-ingress/router-default-dd4fb9b7f-gkbhw] addLogicalPort took 23.329771ms, libovsdb time 1.035634ms I0507 15:31:26.672962 3779 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-l4q24] creating logical port openshift-network-diagnostics_network-check-source-84fd6d9d54-l4q24 for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:26.673083 3779 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-84fd6d9d54-l4q24 I0507 15:31:26.679081 3779 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-b2hsr] creating logical port openshift-console-operator_console-operator-66dbbb6468-b2hsr for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:26.679185 3779 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-b2hsr I0507 15:31:26.682390 3779 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-8hrww] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-8hrww for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:26.682483 3779 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-phjpp] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-phjpp for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:26.682502 3779 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-8hrww I0507 15:31:26.682573 3779 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-phjpp I0507 15:31:26.690434 3779 pod.go:62] [openshift-network-diagnostics/network-check-source-84fd6d9d54-l4q24] pod update took 17.372335ms I0507 15:31:26.690460 3779 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-l4q24] addLogicalPort annotation time took 17.403408ms I0507 15:31:26.690608 3779 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-789c55876f-jb6dr] creating logical port openshift-ingress_router-default-789c55876f-jb6dr for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:26.690696 3779 kube.go:256] Updating pod openshift-ingress/router-default-789c55876f-jb6dr I0507 15:31:26.691726 3779 pods.go:271] [openshift-network-diagnostics/network-check-source-84fd6d9d54-l4q24] addLogicalPort took 18.781902ms, libovsdb time 900.639µs I0507 15:31:26.699678 3779 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-b2hsr] pod update took 20.51506ms I0507 15:31:26.699704 3779 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-b2hsr] addLogicalPort annotation time took 20.542302ms I0507 15:31:26.700726 3779 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-b2hsr] addLogicalPort took 21.656114ms, libovsdb time 757.657µs I0507 15:31:26.705775 3779 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-phjpp] pod update took 23.217275ms I0507 15:31:26.705789 3779 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-8hrww] pod update took 23.310176ms I0507 15:31:26.705807 3779 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-phjpp] addLogicalPort annotation time took 23.250958ms I0507 15:31:26.705811 3779 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-8hrww] addLogicalPort annotation time took 23.334279ms I0507 15:31:26.706995 3779 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-8hrww] addLogicalPort took 24.615981ms, libovsdb time 799.517µs I0507 15:31:26.707297 3779 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-phjpp] addLogicalPort took 24.824915ms, libovsdb time 1.098927ms I0507 15:31:26.711547 3779 pod.go:62] [openshift-ingress/router-default-789c55876f-jb6dr] pod update took 20.879309ms I0507 15:31:26.711566 3779 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-789c55876f-jb6dr] addLogicalPort annotation time took 20.905086ms I0507 15:31:26.712666 3779 pods.go:271] [openshift-ingress/router-default-789c55876f-jb6dr] addLogicalPort took 22.072594ms, libovsdb time 833.497µs I0507 15:31:26.718979 3779 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-gzdlr] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-gzdlr for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:26.719081 3779 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-gzdlr I0507 15:31:26.736286 3779 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-gzdlr] pod update took 17.227293ms I0507 15:31:26.736304 3779 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-gzdlr] addLogicalPort annotation time took 17.248611ms I0507 15:31:26.737563 3779 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-gzdlr] addLogicalPort took 18.593046ms, libovsdb time 892.049µs I0507 15:31:26.846421 3779 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-m4fnx] creating logical port openshift-service-ca-operator_service-ca-operator-bc9564db4-m4fnx for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:26.846518 3779 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-bc9564db4-m4fnx I0507 15:31:26.858814 3779 pod.go:62] [openshift-service-ca-operator/service-ca-operator-bc9564db4-m4fnx] pod update took 12.321091ms I0507 15:31:26.858840 3779 base_network_controller_pods.go:951] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-m4fnx] addLogicalPort annotation time took 12.349104ms I0507 15:31:26.859935 3779 pods.go:271] [openshift-service-ca-operator/service-ca-operator-bc9564db4-m4fnx] addLogicalPort took 13.531481ms, libovsdb time 764.07µs I0507 15:31:26.885748 3779 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-js4k5] creating logical port openshift-insights_insights-operator-6dbb575d6c-js4k5 for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:26.885858 3779 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-js4k5 I0507 15:31:26.899427 3779 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-js4k5] pod update took 13.597098ms I0507 15:31:26.899449 3779 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-js4k5] addLogicalPort annotation time took 13.625481ms I0507 15:31:26.900652 3779 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-js4k5] addLogicalPort took 14.92691ms, libovsdb time 861.437µs I0507 15:31:26.980912 3779 cni.go:452] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-x8t2h a2cc3fa8fc4dc911aa343d647e2e6122786da7e62c40f8bc30c845770bead579 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-x8t2h a2cc3fa8fc4dc911aa343d647e2e6122786da7e62c40f8bc30c845770bead579 network default NAD default NAD key ] I0507 15:31:26.993256 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-6944588dcc-x8t2h, hostIfaceName: a2cc3fa8fc4dc91, network: default, NAD default, SandboxID: "a2cc3fa8fc4dc911aa343d647e2e6122786da7e62c40f8bc30c845770bead579", PCI device ID: , UID: "b5ed6685-0158-4951-8bde-dfeb46ed4a75", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0507 15:31:27.045419 3779 cni.go:473] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-x8t2h a2cc3fa8fc4dc911aa343d647e2e6122786da7e62c40f8bc30c845770bead579 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-x8t2h a2cc3fa8fc4dc911aa343d647e2e6122786da7e62c40f8bc30c845770bead579 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:65:a1:4d:0b:c8\",\"name\":\"a2cc3fa8fc4dc91\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/13cdd250-07a0-4eea-b5a3-a8639bcb6b8f\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:27.052005 3779 cni.go:452] [openshift-network-diagnostics/network-check-source-84fd6d9d54-l4q24 fe5212cb024c1f9e37539c17d3f79e2130db01d435b9e9ee849266483c6266df network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-l4q24 fe5212cb024c1f9e37539c17d3f79e2130db01d435b9e9ee849266483c6266df network default NAD default NAD key ] I0507 15:31:27.062116 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-84fd6d9d54-l4q24, hostIfaceName: fe5212cb024c1f9, network: default, NAD default, SandboxID: "fe5212cb024c1f9e37539c17d3f79e2130db01d435b9e9ee849266483c6266df", PCI device ID: , UID: "ccda09d5-623f-4dce-8cf5-103866461d14", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0507 15:31:27.068749 3779 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-b2hsr 08abcf7ac7d23d665ae11244d6fcaf5e44e916bc41ff995462599f7a0f749695 network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-b2hsr 08abcf7ac7d23d665ae11244d6fcaf5e44e916bc41ff995462599f7a0f749695 network default NAD default NAD key ] I0507 15:31:27.080073 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-b2hsr, hostIfaceName: 08abcf7ac7d23d6, network: default, NAD default, SandboxID: "08abcf7ac7d23d665ae11244d6fcaf5e44e916bc41ff995462599f7a0f749695", PCI device ID: , UID: "fb32d71a-98c6-45d4-97d8-ed765b6fc081", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0507 15:31:27.102508 3779 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-phjpp a5901253e9f9f2bd9406516baeeefc3f67c25b5cd58b0001f33f264b45bf99f7 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-phjpp a5901253e9f9f2bd9406516baeeefc3f67c25b5cd58b0001f33f264b45bf99f7 network default NAD default NAD key ] I0507 15:31:27.113943 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-phjpp, hostIfaceName: a5901253e9f9f2b, network: default, NAD default, SandboxID: "a5901253e9f9f2bd9406516baeeefc3f67c25b5cd58b0001f33f264b45bf99f7", PCI device ID: , UID: "c1e50f26-35c5-4a16-945e-0e9cf7570c87", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0507 15:31:27.115918 3779 cni.go:473] [openshift-network-diagnostics/network-check-source-84fd6d9d54-l4q24 fe5212cb024c1f9e37539c17d3f79e2130db01d435b9e9ee849266483c6266df network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-l4q24 fe5212cb024c1f9e37539c17d3f79e2130db01d435b9e9ee849266483c6266df network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:b3:62:09:c3:1a\",\"name\":\"fe5212cb024c1f9\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/980f6767-f548-4d27-b7a0-99f323cc8d77\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:27.133442 3779 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-b2hsr 08abcf7ac7d23d665ae11244d6fcaf5e44e916bc41ff995462599f7a0f749695 network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-b2hsr 08abcf7ac7d23d665ae11244d6fcaf5e44e916bc41ff995462599f7a0f749695 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:72:e5:0a:1f:da\",\"name\":\"08abcf7ac7d23d6\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/43c25359-cc2b-4c44-b342-993d11d6fff9\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:27.163077 3779 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-phjpp a5901253e9f9f2bd9406516baeeefc3f67c25b5cd58b0001f33f264b45bf99f7 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-phjpp a5901253e9f9f2bd9406516baeeefc3f67c25b5cd58b0001f33f264b45bf99f7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:f1:04:cc:3d:00\",\"name\":\"a5901253e9f9f2b\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5314b0ad-ddc8-442e-a9d9-f7e76221a4f3\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:27.270703 3779 cni.go:452] [openshift-service-ca-operator/service-ca-operator-bc9564db4-m4fnx e03333a97d0b271c6a6f50263472a7b75f2d0994d1aa54a31988afac8afbc28a network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-m4fnx e03333a97d0b271c6a6f50263472a7b75f2d0994d1aa54a31988afac8afbc28a network default NAD default NAD key ] I0507 15:31:27.281599 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-bc9564db4-m4fnx, hostIfaceName: e03333a97d0b271, network: default, NAD default, SandboxID: "e03333a97d0b271c6a6f50263472a7b75f2d0994d1aa54a31988afac8afbc28a", PCI device ID: , UID: "4d9dab10-e75e-43ca-accc-ae8935a7f69f", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0507 15:31:27.319036 3779 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-js4k5 0a50523da42a3527784e0cb02189a44adc0d69dc686ab4fd6df3db8b5d5c4d61 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-js4k5 0a50523da42a3527784e0cb02189a44adc0d69dc686ab4fd6df3db8b5d5c4d61 network default NAD default NAD key ] I0507 15:31:27.329176 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-js4k5, hostIfaceName: 0a50523da42a352, network: default, NAD default, SandboxID: "0a50523da42a3527784e0cb02189a44adc0d69dc686ab4fd6df3db8b5d5c4d61", PCI device ID: , UID: "30373fa2-9175-4ed8-abd1-e3aca2702b19", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0507 15:31:27.335287 3779 cni.go:473] [openshift-service-ca-operator/service-ca-operator-bc9564db4-m4fnx e03333a97d0b271c6a6f50263472a7b75f2d0994d1aa54a31988afac8afbc28a network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-m4fnx e03333a97d0b271c6a6f50263472a7b75f2d0994d1aa54a31988afac8afbc28a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:e9:bd:80:ca:1b\",\"name\":\"e03333a97d0b271\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c8998e75-7d2f-4745-bafc-55fbf95005b3\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:27.377429 3779 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-js4k5 0a50523da42a3527784e0cb02189a44adc0d69dc686ab4fd6df3db8b5d5c4d61 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-js4k5 0a50523da42a3527784e0cb02189a44adc0d69dc686ab4fd6df3db8b5d5c4d61 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:b4:a0:b2:af:50\",\"name\":\"0a50523da42a352\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/039bd56a-b206-4f64-8b07-fa458705916d\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:29.275294 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0507 15:31:30.614133 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0507 15:31:32.243876 3779 obj_retry.go:334] Retry object setup: *v1.Node ip-10-0-0-54.ec2.internal I0507 15:31:32.243904 3779 obj_retry.go:398] Adding new object: *v1.Node ip-10-0-0-54.ec2.internal I0507 15:31:32.244018 3779 obj_retry.go:409] Retry add failed for *v1.Node ip-10-0-0-54.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-0-54.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-54.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-54.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-54.ec2.internal W0507 15:31:32.690091 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0507 15:31:32.890131 3779 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0507 15:31:32.891177 3779 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 1.020119ms I0507 15:31:32.906452 3779 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0507 15:31:33.146164 3779 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-k47c6] creating logical port openshift-kube-storage-version-migrator_migrator-846d9b8bdc-k47c6 for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:33.146280 3779 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-846d9b8bdc-k47c6 I0507 15:31:33.159424 3779 pod.go:62] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-k47c6] pod update took 13.189724ms I0507 15:31:33.159470 3779 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-k47c6] addLogicalPort annotation time took 13.241005ms I0507 15:31:33.160739 3779 pods.go:271] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-k47c6] addLogicalPort took 14.600457ms, libovsdb time 857.307µs I0507 15:31:33.518949 3779 cni.go:452] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-k47c6 b9169dc9feaeec1e3c83323d050c4ee922769c6332f4bb7aee19b5c40b9c0bd9 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-k47c6 b9169dc9feaeec1e3c83323d050c4ee922769c6332f4bb7aee19b5c40b9c0bd9 network default NAD default NAD key ] I0507 15:31:33.528792 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-846d9b8bdc-k47c6, hostIfaceName: b9169dc9feaeec1, network: default, NAD default, SandboxID: "b9169dc9feaeec1e3c83323d050c4ee922769c6332f4bb7aee19b5c40b9c0bd9", PCI device ID: , UID: "4e4d35be-1849-4600-8bb0-e1c558cdbba7", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0507 15:31:33.549143 3779 namespace.go:98] [openshift-service-ca] adding namespace I0507 15:31:33.550297 3779 namespace.go:102] [openshift-service-ca] adding namespace took 1.131456ms I0507 15:31:33.565905 3779 namespace.go:142] [openshift-service-ca] updating namespace I0507 15:31:33.586071 3779 cni.go:473] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-k47c6 b9169dc9feaeec1e3c83323d050c4ee922769c6332f4bb7aee19b5c40b9c0bd9 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-k47c6 b9169dc9feaeec1e3c83323d050c4ee922769c6332f4bb7aee19b5c40b9c0bd9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:05:81:6b:37:e1\",\"name\":\"b9169dc9feaeec1\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3d0a3895-e6bb-4948-a726-04d8aebf444d\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:34.631627 3779 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-c6sv7 I0507 15:31:34.631660 3779 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-c6sv7 I0507 15:31:34.631708 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-2pqpf I0507 15:31:34.631699 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-mqbjf I0507 15:31:34.631724 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-2pqpf I0507 15:31:34.631734 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-mqbjf I0507 15:31:34.631752 3779 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-2pqpf, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-2pqpf: suppressed error logged: pod openshift-multus/network-metrics-daemon-2pqpf: no pod IPs found I0507 15:31:34.631776 3779 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-mqbjf, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-mqbjf: suppressed error logged: pod openshift-network-diagnostics/network-check-target-mqbjf: no pod IPs found I0507 15:31:34.631711 3779 obj_retry.go:409] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-c6sv7, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-c6sv7: suppressed error logged: pod kube-system/global-pull-secret-syncer-c6sv7: no pod IPs found I0507 15:31:35.221730 3779 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-6vvj6] creating logical port openshift-service-ca_service-ca-d9cb48497-6vvj6 for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:35.221871 3779 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-6vvj6 I0507 15:31:35.236580 3779 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-6vvj6] pod update took 14.733465ms I0507 15:31:35.236602 3779 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-6vvj6] addLogicalPort annotation time took 14.761407ms I0507 15:31:35.237831 3779 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-6vvj6] addLogicalPort took 16.123712ms, libovsdb time 839.349µs I0507 15:31:35.587900 3779 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-6vvj6 483098771f542da56364e32ef6bd9595c5150465e9ae7aa855e2256b285d24f6 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-6vvj6 483098771f542da56364e32ef6bd9595c5150465e9ae7aa855e2256b285d24f6 network default NAD default NAD key ] I0507 15:31:35.598160 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-6vvj6, hostIfaceName: 483098771f542da, network: default, NAD default, SandboxID: "483098771f542da56364e32ef6bd9595c5150465e9ae7aa855e2256b285d24f6", PCI device ID: , UID: "651e47ff-4166-4fd2-9b19-c4748e8f039e", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0507 15:31:35.647168 3779 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-6vvj6 483098771f542da56364e32ef6bd9595c5150465e9ae7aa855e2256b285d24f6 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-6vvj6 483098771f542da56364e32ef6bd9595c5150465e9ae7aa855e2256b285d24f6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:71:dd:f3:e5:a0\",\"name\":\"483098771f542da\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4b311a60-8e80-412f-8c58-c4c746d1e506\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:36.859258 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-54.ec2.internal I0507 15:31:36.859283 3779 egressfirewall.go:723] Syncing node "ip-10-0-0-54.ec2.internal" for egress firewall I0507 15:31:36.859292 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-54.ec2.internal: 45.429µs I0507 15:31:36.859312 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0507 15:31:41.116179 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0507 15:31:41.152664 3779 egressfirewall.go:723] Syncing node "ip-10-0-2-215.ec2.internal" for egress firewall I0507 15:31:41.152710 3779 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-215.ec2.internal in network "default" I0507 15:31:41.152726 3779 node_tracker.go:220] Node ip-10-0-2-215.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0507 15:31:41.152735 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-215.ec2.internal I0507 15:31:41.152747 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-215.ec2.internal: 14.305µs E0507 15:31:41.152778 3779 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-2-215.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0507 15:31:41.152803 3779 obj_retry.go:578] Failed to create *v1.Node ip-10-0-2-215.ec2.internal, error: [adding or updating remote node chassis ip-10-0-2-215.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-215.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-215.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-215.ec2.internal, error parsing annotation for node ip-10-0-2-215.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0507 15:31:41.165130 3779 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-2-215.ec2.internal annotation: unexpected end of JSON input I0507 15:31:41.165181 3779 default_network_controller.go:1025] Node "ip-10-0-2-215.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true E0507 15:31:41.165214 3779 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-2-215.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0507 15:31:41.182459 3779 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-2-215.ec2.internal annotation: unexpected end of JSON input E0507 15:31:41.182481 3779 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-2-215.ec2.internal annotation: unexpected end of JSON input I0507 15:31:41.182506 3779 default_network_controller.go:1025] Node "ip-10-0-2-215.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:41.198456 3779 default_network_controller.go:1025] Node "ip-10-0-2-215.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:41.218053 3779 egressfirewall.go:723] Syncing node "ip-10-0-2-215.ec2.internal" for egress firewall I0507 15:31:41.218115 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-215.ec2.internal I0507 15:31:41.218131 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-215.ec2.internal: 19.579µs I0507 15:31:41.218134 3779 default_network_controller.go:1025] Node "ip-10-0-2-215.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:41.398455 3779 egressfirewall.go:723] Syncing node "ip-10-0-2-215.ec2.internal" for egress firewall I0507 15:31:41.398503 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-215.ec2.internal I0507 15:31:41.398519 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-215.ec2.internal: 20.793µs I0507 15:31:41.398552 3779 default_network_controller.go:1025] Node "ip-10-0-2-215.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:41.418501 3779 default_network_controller.go:1025] Node "ip-10-0-2-215.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0507 15:31:41.487826 3779 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-5df8c6556-6kws4 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-5df8c6556-6kws4: no pod IPs found W0507 15:31:41.513815 3779 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-5849cb56c9-n9285 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-5849cb56c9-n9285: no pod IPs found I0507 15:31:41.656361 3779 default_network_controller.go:1025] Node "ip-10-0-2-215.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:42.320979 3779 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-54.ec2.internal in network "default" I0507 15:31:42.321031 3779 node_tracker.go:235] Node ip-10-0-0-54.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-54.ec2.internal" I0507 15:31:42.321046 3779 node_tracker.go:169] Node ip-10-0-0-54.ec2.internal switch + router changed, syncing services in network "default" I0507 15:31:42.321047 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "ip-10-0-0-54.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:42.321055 3779 services_controller.go:574] Full service sync requested for network=default W0507 15:31:42.321278 3779 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0507 15:31:42.321462 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0507 15:31:42.360630 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "ip-10-0-0-54.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:42.506059 3779 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-54.ec2.internal in network "default" I0507 15:31:42.506077 3779 default_network_controller.go:1025] Node "ip-10-0-0-54.ec2.internal" in remote zone "ip-10-0-0-54.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:42.506117 3779 node_tracker.go:169] Node ip-10-0-0-54.ec2.internal switch + router changed, syncing services in network "default" I0507 15:31:42.506129 3779 services_controller.go:574] Full service sync requested for network=default W0507 15:31:42.506205 3779 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0507 15:31:42.506549 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0507 15:31:42.507031 3779 zone_ic_handler.go:195] Time taken to create transit switch: 270.265µs I0507 15:31:42.507045 3779 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-54.ec2.internal for the network default I0507 15:31:42.508164 3779 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-54.ec2.internal" on network "default" took: 1.417427ms I0507 15:31:42.552516 3779 cni.go:452] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-kc6jg 01d1fb600b22b122be1eb7481704590a00f821b4dbc81f2745efd4290f79ba9e network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-kc6jg 01d1fb600b22b122be1eb7481704590a00f821b4dbc81f2745efd4290f79ba9e network default NAD default NAD key ] I0507 15:31:42.562809 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67f4d568c7-kc6jg, hostIfaceName: 01d1fb600b22b12, network: default, NAD default, SandboxID: "01d1fb600b22b122be1eb7481704590a00f821b4dbc81f2745efd4290f79ba9e", PCI device ID: , UID: "6ed1ea03-e704-4a30-942d-a1da5163a61d", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0507 15:31:42.614035 3779 cni.go:473] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-kc6jg 01d1fb600b22b122be1eb7481704590a00f821b4dbc81f2745efd4290f79ba9e network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-kc6jg 01d1fb600b22b122be1eb7481704590a00f821b4dbc81f2745efd4290f79ba9e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:b3:f7:5b:2d:ab\",\"name\":\"01d1fb600b22b12\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3393229d-75d8-46a5-bf1f-bf6ee32ff129\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:45.068984 3779 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-54.ec2.internal in network "default" I0507 15:31:45.069058 3779 node_tracker.go:169] Node ip-10-0-0-54.ec2.internal switch + router changed, syncing services in network "default" I0507 15:31:45.069060 3779 egressfirewall.go:723] Syncing node "ip-10-0-0-54.ec2.internal" for egress firewall I0507 15:31:45.068992 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-54.ec2.internal I0507 15:31:45.069074 3779 services_controller.go:574] Full service sync requested for network=default I0507 15:31:45.069082 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-54.ec2.internal: 92.897µs I0507 15:31:45.069084 3779 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-54.ec2.internal", IP: 10.0.0.54 to PMTUD blocking rules I0507 15:31:45.069096 3779 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-54.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0507 15:31:45.069269 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0507 15:31:45.069561 3779 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0507 15:31:45.075539 3779 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-54.ec2.internal. Updating no re-route policies I0507 15:31:45.088851 3779 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-54.ec2.internal", IP: 10.0.0.54 to PMTUD blocking rules I0507 15:31:45.088872 3779 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-54.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0507 15:31:51.437531 3779 default_network_controller.go:1025] Node "ip-10-0-2-215.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:53.071943 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-215.ec2.internal I0507 15:31:53.071953 3779 egressfirewall.go:723] Syncing node "ip-10-0-2-215.ec2.internal" for egress firewall I0507 15:31:53.071963 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-215.ec2.internal: 27.493µs I0507 15:31:53.071978 3779 default_network_controller.go:1025] Node "ip-10-0-2-215.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0507 15:31:57.481631 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0507 15:31:57.483401 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics I0507 15:31:57.653431 3779 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-rrqct] creating logical port openshift-console_downloads-5c54f4944f-rrqct for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:57.653527 3779 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-rrqct I0507 15:31:57.679079 3779 pod.go:62] [openshift-console/downloads-5c54f4944f-rrqct] pod update took 25.580945ms I0507 15:31:57.679102 3779 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-rrqct] addLogicalPort annotation time took 25.609584ms I0507 15:31:57.680266 3779 pods.go:271] [openshift-console/downloads-5c54f4944f-rrqct] addLogicalPort took 26.854265ms, libovsdb time 815.594µs W0507 15:31:57.689648 3779 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-579668488c-4g9k9 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-579668488c-4g9k9: no pod IPs found I0507 15:31:57.718068 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-54.ec2.internal I0507 15:31:57.718117 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-54.ec2.internal: 56.288µs I0507 15:31:57.749926 3779 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-dzlvj] creating logical port openshift-insights_insights-runtime-extractor-dzlvj for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:57.750040 3779 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-dzlvj I0507 15:31:57.781068 3779 pod.go:62] [openshift-insights/insights-runtime-extractor-dzlvj] pod update took 31.062573ms I0507 15:31:57.781090 3779 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-dzlvj] addLogicalPort annotation time took 31.090722ms I0507 15:31:57.782313 3779 pods.go:271] [openshift-insights/insights-runtime-extractor-dzlvj] addLogicalPort took 32.401883ms, libovsdb time 859.343µs I0507 15:31:57.939221 3779 default_network_controller.go:1025] Node "ip-10-0-2-215.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:31:58.101077 3779 cni.go:452] [openshift-insights/insights-runtime-extractor-dzlvj a713698cd97dcd35becd81f9eec25133707d647bd2ac1d937bb31eaeac15c94f network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-dzlvj a713698cd97dcd35becd81f9eec25133707d647bd2ac1d937bb31eaeac15c94f network default NAD default NAD key ] I0507 15:31:58.110538 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-dzlvj, hostIfaceName: a713698cd97dcd3, network: default, NAD default, SandboxID: "a713698cd97dcd35becd81f9eec25133707d647bd2ac1d937bb31eaeac15c94f", PCI device ID: , UID: "669e70fd-5900-4060-8ba5-f49bd6315b9a", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0507 15:31:58.162448 3779 cni.go:473] [openshift-insights/insights-runtime-extractor-dzlvj a713698cd97dcd35becd81f9eec25133707d647bd2ac1d937bb31eaeac15c94f network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-dzlvj a713698cd97dcd35becd81f9eec25133707d647bd2ac1d937bb31eaeac15c94f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:b8:6a:82:48:a7\",\"name\":\"a713698cd97dcd3\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f0333f21-f0e2-4a32-a9c3-4cd6d08e38a8\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:58.311326 3779 cni.go:452] [openshift-console/downloads-5c54f4944f-rrqct e63ee5c8fa1f1a41bc330a6d7e95f925d92ceee1302ee25fe81f5890fd8f523b network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-rrqct e63ee5c8fa1f1a41bc330a6d7e95f925d92ceee1302ee25fe81f5890fd8f523b network default NAD default NAD key ] I0507 15:31:58.321352 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-rrqct, hostIfaceName: e63ee5c8fa1f1a4, network: default, NAD default, SandboxID: "e63ee5c8fa1f1a41bc330a6d7e95f925d92ceee1302ee25fe81f5890fd8f523b", PCI device ID: , UID: "95a39df0-ed35-4d0f-a208-009b5320d24c", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0507 15:31:58.371168 3779 cni.go:473] [openshift-console/downloads-5c54f4944f-rrqct e63ee5c8fa1f1a41bc330a6d7e95f925d92ceee1302ee25fe81f5890fd8f523b network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-rrqct e63ee5c8fa1f1a41bc330a6d7e95f925d92ceee1302ee25fe81f5890fd8f523b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:16:b4:f2:47:29\",\"name\":\"e63ee5c8fa1f1a4\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5a88c18d-e74a-419a-8e73-b80273629672\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:58.755077 3779 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-jwvtn 7de61e8c9b2553b11eea5328570575ce969f34b8b22d0b518d067f7e4984af6e network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-jwvtn 7de61e8c9b2553b11eea5328570575ce969f34b8b22d0b518d067f7e4984af6e network default NAD default NAD key ] I0507 15:31:58.761440 3779 cni.go:452] [openshift-ingress/router-default-dd4fb9b7f-gkbhw 29b18b7c23b873fbeb27a80eb924e2f6dba04b7eda3df989eb00077671f83eec network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-dd4fb9b7f-gkbhw 29b18b7c23b873fbeb27a80eb924e2f6dba04b7eda3df989eb00077671f83eec network default NAD default NAD key ] I0507 15:31:58.765729 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-jwvtn, hostIfaceName: 7de61e8c9b2553b, network: default, NAD default, SandboxID: "7de61e8c9b2553b11eea5328570575ce969f34b8b22d0b518d067f7e4984af6e", PCI device ID: , UID: "d23e28f7-70e0-40e4-9256-9cf79af79fe4", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0507 15:31:58.765758 3779 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5849cb56c9-wswmc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:31:58.765784 3779 pods.go:176] Deleting pod: openshift-image-registry/image-registry-5849cb56c9-wswmc I0507 15:31:58.767489 3779 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-5849cb56c9-wswmc, ips: 10.128.0.6 I0507 15:31:58.767528 3779 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5849cb56c9-wswmc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:31:58.776267 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-dd4fb9b7f-gkbhw, hostIfaceName: 29b18b7c23b873f, network: default, NAD default, SandboxID: "29b18b7c23b873fbeb27a80eb924e2f6dba04b7eda3df989eb00077671f83eec", PCI device ID: , UID: "e7ae83d1-a8eb-41ec-9f16-8d712ff83e35", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0507 15:31:58.780929 3779 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5849cb56c9-wswmc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:31:58.780953 3779 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5849cb56c9-wswmc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:31:58.781005 3779 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6569f87b65-qhn6j] creating logical port openshift-image-registry_image-registry-6569f87b65-qhn6j for pod on switch ip-10-0-1-119.ec2.internal I0507 15:31:58.781099 3779 kube.go:256] Updating pod openshift-image-registry/image-registry-6569f87b65-qhn6j I0507 15:31:58.783695 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5849cb56c9-wswmc I0507 15:31:58.783719 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5849cb56c9-wswmc I0507 15:31:58.813737 3779 pod.go:62] [openshift-image-registry/image-registry-6569f87b65-qhn6j] pod update took 32.664654ms I0507 15:31:58.813768 3779 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-6569f87b65-qhn6j] addLogicalPort annotation time took 32.700595ms I0507 15:31:58.815051 3779 pods.go:271] [openshift-image-registry/image-registry-6569f87b65-qhn6j] addLogicalPort took 34.063878ms, libovsdb time 880.114µs I0507 15:31:58.820946 3779 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-jwvtn 7de61e8c9b2553b11eea5328570575ce969f34b8b22d0b518d067f7e4984af6e network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-jwvtn 7de61e8c9b2553b11eea5328570575ce969f34b8b22d0b518d067f7e4984af6e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:26:c0:d2:a3:f5\",\"name\":\"7de61e8c9b2553b\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dec54480-8e71-44c2-b64b-ce6489cd32d5\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:58.843128 3779 cni.go:473] [openshift-ingress/router-default-dd4fb9b7f-gkbhw 29b18b7c23b873fbeb27a80eb924e2f6dba04b7eda3df989eb00077671f83eec network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-dd4fb9b7f-gkbhw 29b18b7c23b873fbeb27a80eb924e2f6dba04b7eda3df989eb00077671f83eec network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:b4:40:d1:7f:0b\",\"name\":\"29b18b7c23b873f\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ccc8d978-70f1-4776-bfa7-c3d3f9a3689f\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:58.866028 3779 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-8hrww 5a7810f0f5944572804f9e0e281fd36d11508c234e5c68a93c3be99a636335d6 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-8hrww 5a7810f0f5944572804f9e0e281fd36d11508c234e5c68a93c3be99a636335d6 network default NAD default NAD key ] I0507 15:31:58.869550 3779 cni.go:452] [openshift-ingress/router-default-789c55876f-jb6dr 74572ab0afe8a34f827ff42ad8eba7a47703c0ef207bbca82d549365b0eae765 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-789c55876f-jb6dr 74572ab0afe8a34f827ff42ad8eba7a47703c0ef207bbca82d549365b0eae765 network default NAD default NAD key ] I0507 15:31:58.880078 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-8hrww, hostIfaceName: 5a7810f0f594457, network: default, NAD default, SandboxID: "5a7810f0f5944572804f9e0e281fd36d11508c234e5c68a93c3be99a636335d6", PCI device ID: , UID: "422b39c5-1f06-401d-b1bd-394de96d5edf", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0507 15:31:58.882311 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-789c55876f-jb6dr, hostIfaceName: 74572ab0afe8a34, network: default, NAD default, SandboxID: "74572ab0afe8a34f827ff42ad8eba7a47703c0ef207bbca82d549365b0eae765", PCI device ID: , UID: "f77cae34-be64-435d-9a1a-0fb8bcebaf77", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0507 15:31:58.900397 3779 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-gzdlr 1fe4463e4feb329b3776c53a0cd6e6ad5d3c56b6f29eba6431eec8c72c0c9269 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-gzdlr 1fe4463e4feb329b3776c53a0cd6e6ad5d3c56b6f29eba6431eec8c72c0c9269 network default NAD default NAD key ] I0507 15:31:58.914773 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-gzdlr, hostIfaceName: 1fe4463e4feb329, network: default, NAD default, SandboxID: "1fe4463e4feb329b3776c53a0cd6e6ad5d3c56b6f29eba6431eec8c72c0c9269", PCI device ID: , UID: "04afb369-5698-4c25-980a-9d0018d0490f", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0507 15:31:58.942129 3779 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-8hrww 5a7810f0f5944572804f9e0e281fd36d11508c234e5c68a93c3be99a636335d6 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-8hrww 5a7810f0f5944572804f9e0e281fd36d11508c234e5c68a93c3be99a636335d6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:6e:88:19:9c:8f\",\"name\":\"5a7810f0f594457\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bdb94305-4262-41e6-bb6c-bc2059cf6ab4\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:58.961066 3779 cni.go:473] [openshift-ingress/router-default-789c55876f-jb6dr 74572ab0afe8a34f827ff42ad8eba7a47703c0ef207bbca82d549365b0eae765 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-789c55876f-jb6dr 74572ab0afe8a34f827ff42ad8eba7a47703c0ef207bbca82d549365b0eae765 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:99:e4:06:cf:cc\",\"name\":\"74572ab0afe8a34\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e899c57e-717a-4e95-a674-b5dde4e668e3\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:31:58.981470 3779 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-gzdlr 1fe4463e4feb329b3776c53a0cd6e6ad5d3c56b6f29eba6431eec8c72c0c9269 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-gzdlr 1fe4463e4feb329b3776c53a0cd6e6ad5d3c56b6f29eba6431eec8c72c0c9269 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:38:a6:0c:e6:ca\",\"name\":\"1fe4463e4feb329\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4b6195b7-84ac-4627-9bbf-4ab5cb86ef38\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0507 15:31:59.044085 3779 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0507 15:31:59.044108 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0507 15:31:59.047988 3779 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:31:59.048004 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:31:59.048011 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:31:59.077556 3779 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0507 15:31:59.142817 3779 cni.go:452] [openshift-image-registry/image-registry-6569f87b65-qhn6j 936b2177fcb76a986b3bcaf3b385b5a83d30e3e4391401a73cc6ee6fbb1f7ef2 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-6569f87b65-qhn6j 936b2177fcb76a986b3bcaf3b385b5a83d30e3e4391401a73cc6ee6fbb1f7ef2 network default NAD default NAD key ] I0507 15:31:59.153714 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6569f87b65-qhn6j, hostIfaceName: 936b2177fcb76a9, network: default, NAD default, SandboxID: "936b2177fcb76a986b3bcaf3b385b5a83d30e3e4391401a73cc6ee6fbb1f7ef2", PCI device ID: , UID: "bde71fc4-bf99-41b2-90cc-d666421dc12e", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0507 15:31:59.209379 3779 cni.go:473] [openshift-image-registry/image-registry-6569f87b65-qhn6j 936b2177fcb76a986b3bcaf3b385b5a83d30e3e4391401a73cc6ee6fbb1f7ef2 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-6569f87b65-qhn6j 936b2177fcb76a986b3bcaf3b385b5a83d30e3e4391401a73cc6ee6fbb1f7ef2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:8a:5b:43:57:c2\",\"name\":\"936b2177fcb76a9\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/48f80de8-1a6a-411b-806d-f6c4b070ce4f\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0507 15:31:59.769982 3779 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0507 15:31:59.770060 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0507 15:31:59.771752 3779 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:31:59.771768 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:31:59.771774 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:31:59.791948 3779 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:31:59.792030 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:31:59.792060 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:31:59.794422 3779 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0507 15:31:59.794491 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0507 15:31:59.818650 3779 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0507 15:32:00.062420 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0507 15:32:00.062520 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0507 15:32:00.062423 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0507 15:32:00.062685 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0507 15:32:00.062716 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0507 15:32:00.062727 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0507 15:32:00.062797 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:32:00.062809 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:32:00.062817 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:32:00.062835 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:32:00.062846 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:32:00.062851 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:32:00.787201 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:32:00.787230 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:32:00.787240 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0507 15:32:00.787462 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0507 15:32:00.787479 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0507 15:32:00.913445 3779 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0507 15:32:01.022328 3779 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0507 15:32:01.943830 3779 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pnzlz] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-pnzlz for pod on switch ip-10-0-1-119.ec2.internal I0507 15:32:01.943926 3779 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pnzlz I0507 15:32:01.961560 3779 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pnzlz] pod update took 17.656655ms I0507 15:32:01.961584 3779 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pnzlz] addLogicalPort annotation time took 17.68648ms I0507 15:32:01.962996 3779 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pnzlz] addLogicalPort took 19.185353ms, libovsdb time 1.009258ms I0507 15:32:02.243761 3779 obj_retry.go:334] Retry object setup: *v1.Node ip-10-0-2-215.ec2.internal I0507 15:32:02.243790 3779 obj_retry.go:398] Adding new object: *v1.Node ip-10-0-2-215.ec2.internal I0507 15:32:02.243915 3779 obj_retry.go:409] Retry add failed for *v1.Node ip-10-0-2-215.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-2-215.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-215.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-215.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-215.ec2.internal I0507 15:32:02.377928 3779 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pnzlz 8b86290f6b33b486a8c3c52cdb1e86c953c9183ef3acfe6aa8c2a3b2faacd6ce network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pnzlz 8b86290f6b33b486a8c3c52cdb1e86c953c9183ef3acfe6aa8c2a3b2faacd6ce network default NAD default NAD key ] I0507 15:32:02.389749 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-pnzlz, hostIfaceName: 8b86290f6b33b48, network: default, NAD default, SandboxID: "8b86290f6b33b486a8c3c52cdb1e86c953c9183ef3acfe6aa8c2a3b2faacd6ce", PCI device ID: , UID: "ddbe9e90-1e09-4640-a431-b8e3e6ff802a", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0507 15:32:02.446955 3779 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pnzlz 8b86290f6b33b486a8c3c52cdb1e86c953c9183ef3acfe6aa8c2a3b2faacd6ce network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-pnzlz 8b86290f6b33b486a8c3c52cdb1e86c953c9183ef3acfe6aa8c2a3b2faacd6ce network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:9b:46:4e:47:ad\",\"name\":\"8b86290f6b33b48\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/27ee1f21-e7e7-4e4e-9258-0551233e8e51\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0507 15:32:03.083106 3779 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0507 15:32:03.083125 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0507 15:32:03.083131 3779 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0507 15:32:04.631893 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-m7lkc I0507 15:32:04.631907 3779 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-9qjvx I0507 15:32:04.631922 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-m7lkc I0507 15:32:04.631931 3779 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-9qjvx I0507 15:32:04.631960 3779 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-m7lkc, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-m7lkc: suppressed error logged: pod openshift-multus/network-metrics-daemon-m7lkc: no pod IPs found I0507 15:32:04.631971 3779 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-9qjvx, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-9qjvx: suppressed error logged: pod openshift-network-diagnostics/network-check-target-9qjvx: no pod IPs found I0507 15:32:04.631890 3779 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-jqzbf I0507 15:32:04.631986 3779 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-jqzbf I0507 15:32:04.632002 3779 obj_retry.go:409] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-jqzbf, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-jqzbf: suppressed error logged: pod kube-system/global-pull-secret-syncer-jqzbf: no pod IPs found W0507 15:32:04.798830 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0507 15:32:04.818585 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0507 15:32:05.745187 3779 base_network_controller_pods.go:487] [default/openshift-console/console-54cdc46f5-dqlrx] creating logical port openshift-console_console-54cdc46f5-dqlrx for pod on switch ip-10-0-1-119.ec2.internal I0507 15:32:05.745328 3779 kube.go:256] Updating pod openshift-console/console-54cdc46f5-dqlrx I0507 15:32:05.762273 3779 pod.go:62] [openshift-console/console-54cdc46f5-dqlrx] pod update took 16.970892ms I0507 15:32:05.762302 3779 base_network_controller_pods.go:951] [default/openshift-console/console-54cdc46f5-dqlrx] addLogicalPort annotation time took 17.011825ms I0507 15:32:05.763699 3779 pods.go:271] [openshift-console/console-54cdc46f5-dqlrx] addLogicalPort took 18.534653ms, libovsdb time 972.037µs I0507 15:32:06.111806 3779 cni.go:452] [openshift-console/console-54cdc46f5-dqlrx 85a78049810ac08235b219339603a6a13f486e460c5525e9ee877c6016faedfb network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-54cdc46f5-dqlrx 85a78049810ac08235b219339603a6a13f486e460c5525e9ee877c6016faedfb network default NAD default NAD key ] I0507 15:32:06.122858 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-54cdc46f5-dqlrx, hostIfaceName: 85a78049810ac08, network: default, NAD default, SandboxID: "85a78049810ac08235b219339603a6a13f486e460c5525e9ee877c6016faedfb", PCI device ID: , UID: "3b3adb78-d485-4f58-9f03-26da6281a920", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0507 15:32:06.183125 3779 cni.go:473] [openshift-console/console-54cdc46f5-dqlrx 85a78049810ac08235b219339603a6a13f486e460c5525e9ee877c6016faedfb network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-54cdc46f5-dqlrx 85a78049810ac08235b219339603a6a13f486e460c5525e9ee877c6016faedfb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:30:c0:13:b8:a4\",\"name\":\"85a78049810ac08\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/21a755e6-42a6-445a-9807-6089ac98d6d6\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:32:08.702751 3779 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-215.ec2.internal in network "default" I0507 15:32:08.702793 3779 node_tracker.go:235] Node ip-10-0-2-215.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-215.ec2.internal" I0507 15:32:08.702785 3779 default_network_controller.go:1025] Node "ip-10-0-2-215.ec2.internal" in remote zone "ip-10-0-2-215.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0507 15:32:08.702809 3779 node_tracker.go:169] Node ip-10-0-2-215.ec2.internal switch + router changed, syncing services in network "default" I0507 15:32:08.702817 3779 services_controller.go:574] Full service sync requested for network=default W0507 15:32:08.703106 3779 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0507 15:32:08.703137 3779 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0507 15:32:08.703143 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0507 15:32:08.703147 3779 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0507 15:32:08.851744 3779 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-215.ec2.internal in network "default" I0507 15:32:08.851809 3779 node_tracker.go:169] Node ip-10-0-2-215.ec2.internal switch + router changed, syncing services in network "default" I0507 15:32:08.851823 3779 services_controller.go:574] Full service sync requested for network=default I0507 15:32:08.851827 3779 default_network_controller.go:1025] Node "ip-10-0-2-215.ec2.internal" in remote zone "ip-10-0-2-215.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0507 15:32:08.852097 3779 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0507 15:32:08.852163 3779 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0507 15:32:08.852175 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0507 15:32:08.852181 3779 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0507 15:32:08.852882 3779 zone_ic_handler.go:195] Time taken to create transit switch: 270.428µs I0507 15:32:08.852898 3779 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-215.ec2.internal for the network default I0507 15:32:08.853893 3779 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-215.ec2.internal" on network "default" took: 1.292769ms I0507 15:32:11.414539 3779 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-215.ec2.internal in network "default" I0507 15:32:11.414578 3779 egressfirewall.go:723] Syncing node "ip-10-0-2-215.ec2.internal" for egress firewall I0507 15:32:11.414582 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-215.ec2.internal I0507 15:32:11.414602 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-215.ec2.internal: 24.589µs I0507 15:32:11.414615 3779 node_tracker.go:169] Node ip-10-0-2-215.ec2.internal switch + router changed, syncing services in network "default" I0507 15:32:11.414632 3779 services_controller.go:574] Full service sync requested for network=default I0507 15:32:11.414657 3779 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-215.ec2.internal", IP: 10.0.2.215 to PMTUD blocking rules I0507 15:32:11.414672 3779 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-215.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0507 15:32:11.415198 3779 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0507 15:32:11.415259 3779 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0507 15:32:11.415274 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0507 15:32:11.415281 3779 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0507 15:32:11.420802 3779 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-215.ec2.internal. Updating no re-route policies I0507 15:32:11.432008 3779 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-215.ec2.internal", IP: 10.0.2.215 to PMTUD blocking rules I0507 15:32:11.432025 3779 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-215.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0507 15:32:14.848827 3779 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-215.ec2.internal I0507 15:32:14.848869 3779 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-215.ec2.internal: 49.989µs I0507 15:32:15.455599 3779 cni.go:452] [openshift-dns/dns-default-mtm7b 89bcba8edd69acd825c45f32c47ba939e60550b35076755a77692705735b080a network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-mtm7b 89bcba8edd69acd825c45f32c47ba939e60550b35076755a77692705735b080a network default NAD default NAD key ] I0507 15:32:15.465252 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-mtm7b, hostIfaceName: 89bcba8edd69acd, network: default, NAD default, SandboxID: "89bcba8edd69acd825c45f32c47ba939e60550b35076755a77692705735b080a", PCI device ID: , UID: "69b63391-95a1-4a7d-b3f7-70363379a00e", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0507 15:32:15.521625 3779 cni.go:473] [openshift-dns/dns-default-mtm7b 89bcba8edd69acd825c45f32c47ba939e60550b35076755a77692705735b080a network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-mtm7b 89bcba8edd69acd825c45f32c47ba939e60550b35076755a77692705735b080a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:df:3d:ee:fa:95\",\"name\":\"89bcba8edd69acd\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/207b5d45-d786-4284-b266-c34d16ffea33\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0507 15:32:15.847415 3779 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0507 15:32:15.869451 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0507 15:32:15.894566 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0507 15:32:16.166328 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0507 15:32:16.266665 3779 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0507 15:32:16.860489 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0507 15:32:20.055411 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0507 15:32:20.055437 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0507 15:32:20.837610 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0507 15:32:24.258002 3779 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0507 15:32:24.258019 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0507 15:32:24.258025 3779 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0507 15:32:24.350667 3779 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0507 15:32:24.350686 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0507 15:32:24.350691 3779 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0507 15:32:25.313312 3779 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-7ffb486d4f-pnkn9 IP address from the namespace address-set, err: pod openshift-console/console-7ffb486d4f-pnkn9: no pod IPs found W0507 15:32:27.869008 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0507 15:32:27.869032 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0507 15:32:27.869041 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0507 15:32:28.373779 3779 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0507 15:32:28.373793 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0507 15:32:28.390051 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0507 15:32:28.390069 3779 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0507 15:32:28.390075 3779 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0507 15:32:28.390078 3779 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0507 15:32:29.361054 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0507 15:32:29.410499 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0507 15:32:30.154237 3779 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0507 15:32:30.300654 3779 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0507 15:32:30.300672 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0507 15:32:32.320158 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0507 15:32:33.256681 3779 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0507 15:32:33.256701 3779 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0507 15:32:33.256709 3779 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0507 15:32:33.256715 3779 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0507 15:32:41.409900 3779 obj_retry.go:498] Detected object openshift-console/console-54cdc46f5-6sbrf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:32:41.410459 3779 obj_retry.go:498] Detected object openshift-console/console-54cdc46f5-6sbrf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:32:42.309325 3779 obj_retry.go:492] Detected object openshift-console/console-54cdc46f5-6sbrf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:32:42.309373 3779 obj_retry.go:492] Detected object openshift-console/console-54cdc46f5-6sbrf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:32:42.316975 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-54cdc46f5-6sbrf I0507 15:32:42.317000 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-54cdc46f5-6sbrf I0507 15:32:45.408743 3779 cni.go:452] [openshift-multus/network-metrics-daemon-mpbjx a5a2b34da46b737adaa3b5810339612f8516e708ad2fb2684daa29bdfde394f7 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-mpbjx a5a2b34da46b737adaa3b5810339612f8516e708ad2fb2684daa29bdfde394f7 network default NAD default NAD key ] I0507 15:32:45.417829 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-mpbjx, hostIfaceName: a5a2b34da46b737, network: default, NAD default, SandboxID: "a5a2b34da46b737adaa3b5810339612f8516e708ad2fb2684daa29bdfde394f7", PCI device ID: , UID: "35920096-b0a9-4618-91b9-30b3baaf3274", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0507 15:32:45.470627 3779 cni.go:473] [openshift-multus/network-metrics-daemon-mpbjx a5a2b34da46b737adaa3b5810339612f8516e708ad2fb2684daa29bdfde394f7 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-mpbjx a5a2b34da46b737adaa3b5810339612f8516e708ad2fb2684daa29bdfde394f7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:04:cb:80:62:37\",\"name\":\"a5a2b34da46b737\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/980e2b1c-a818-49b5-b7e1-30de1ac1b494\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:32:46.215174 3779 obj_retry.go:498] Detected object openshift-image-registry/image-registry-579668488c-n86dg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:32:46.215802 3779 obj_retry.go:498] Detected object openshift-image-registry/image-registry-579668488c-n86dg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:32:46.611218 3779 obj_retry.go:492] Detected object openshift-image-registry/image-registry-579668488c-n86dg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:32:46.611245 3779 obj_retry.go:492] Detected object openshift-image-registry/image-registry-579668488c-n86dg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:32:46.618540 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-579668488c-n86dg I0507 15:32:46.618574 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-579668488c-n86dg W0507 15:32:47.992964 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0507 15:32:47.993042 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0507 15:32:47.993097 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0507 15:32:49.002382 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0507 15:32:52.678977 3779 obj_retry.go:498] Detected object openshift-console/console-7ffb486d4f-dt2bx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:32:52.679592 3779 obj_retry.go:498] Detected object openshift-console/console-7ffb486d4f-dt2bx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:32:53.500547 3779 obj_retry.go:492] Detected object openshift-console/console-7ffb486d4f-dt2bx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:32:53.500572 3779 obj_retry.go:492] Detected object openshift-console/console-7ffb486d4f-dt2bx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:32:53.504053 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7ffb486d4f-dt2bx I0507 15:32:53.504086 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7ffb486d4f-dt2bx W0507 15:33:03.066311 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0507 15:33:03.066380 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0507 15:33:03.066399 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0507 15:33:04.066962 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0507 15:33:08.083698 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0507 15:33:08.083711 3779 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0507 15:33:08.083753 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0507 15:33:09.103299 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0507 15:33:16.213766 3779 cni.go:452] [openshift-ingress/router-default-dd4fb9b7f-gkbhw 29b18b7c23b873fbeb27a80eb924e2f6dba04b7eda3df989eb00077671f83eec network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-dd4fb9b7f-gkbhw 29b18b7c23b873fbeb27a80eb924e2f6dba04b7eda3df989eb00077671f83eec network default NAD default NAD key ] I0507 15:33:16.299055 3779 cni.go:473] [openshift-ingress/router-default-dd4fb9b7f-gkbhw 29b18b7c23b873fbeb27a80eb924e2f6dba04b7eda3df989eb00077671f83eec network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-dd4fb9b7f-gkbhw 29b18b7c23b873fbeb27a80eb924e2f6dba04b7eda3df989eb00077671f83eec network default NAD default NAD key default], result "{}", err I0507 15:33:16.347252 3779 obj_retry.go:498] Detected object openshift-ingress/router-default-dd4fb9b7f-gkbhw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:33:16.347284 3779 pods.go:176] Deleting pod: openshift-ingress/router-default-dd4fb9b7f-gkbhw I0507 15:33:16.348627 3779 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-dd4fb9b7f-gkbhw, ips: 10.128.0.13 I0507 15:33:16.348659 3779 obj_retry.go:498] Detected object openshift-ingress/router-default-dd4fb9b7f-gkbhw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:33:17.126110 3779 obj_retry.go:492] Detected object openshift-ingress/router-default-dd4fb9b7f-gkbhw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:33:17.126136 3779 obj_retry.go:492] Detected object openshift-ingress/router-default-dd4fb9b7f-gkbhw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:33:17.129472 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-dd4fb9b7f-gkbhw I0507 15:33:17.129495 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-dd4fb9b7f-gkbhw I0507 15:33:17.554921 3779 cni.go:452] [openshift-console/console-54cdc46f5-dqlrx 85a78049810ac08235b219339603a6a13f486e460c5525e9ee877c6016faedfb network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-54cdc46f5-dqlrx 85a78049810ac08235b219339603a6a13f486e460c5525e9ee877c6016faedfb network default NAD default NAD key ] I0507 15:33:17.625687 3779 cni.go:473] [openshift-console/console-54cdc46f5-dqlrx 85a78049810ac08235b219339603a6a13f486e460c5525e9ee877c6016faedfb network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-54cdc46f5-dqlrx 85a78049810ac08235b219339603a6a13f486e460c5525e9ee877c6016faedfb network default NAD default NAD key default], result "{}", err I0507 15:33:17.674084 3779 obj_retry.go:498] Detected object openshift-console/console-54cdc46f5-dqlrx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:33:17.674117 3779 pods.go:176] Deleting pod: openshift-console/console-54cdc46f5-dqlrx I0507 15:33:17.675718 3779 pods.go:236] Attempting to release IPs for pod: openshift-console/console-54cdc46f5-dqlrx, ips: 10.128.0.29 I0507 15:33:17.675753 3779 obj_retry.go:498] Detected object openshift-console/console-54cdc46f5-dqlrx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:33:18.128383 3779 obj_retry.go:492] Detected object openshift-console/console-54cdc46f5-dqlrx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:33:18.128410 3779 obj_retry.go:492] Detected object openshift-console/console-54cdc46f5-dqlrx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:33:18.135196 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-54cdc46f5-dqlrx I0507 15:33:18.135225 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-54cdc46f5-dqlrx I0507 15:33:42.539467 3779 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:33:42.539992 3779 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:33:42.855263 3779 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:33:42.855288 3779 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:33:42.861667 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0507 15:33:42.861689 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0507 15:33:48.865831 3779 base_network_controller_pods.go:487] [default/openshift-console/console-686b9d8c96-465kz] creating logical port openshift-console_console-686b9d8c96-465kz for pod on switch ip-10-0-1-119.ec2.internal I0507 15:33:48.865949 3779 kube.go:256] Updating pod openshift-console/console-686b9d8c96-465kz I0507 15:33:48.900678 3779 pod.go:62] [openshift-console/console-686b9d8c96-465kz] pod update took 34.766321ms I0507 15:33:48.900706 3779 base_network_controller_pods.go:951] [default/openshift-console/console-686b9d8c96-465kz] addLogicalPort annotation time took 34.799118ms I0507 15:33:48.901973 3779 pods.go:271] [openshift-console/console-686b9d8c96-465kz] addLogicalPort took 36.160263ms, libovsdb time 889.302µs I0507 15:33:49.231075 3779 cni.go:452] [openshift-console/console-686b9d8c96-465kz e868baddf32dadabc9ae3a79348b706b4b3c804df6e4cbf785b62c84e410356c network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-686b9d8c96-465kz e868baddf32dadabc9ae3a79348b706b4b3c804df6e4cbf785b62c84e410356c network default NAD default NAD key ] I0507 15:33:49.240544 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-686b9d8c96-465kz, hostIfaceName: e868baddf32dada, network: default, NAD default, SandboxID: "e868baddf32dadabc9ae3a79348b706b4b3c804df6e4cbf785b62c84e410356c", PCI device ID: , UID: "0d77a51c-0efb-4a8f-976b-c225f08be813", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0507 15:33:49.292746 3779 cni.go:473] [openshift-console/console-686b9d8c96-465kz e868baddf32dadabc9ae3a79348b706b4b3c804df6e4cbf785b62c84e410356c network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-686b9d8c96-465kz e868baddf32dadabc9ae3a79348b706b4b3c804df6e4cbf785b62c84e410356c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:b8:35:d3:d5:a5\",\"name\":\"e868baddf32dada\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cf918efd-3e44-44b6-8bdc-0c2bc1f1a5b1\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:33:53.929384 3779 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:33:53.929981 3779 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:33:54.628401 3779 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:33:54.628434 3779 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:33:54.635375 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0507 15:33:54.635404 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 W0507 15:33:59.310189 3779 util.go:959] Failed to build node endpoints for node ip-10-0-1-119.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0507 15:33:59.667856 3779 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0507 15:34:05.124574 3779 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:34:05.125090 3779 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:34:05.790246 3779 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:05.790274 3779 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:05.797117 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0507 15:34:05.797147 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0507 15:34:11.368279 3779 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf 4ed842bccfef35deec86440aa5e34c856e17289401e54adacd710775473bf3e0 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf 4ed842bccfef35deec86440aa5e34c856e17289401e54adacd710775473bf3e0 network default NAD default NAD key ] I0507 15:34:11.458701 3779 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf 4ed842bccfef35deec86440aa5e34c856e17289401e54adacd710775473bf3e0 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf 4ed842bccfef35deec86440aa5e34c856e17289401e54adacd710775473bf3e0 network default NAD default NAD key default], result "{}", err I0507 15:34:11.501545 3779 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:34:11.501582 3779 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf I0507 15:34:11.502935 3779 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf, ips: 10.128.0.9 I0507 15:34:11.502966 3779 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:34:12.392442 3779 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:12.392473 3779 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29636130-79wpf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:14.166451 3779 obj_retry.go:498] Detected object openshift-console/console-5cc6b85768-jl994 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:34:14.167023 3779 obj_retry.go:498] Detected object openshift-console/console-5cc6b85768-jl994 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:34:14.851963 3779 obj_retry.go:492] Detected object openshift-console/console-5cc6b85768-jl994 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:14.851987 3779 obj_retry.go:492] Detected object openshift-console/console-5cc6b85768-jl994 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:14.858873 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5cc6b85768-jl994 I0507 15:34:14.858900 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5cc6b85768-jl994 I0507 15:34:18.388777 3779 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl] creating logical port openshift-deployment-validation-operator_99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl for pod on switch ip-10-0-1-119.ec2.internal I0507 15:34:18.388875 3779 kube.go:256] Updating pod openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl I0507 15:34:18.401910 3779 pod.go:62] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl] pod update took 13.064093ms I0507 15:34:18.401933 3779 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl] addLogicalPort annotation time took 13.09397ms I0507 15:34:18.403175 3779 pods.go:271] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl] addLogicalPort took 14.415978ms, libovsdb time 872.625µs I0507 15:34:18.759895 3779 cni.go:452] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl 0d7c7298bf6bfdd3e058ad8cec67a852d5fa2885549a15170d850ea14a0f4fbe network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl 0d7c7298bf6bfdd3e058ad8cec67a852d5fa2885549a15170d850ea14a0f4fbe network default NAD default NAD key ] I0507 15:34:18.769506 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: 99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl, hostIfaceName: 0d7c7298bf6bfdd, network: default, NAD default, SandboxID: "0d7c7298bf6bfdd3e058ad8cec67a852d5fa2885549a15170d850ea14a0f4fbe", PCI device ID: , UID: "691b1db7-fc6a-4cfa-aba8-fc6c841dc7cc", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0507 15:34:18.821877 3779 cni.go:473] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl 0d7c7298bf6bfdd3e058ad8cec67a852d5fa2885549a15170d850ea14a0f4fbe network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl 0d7c7298bf6bfdd3e058ad8cec67a852d5fa2885549a15170d850ea14a0f4fbe network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:9a:32:6e:ee:0b\",\"name\":\"0d7c7298bf6bfdd\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0be7e1ad-c43f-4d82-90cc-8c266199a350\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:34:24.645080 3779 obj_retry.go:498] Detected object openshift-console/console-5cc6b85768-sldch of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:34:24.645817 3779 obj_retry.go:498] Detected object openshift-console/console-5cc6b85768-sldch of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:34:24.755928 3779 obj_retry.go:492] Detected object openshift-console/console-5cc6b85768-sldch of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:24.755955 3779 obj_retry.go:492] Detected object openshift-console/console-5cc6b85768-sldch of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:24.763171 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5cc6b85768-sldch I0507 15:34:24.763199 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5cc6b85768-sldch I0507 15:34:39.499545 3779 cni.go:452] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl 0d7c7298bf6bfdd3e058ad8cec67a852d5fa2885549a15170d850ea14a0f4fbe network default NAD default NAD key ] DEL starting CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl 0d7c7298bf6bfdd3e058ad8cec67a852d5fa2885549a15170d850ea14a0f4fbe network default NAD default NAD key ] I0507 15:34:39.567535 3779 cni.go:473] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl 0d7c7298bf6bfdd3e058ad8cec67a852d5fa2885549a15170d850ea14a0f4fbe network default NAD default NAD key default] DEL finished CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl 0d7c7298bf6bfdd3e058ad8cec67a852d5fa2885549a15170d850ea14a0f4fbe network default NAD default NAD key default], result "{}", err I0507 15:34:39.613069 3779 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:34:39.613129 3779 pods.go:176] Deleting pod: openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl I0507 15:34:39.614562 3779 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl, ips: 10.128.0.31 I0507 15:34:39.614588 3779 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:34:39.614609 3779 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0507 15:34:39.614618 3779 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0507 15:34:39.614818 3779 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0507 15:34:39.614838 3779 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0507 15:34:40.640572 3779 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:40.640598 3779 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:40.640611 3779 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:40.640623 3779 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dnc5jl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:43.490986 3779 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0507 15:34:43.500736 3779 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0507 15:34:57.232426 3779 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:34:57.233015 3779 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:34:58.033452 3779 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:58.033476 3779 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:34:58.041104 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0507 15:34:58.041136 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0507 15:36:31.655973 3779 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0507 15:36:37.401419 3779 obj_retry.go:498] Detected object openshift-console/console-686b9d8c96-77rqk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:36:37.402009 3779 obj_retry.go:498] Detected object openshift-console/console-686b9d8c96-77rqk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:36:37.714495 3779 obj_retry.go:492] Detected object openshift-console/console-686b9d8c96-77rqk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:36:37.714520 3779 obj_retry.go:492] Detected object openshift-console/console-686b9d8c96-77rqk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:36:37.719770 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-686b9d8c96-77rqk I0507 15:36:37.719798 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-686b9d8c96-77rqk I0507 15:36:46.530607 3779 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=147 I0507 15:36:48.451072 3779 cni.go:452] [openshift-console/console-686b9d8c96-465kz e868baddf32dadabc9ae3a79348b706b4b3c804df6e4cbf785b62c84e410356c network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-686b9d8c96-465kz e868baddf32dadabc9ae3a79348b706b4b3c804df6e4cbf785b62c84e410356c network default NAD default NAD key ] I0507 15:36:48.519442 3779 cni.go:473] [openshift-console/console-686b9d8c96-465kz e868baddf32dadabc9ae3a79348b706b4b3c804df6e4cbf785b62c84e410356c network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-686b9d8c96-465kz e868baddf32dadabc9ae3a79348b706b4b3c804df6e4cbf785b62c84e410356c network default NAD default NAD key default], result "{}", err I0507 15:36:48.566858 3779 obj_retry.go:498] Detected object openshift-console/console-686b9d8c96-465kz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:36:48.566892 3779 pods.go:176] Deleting pod: openshift-console/console-686b9d8c96-465kz I0507 15:36:48.568165 3779 pods.go:236] Attempting to release IPs for pod: openshift-console/console-686b9d8c96-465kz, ips: 10.128.0.30 I0507 15:36:48.568196 3779 obj_retry.go:498] Detected object openshift-console/console-686b9d8c96-465kz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:36:49.062245 3779 obj_retry.go:492] Detected object openshift-console/console-686b9d8c96-465kz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:36:49.062273 3779 obj_retry.go:492] Detected object openshift-console/console-686b9d8c96-465kz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:36:49.069942 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-686b9d8c96-465kz I0507 15:36:49.069976 3779 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-686b9d8c96-465kz I0507 15:37:00.529421 3779 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=87 I0507 15:37:18.532089 3779 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=14 I0507 15:37:39.637923 3779 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0507 15:37:45.617993 3779 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0507 15:37:49.649240 3779 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0507 15:37:52.627558 3779 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0507 15:37:52.643910 3779 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0507 15:37:57.528789 3779 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0507 15:38:24.633031 3779 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0507 15:39:27.532311 3779 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0507 15:39:45.308036 3779 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-42lh2] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-5d7684fb5d-42lh2 for pod on switch ip-10-0-1-119.ec2.internal I0507 15:39:45.308164 3779 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-42lh2 I0507 15:39:45.335040 3779 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-42lh2] pod update took 26.903654ms I0507 15:39:45.335073 3779 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-42lh2] addLogicalPort annotation time took 26.941556ms I0507 15:39:45.336300 3779 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-42lh2] addLogicalPort took 28.282365ms, libovsdb time 846.499µs I0507 15:39:45.669990 3779 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-42lh2 13866e0e463b40c919cdc3c7c3087cf0c25f825c82ae71596d1a75b6e1200937 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-42lh2 13866e0e463b40c919cdc3c7c3087cf0c25f825c82ae71596d1a75b6e1200937 network default NAD default NAD key ] I0507 15:39:45.679454 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-5d7684fb5d-42lh2, hostIfaceName: 13866e0e463b40c, network: default, NAD default, SandboxID: "13866e0e463b40c919cdc3c7c3087cf0c25f825c82ae71596d1a75b6e1200937", PCI device ID: , UID: "810598bc-0bbf-4cb8-a3dc-8f059e474f53", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0507 15:39:45.734014 3779 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-42lh2 13866e0e463b40c919cdc3c7c3087cf0c25f825c82ae71596d1a75b6e1200937 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-42lh2 13866e0e463b40c919cdc3c7c3087cf0c25f825c82ae71596d1a75b6e1200937 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:d0:ee:56:b3:87\",\"name\":\"13866e0e463b40c\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7da2a0b0-3afe-4ce7-9ff5-1500d492a248\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:39:48.533778 3779 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=754 I0507 15:40:00.180064 3779 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29636140-kpzs6 for pod on switch ip-10-0-1-119.ec2.internal I0507 15:40:00.180184 3779 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 I0507 15:40:00.194726 3779 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6] pod update took 14.566278ms I0507 15:40:00.194753 3779 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6] addLogicalPort annotation time took 14.597841ms I0507 15:40:00.196199 3779 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6] addLogicalPort took 16.148005ms, libovsdb time 960.844µs I0507 15:40:00.551913 3779 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 92e350e69fe47d7b1d71caea562f18a12b35e13dfdda043171832268754dda08 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 92e350e69fe47d7b1d71caea562f18a12b35e13dfdda043171832268754dda08 network default NAD default NAD key ] I0507 15:40:00.562027 3779 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29636140-kpzs6, hostIfaceName: 92e350e69fe47d7, network: default, NAD default, SandboxID: "92e350e69fe47d7b1d71caea562f18a12b35e13dfdda043171832268754dda08", PCI device ID: , UID: "a97185de-708e-4360-9911-7e98297d7529", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0507 15:40:00.616737 3779 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 92e350e69fe47d7b1d71caea562f18a12b35e13dfdda043171832268754dda08 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 92e350e69fe47d7b1d71caea562f18a12b35e13dfdda043171832268754dda08 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:f1:a1:37:0b:ba\",\"name\":\"92e350e69fe47d7\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/783159d0-053f-4542-b631-2f4b49893355\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0507 15:40:03.905384 3779 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 92e350e69fe47d7b1d71caea562f18a12b35e13dfdda043171832268754dda08 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 92e350e69fe47d7b1d71caea562f18a12b35e13dfdda043171832268754dda08 network default NAD default NAD key ] I0507 15:40:03.975076 3779 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 92e350e69fe47d7b1d71caea562f18a12b35e13dfdda043171832268754dda08 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 92e350e69fe47d7b1d71caea562f18a12b35e13dfdda043171832268754dda08 network default NAD default NAD key default], result "{}", err I0507 15:40:04.016263 3779 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0507 15:40:04.016296 3779 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 I0507 15:40:04.017703 3779 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6, ips: 10.128.0.33 I0507 15:40:04.017735 3779 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0507 15:40:04.917744 3779 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0507 15:40:04.917790 3779 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29636140-kpzs6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0507 15:40:16.961380 3779 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0507 15:40:22.622611 3779 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0507 15:40:45.618458 3779 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0507 15:40:50.657659 3779 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0507 15:42:40.535315 3779 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=7 I0507 15:42:45.534452 3779 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=9 I0507 15:43:07.532801 3779 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=39 I0507 15:43:29.630827 3779 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7