global # Drop resource limit checks to mitigate https://issues.redhat.com/browse/OCPBUGS-21803 in HAProxy 2.6. no strict-limits maxconn 50000 nbthread 4 daemon ca-base /etc/ssl crt-base /etc/ssl # TODO: Check if we can get reload to be faster by saving server state. # server-state-file /var/lib/haproxy/run/haproxy.state stats socket /var/lib/haproxy/run/haproxy.sock mode 600 level admin expose-fd listeners stats timeout 2m # Increase the default request size to be comparable to modern cloud load balancers (ALB: 64kb), affects # total memory use when large numbers of connections are open. # In OCP 4.8, this value is adjustable via the IngressController API. # Cluster administrators are still encouraged to use the default values provided below. tune.maxrewrite 8192 tune.bufsize 32768 # Configure the TLS versions we support ssl-default-bind-options ssl-min-ver TLSv1.2 # The default cipher suite can be selected from the three sets recommended by https://wiki.mozilla.org/Security/Server_Side_TLS, # or the user can provide one using the ROUTER_CIPHERS environment variable. # By default when a cipher set is not provided, intermediate is used. # user provided list of ciphers (Colon separated list as seen above) # the env default is not used here since we can't get here with empty ROUTER_CIPHERS tune.ssl.default-dh-param 2048 ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305 ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 # By default when a ROUTER_CURVES is not defined HAProxy # will use its built-in default supported groups for TLS key exchange. ssl-default-bind-curves X25519MLKEM768:X25519:P-256:P-384:P-521 defaults maxconn 50000 # To configure custom default errors, you can either uncomment the # line below (server ... 127.0.0.1:8080) and point it to your custom # backend service or alternatively, you can send a custom 503 or 404 error. # # server openshift_backend 127.0.0.1:8080 errorfile 503 /var/lib/haproxy/conf/error-page-503.http errorfile 404 /var/lib/haproxy/conf/error-page-404.http timeout connect 5s timeout client 30s timeout client-fin 1s timeout server 30s timeout server-fin 1s timeout http-request 10s timeout http-keep-alive 300s # Long timeout for WebSocket connections. timeout tunnel 1h frontend public bind :80 accept-proxy mode http tcp-request inspect-delay 5s tcp-request content accept if HTTP monitor-uri /_______internal_router_healthz # Mitigate CVE-2023-40225 (Proxy forwards malformed empty Content-Length headers) http-request deny if { hdr_len(content-length) 0 } # Strip off Proxy headers to prevent HTTpoxy (https://httpoxy.org/) http-request del-header Proxy # Strip off X-SSL* headers for plain HTTP if not explicitly disabled. # This prevents unauthenticated spoofing of mutual TLS client identities. http-request del-header X-SSL http-request del-header X-SSL-Client-CN http-request del-header X-SSL-Client-DER http-request del-header X-SSL-Client-DN http-request del-header X-SSL-Client-NotAfter http-request del-header X-SSL-Client-NotBefore http-request del-header X-SSL-Client-SHA1 http-request del-header X-SSL-Client-Serial http-request del-header X-SSL-Client-Subject http-request del-header X-SSL-Client-Verify http-request del-header X-SSL-Client-Version http-request del-header X-SSL-Issuer # DNS labels are case insensitive (RFC 4343), we need to convert the hostname into lowercase # before matching, or any requests containing uppercase characters will never match. http-request set-header Host %[req.hdr(Host),lower] # check if we need to redirect/force using https. acl secure_redirect base,map_reg_int(/var/lib/haproxy/conf/os_route_http_redirect.map) -m bool http-request redirect location https://%[req.hdr(host),regsub(:[0-9]+$,,)]%[url] code 302 if secure_redirect use_backend %[base,map_reg(/var/lib/haproxy/conf/os_http_be.map)] default_backend openshift_default # public ssl accepts all connections and isn't checking certificates yet certificates to use will be # determined by the next backend in the chain which may be an app backend (passthrough termination) or a backend # that terminates encryption in this router (edge) frontend public_ssl bind :443 accept-proxy tcp-request inspect-delay 5s tcp-request content accept if { req_ssl_hello_type 1 } # if the connection is SNI and the route is a passthrough don't use the termination backend, just use the tcp backend # for the SNI case, we also need to compare it in case-insensitive mode (by converting it to lowercase) as RFC 4343 says acl sni req.ssl_sni -m found acl sni_passthrough req.ssl_sni,lower,map_reg(/var/lib/haproxy/conf/os_sni_passthrough.map) -m found use_backend %[req.ssl_sni,lower,map_reg(/var/lib/haproxy/conf/os_tcp_be.map)] if sni sni_passthrough # if the route is SNI and NOT passthrough enter the termination flow use_backend be_sni if sni # non SNI requests should enter a default termination backend rather than the custom cert SNI backend since it # will not be able to match a cert to an SNI host default_backend be_no_sni ########################################################################## # TLS SNI # # When using SNI we can terminate encryption with custom certificates. # Certs will be stored in a directory and will be matched with the SNI host header # which must exist in the CN of the certificate. Certificates must be concatenated # as a single file (handled by the plugin writer) per the haproxy documentation. # # Finally, check re-encryption settings and re-encrypt or just pass along the unencrypted # traffic ########################################################################## backend be_sni server fe_sni unix@/var/lib/haproxy/run/haproxy-sni.sock weight 1 send-proxy frontend fe_sni # terminate ssl on edge bind unix@/var/lib/haproxy/run/haproxy-sni.sock ssl crt /var/lib/haproxy/router/certs/default.pem crt-list /var/lib/haproxy/conf/cert_config.map accept-proxy no-alpn mode http # Mitigate CVE-2023-40225 (Proxy forwards malformed empty Content-Length headers) http-request deny if { hdr_len(content-length) 0 } # Strip off Proxy headers to prevent HTTpoxy (https://httpoxy.org/) http-request del-header Proxy # Strip off X-SSL* headers if not explicitly disabled. # This prevents unauthenticated spoofing of mutual TLS client identities # when mutual TLS is not enabled and so the headers are not set below. http-request del-header X-SSL http-request del-header X-SSL-Client-CN http-request del-header X-SSL-Client-DER http-request del-header X-SSL-Client-DN http-request del-header X-SSL-Client-NotAfter http-request del-header X-SSL-Client-NotBefore http-request del-header X-SSL-Client-SHA1 http-request del-header X-SSL-Client-Serial http-request del-header X-SSL-Client-Subject http-request del-header X-SSL-Client-Verify http-request del-header X-SSL-Client-Version http-request del-header X-SSL-Issuer # DNS labels are case insensitive (RFC 4343), we need to convert the hostname into lowercase # before matching, or any requests containing uppercase characters will never match. http-request set-header Host %[req.hdr(Host),lower] # map to backend # Search from most specific to general path (host case). # Note: If no match, haproxy uses the default_backend, no other # use_backend directives below this will be processed. use_backend %[base,map_reg(/var/lib/haproxy/conf/os_edge_reencrypt_be.map)] default_backend openshift_default ########################################################################## # END TLS SNI ########################################################################## ########################################################################## # TLS NO SNI # # When we don't have SNI the only thing we can try to do is terminate the encryption # using our wild card certificate. Once that is complete we can either re-encrypt # the traffic or pass it on to the backends ########################################################################## # backend for when sni does not exist, or ssl term needs to happen on the edge backend be_no_sni server fe_no_sni unix@/var/lib/haproxy/run/haproxy-no-sni.sock weight 1 send-proxy frontend fe_no_sni # terminate ssl on edge bind unix@/var/lib/haproxy/run/haproxy-no-sni.sock ssl crt /var/lib/haproxy/router/certs/default.pem accept-proxy no-alpn mode http # Mitigate CVE-2023-40225 (Proxy forwards malformed empty Content-Length headers) http-request deny if { hdr_len(content-length) 0 } # Strip off Proxy headers to prevent HTTpoxy (https://httpoxy.org/) http-request del-header Proxy # Strip off X-SSL* headers if not explicitly disabled. # This prevents unauthenticated spoofing of mutual TLS client identities # when mutual TLS is not enabled and so the headers are not set below. http-request del-header X-SSL http-request del-header X-SSL-Client-CN http-request del-header X-SSL-Client-DER http-request del-header X-SSL-Client-DN http-request del-header X-SSL-Client-NotAfter http-request del-header X-SSL-Client-NotBefore http-request del-header X-SSL-Client-SHA1 http-request del-header X-SSL-Client-Serial http-request del-header X-SSL-Client-Subject http-request del-header X-SSL-Client-Verify http-request del-header X-SSL-Client-Version http-request del-header X-SSL-Issuer # DNS labels are case insensitive (RFC 4343), we need to convert the hostname into lowercase # before matching, or any requests containing uppercase characters will never match. http-request set-header Host %[req.hdr(Host),lower] # map to backend # Search from most specific to general path (host case). # Note: If no match, haproxy uses the default_backend, no other # use_backend directives below this will be processed. use_backend %[base,map_reg(/var/lib/haproxy/conf/os_edge_reencrypt_be.map)] default_backend openshift_default ########################################################################## # END TLS NO SNI ########################################################################## backend openshift_default mode http option forwardfor #option http-keep-alive option http-pretend-keepalive ##-------------- app level backends ---------------- # Plain http backend or backend with TLS terminated at the edge or a # secure backend with re-encryption. backend be_secure:openshift-console:console mode http option redispatch option forwardfor balance random timeout server 5m timeout check 5000ms http-request add-header X-Forwarded-Host %[req.hdr(host)] http-request add-header X-Forwarded-Port %[dst_port] http-request add-header X-Forwarded-Proto http if !{ ssl_fc } http-request add-header X-Forwarded-Proto https if { ssl_fc } http-request add-header X-Forwarded-Proto-Version h2 if { ssl_fc_alpn -i h2 } http-request add-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)] cookie 1e2670d92730b515ce3a1bb65da45062 insert indirect nocache httponly secure attr SameSite=None server pod:console-6cd667d795-5pxzj:console:https:10.134.0.17:8443 10.134.0.17:8443 cookie 41f871b59b73e5608a92e6a9f928e1ba weight 1 ssl verifyhost console.openshift-console.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt # Plain http backend or backend with TLS terminated at the edge or a # secure backend with re-encryption. backend be_edge_http:openshift-console:downloads mode http option redispatch option forwardfor balance random timeout check 5000ms http-request add-header X-Forwarded-Host %[req.hdr(host)] http-request add-header X-Forwarded-Port %[dst_port] http-request add-header X-Forwarded-Proto http if !{ ssl_fc } http-request add-header X-Forwarded-Proto https if { ssl_fc } http-request add-header X-Forwarded-Proto-Version h2 if { ssl_fc_alpn -i h2 } http-request add-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)] cookie a663438294fbd72a8e16964e97c8ecde insert indirect nocache httponly secure attr SameSite=None server pod:downloads-6dd789c8b7-vhvx7:downloads:http:10.132.0.29:8080 10.132.0.29:8080 cookie 6592e34c15198c4e395d9c07109d2b30 weight 1 # Secure backend, pass through backend be_tcp:openshift-ingress-canary:canary balance roundrobin hash-type consistent timeout check 5000ms server pod:ingress-canary-62bwf:ingress-canary:8443-tcp:10.132.0.16:8443 10.132.0.16:8443 weight 1 check inter 5000ms server pod:ingress-canary-4kmqj:ingress-canary:8443-tcp:10.133.0.6:8443 10.133.0.6:8443 weight 1 check inter 5000ms server pod:ingress-canary-nlz6k:ingress-canary:8443-tcp:10.134.0.6:8443 10.134.0.6:8443 weight 1 check inter 5000ms # Plain http backend or backend with TLS terminated at the edge or a # secure backend with re-encryption. backend be_secure:openshift-ingress:data-science-gateway mode http option redispatch option forwardfor balance random timeout check 5000ms http-request add-header X-Forwarded-Host %[req.hdr(host)] http-request add-header X-Forwarded-Port %[dst_port] http-request add-header X-Forwarded-Proto http if !{ ssl_fc } http-request add-header X-Forwarded-Proto https if { ssl_fc } http-request add-header X-Forwarded-Proto-Version h2 if { ssl_fc_alpn -i h2 } http-request add-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)] cookie 4e861a20d574b61bb2467f8aec974d49 insert indirect nocache httponly secure attr SameSite=None server pod:data-science-gateway-data-science-gateway-class-69f549cf5c44lqr:data-science-gateway-data-science-gateway-class:https:10.132.0.38:443 10.132.0.38:443 cookie e64fe11d88a388c70a1b7cbdc0f0c572 weight 1 ssl verifyhost data-science-gateway-data-science-gateway-class.openshift-ingress.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt # Plain http backend or backend with TLS terminated at the edge or a # secure backend with re-encryption. backend be_secure:openshift-ingress:maas-gateway-route mode http option redispatch option forwardfor balance random timeout check 5000ms http-request add-header X-Forwarded-Host %[req.hdr(host)] http-request add-header X-Forwarded-Port %[dst_port] http-request add-header X-Forwarded-Proto http if !{ ssl_fc } http-request add-header X-Forwarded-Proto https if { ssl_fc } http-request add-header X-Forwarded-Proto-Version h2 if { ssl_fc_alpn -i h2 } http-request add-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)] cookie 0cb50dfbf93eff310d377e457f2ee7d6 insert indirect nocache httponly secure attr SameSite=None server pod:maas-default-gateway-openshift-default-58d898bd55-cccdj:maas-default-gateway-openshift-default:https:10.132.0.151:443 10.132.0.151:443 cookie e628f793b62c198f7bbf3b4356db6b7b weight 1 ssl verifyhost maas-default-gateway-openshift-default.openshift-ingress.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt check inter 5000ms server pod:maas-default-gateway-openshift-default-58d898bd55-vqn2k:maas-default-gateway-openshift-default:https:10.132.0.238:443 10.132.0.238:443 cookie c21016c530ef45c1d1f8ec635c8464f5 weight 1 ssl verifyhost maas-default-gateway-openshift-default.openshift-ingress.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt check inter 5000ms server pod:maas-default-gateway-openshift-default-58d898bd55-t8qk4:maas-default-gateway-openshift-default:https:10.132.0.240:443 10.132.0.240:443 cookie 0efacdc4a830cd4ba39dd8f4e75592b8 weight 1 ssl verifyhost maas-default-gateway-openshift-default.openshift-ingress.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt check inter 5000ms server pod:maas-default-gateway-openshift-default-58d898bd55-jkrrj:maas-default-gateway-openshift-default:https:10.133.0.32:443 10.133.0.32:443 cookie 6fa75f9b55770fc64f78b8f5ed70e622 weight 1 ssl verifyhost maas-default-gateway-openshift-default.openshift-ingress.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt check inter 5000ms server pod:maas-default-gateway-openshift-default-58d898bd55-zjsq8:maas-default-gateway-openshift-default:https:10.133.0.131:443 10.133.0.131:443 cookie 252d436b4a23a36364eb7d832771e43c weight 1 ssl verifyhost maas-default-gateway-openshift-default.openshift-ingress.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt check inter 5000ms server pod:maas-default-gateway-openshift-default-58d898bd55-r67h6:maas-default-gateway-openshift-default:https:10.133.0.132:443 10.133.0.132:443 cookie f65bebae563a6c08e48f33bb750311d5 weight 1 ssl verifyhost maas-default-gateway-openshift-default.openshift-ingress.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt check inter 5000ms server pod:maas-default-gateway-openshift-default-58d898bd55-rxs7f:maas-default-gateway-openshift-default:https:10.134.0.91:443 10.134.0.91:443 cookie 1510cac3f98a5e91a55005dd0f16c1a1 weight 1 ssl verifyhost maas-default-gateway-openshift-default.openshift-ingress.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt check inter 5000ms server pod:maas-default-gateway-openshift-default-58d898bd55-mdfhv:maas-default-gateway-openshift-default:https:10.134.0.94:443 10.134.0.94:443 cookie 58c1b841c7cbb2c2dbf0f73bca98f3dd weight 1 ssl verifyhost maas-default-gateway-openshift-default.openshift-ingress.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt check inter 5000ms server pod:maas-default-gateway-openshift-default-58d898bd55-f9fls:maas-default-gateway-openshift-default:https:10.134.0.144:443 10.134.0.144:443 cookie d74513c76cfca074b0d109def2b889b0 weight 1 ssl verifyhost maas-default-gateway-openshift-default.openshift-ingress.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt check inter 5000ms server pod:maas-default-gateway-openshift-default-58d898bd55-6qfjx:maas-default-gateway-openshift-default:https:10.134.0.147:443 10.134.0.147:443 cookie 9c50b8ad4b773ace92a4ae025a9f2b04 weight 1 ssl verifyhost maas-default-gateway-openshift-default.openshift-ingress.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt check inter 5000ms # Plain http backend or backend with TLS terminated at the edge or a # secure backend with re-encryption. backend be_secure:openshift-monitoring:alertmanager-main mode http option redispatch option forwardfor balance random timeout check 5000ms http-request add-header X-Forwarded-Host %[req.hdr(host)] http-request add-header X-Forwarded-Port %[dst_port] http-request add-header X-Forwarded-Proto http if !{ ssl_fc } http-request add-header X-Forwarded-Proto https if { ssl_fc } http-request add-header X-Forwarded-Proto-Version h2 if { ssl_fc_alpn -i h2 } http-request add-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)] cookie 4e2f052a85d89824d1855ef94e095b2b insert indirect nocache httponly secure attr SameSite=None server pod:alertmanager-main-0:alertmanager-main:web:10.133.0.13:9095 10.133.0.13:9095 cookie 717ebd7b92a2fcf7f9c309e44609e83a weight 1 ssl verifyhost alertmanager-main.openshift-monitoring.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt # Plain http backend or backend with TLS terminated at the edge or a # secure backend with re-encryption. backend be_secure:openshift-monitoring:prometheus-k8s mode http option redispatch option forwardfor balance random timeout check 5000ms http-request add-header X-Forwarded-Host %[req.hdr(host)] http-request add-header X-Forwarded-Port %[dst_port] http-request add-header X-Forwarded-Proto http if !{ ssl_fc } http-request add-header X-Forwarded-Proto https if { ssl_fc } http-request add-header X-Forwarded-Proto-Version h2 if { ssl_fc_alpn -i h2 } http-request add-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)] cookie 97f6663f1252f61343f526ad937cfc1a insert indirect nocache httponly secure attr SameSite=None server pod:prometheus-k8s-0:prometheus-k8s:web:10.133.0.14:9091 10.133.0.14:9091 cookie ebfc4edf00118acfdbbf18d8996bf44f weight 1 ssl verifyhost prometheus-k8s.openshift-monitoring.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt # Plain http backend or backend with TLS terminated at the edge or a # secure backend with re-encryption. backend be_secure:openshift-monitoring:prometheus-k8s-federate mode http option redispatch option forwardfor balance random timeout check 5000ms http-request add-header X-Forwarded-Host %[req.hdr(host)] http-request add-header X-Forwarded-Port %[dst_port] http-request add-header X-Forwarded-Proto http if !{ ssl_fc } http-request add-header X-Forwarded-Proto https if { ssl_fc } http-request add-header X-Forwarded-Proto-Version h2 if { ssl_fc_alpn -i h2 } http-request add-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)] cookie 55cbf826fb5b0c929ec0c03819d877e5 insert indirect nocache httponly secure attr SameSite=None server pod:prometheus-k8s-0:prometheus-k8s:web:10.133.0.14:9091 10.133.0.14:9091 cookie ebfc4edf00118acfdbbf18d8996bf44f weight 1 ssl verifyhost prometheus-k8s.openshift-monitoring.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt # Plain http backend or backend with TLS terminated at the edge or a # secure backend with re-encryption. backend be_secure:openshift-monitoring:thanos-querier mode http option redispatch option forwardfor balance random timeout check 5000ms http-request add-header X-Forwarded-Host %[req.hdr(host)] http-request add-header X-Forwarded-Port %[dst_port] http-request add-header X-Forwarded-Proto http if !{ ssl_fc } http-request add-header X-Forwarded-Proto https if { ssl_fc } http-request add-header X-Forwarded-Proto-Version h2 if { ssl_fc_alpn -i h2 } http-request add-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)] cookie ee4d5f50aeaffc63a5a5fc30a3072a27 insert indirect nocache httponly secure attr SameSite=None server pod:thanos-querier-849d45575d-5ns87:thanos-querier:web:10.134.0.11:9091 10.134.0.11:9091 cookie 4712a4935de3613536e880496ebaab73 weight 1 ssl verifyhost thanos-querier.openshift-monitoring.svc verify required ca-file /var/run/configmaps/service-ca/service-ca.crt