From 776e436e35aafe9adb585e18bb0baddf689d148b Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Tue, 3 Oct 2023 10:12:25 -0700 Subject: [PATCH 1/2] Support building and deploying multi-arch linux amd64 and arm64 images --- Dockerfile | 23 +++++++++----- deploy/concierge/deployment.yaml | 8 +++++ .../local-user-authenticator/deployment.yaml | 11 ++++++- deploy/supervisor/deployment.yaml | 11 ++++++- hack/Dockerfile_fips | 31 ++++++++++++------- 5 files changed, 64 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb5f202e..b3544f4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,23 +3,29 @@ # Copyright 2020-2023 the Pinniped contributors. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -FROM golang:1.21.1 as build-env +# Prepare to cross-compile by always running the build stage in the build platform, not the target platform. +FROM --platform=$BUILDPLATFORM golang:1.21.1 as build-env WORKDIR /work -COPY . . + ARG GOPROXY ARG KUBE_GIT_VERSION ENV KUBE_GIT_VERSION=$KUBE_GIT_VERSION -# Build the executable binary (CGO_ENABLED=0 means static linking) -# Pass in GOCACHE (build cache) and GOMODCACHE (module cache) so they -# can be re-used between image builds. +# These will be set by buildkit automatically, e.g. TARGETOS set to "linux" and TARGETARCH set to "amd64" or "arm64". +# Useful for building multi-arch container images. +ARG TARGETOS +ARG TARGETARCH + +# Build the statically linked (CGO_ENABLED=0) binary. +# Mount source, build cache, and module cache for performance reasons. +# See https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ RUN \ + --mount=target=. \ --mount=type=cache,target=/cache/gocache \ --mount=type=cache,target=/cache/gomodcache \ - mkdir out && \ - export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache CGO_ENABLED=0 GOOS=linux GOARCH=amd64 && \ + export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH && \ go build -v -trimpath -ldflags "$(hack/get-ldflags.sh) -w -s" -o /usr/local/bin/pinniped-concierge-kube-cert-agent ./cmd/pinniped-concierge-kube-cert-agent/... && \ go build -v -trimpath -ldflags "$(hack/get-ldflags.sh) -w -s" -o /usr/local/bin/pinniped-server ./cmd/pinniped-server/... && \ ln -s /usr/local/bin/pinniped-server /usr/local/bin/pinniped-concierge && \ @@ -27,6 +33,9 @@ RUN \ ln -s /usr/local/bin/pinniped-server /usr/local/bin/local-user-authenticator # Use a distroless runtime image with CA certificates, timezone data, and not much else. +# Note that we are not using --platform here, so it will choose the base image for the target platform, not the build platform. +# By using "distroless/static" instead of "distroless/static-debianXX" we can float on the latest stable version of debian. +# See https://github.com/GoogleContainerTools/distroless#base-operating-system FROM gcr.io/distroless/static:nonroot@sha256:2a9e2b4fa771d31fe3346a873be845bfc2159695b9f90ca08e950497006ccc2e # Copy the server binary from the build-env stage. diff --git a/deploy/concierge/deployment.yaml b/deploy/concierge/deployment.yaml index bd1c690f..9b3c344b 100644 --- a/deploy/concierge/deployment.yaml +++ b/deploy/concierge/deployment.yaml @@ -245,6 +245,14 @@ spec: effect: NoSchedule - key: node-role.kubernetes.io/control-plane #! The new name for these nodes as of Kubernetes 1.24. effect: NoSchedule + - key: kubernetes.io/arch + effect: NoSchedule + operator: Equal + value: amd64 #! Allow running on amd64 nodes. + - key: kubernetes.io/arch + effect: NoSchedule + operator: Equal + value: arm64 #! Also allow running on arm64 nodes. #! This will help make sure our multiple pods run on different nodes, making #! our deployment "more" "HA". affinity: diff --git a/deploy/local-user-authenticator/deployment.yaml b/deploy/local-user-authenticator/deployment.yaml index 5098422a..fd92dda5 100644 --- a/deploy/local-user-authenticator/deployment.yaml +++ b/deploy/local-user-authenticator/deployment.yaml @@ -1,4 +1,4 @@ -#! Copyright 2020-2022 the Pinniped contributors. All Rights Reserved. +#! Copyright 2020-2023 the Pinniped contributors. All Rights Reserved. #! SPDX-License-Identifier: Apache-2.0 #@ load("@ytt:data", "data") @@ -76,6 +76,15 @@ spec: #! `--validate=false` flag. Note that installing via `kapp` does not complain about this validation error. seccompProfile: type: "RuntimeDefault" + tolerations: + - key: kubernetes.io/arch + effect: NoSchedule + operator: Equal + value: amd64 #! Allow running on amd64 nodes. + - key: kubernetes.io/arch + effect: NoSchedule + operator: Equal + value: arm64 #! Also allow running on arm64 nodes. --- apiVersion: v1 kind: Service diff --git a/deploy/supervisor/deployment.yaml b/deploy/supervisor/deployment.yaml index 30791a1b..909e424b 100644 --- a/deploy/supervisor/deployment.yaml +++ b/deploy/supervisor/deployment.yaml @@ -1,4 +1,4 @@ -#! Copyright 2020-2022 the Pinniped contributors. All Rights Reserved. +#! Copyright 2020-2023 the Pinniped contributors. All Rights Reserved. #! SPDX-License-Identifier: Apache-2.0 #@ load("@ytt:data", "data") @@ -190,6 +190,15 @@ spec: - name: socket emptyDir: {} #@ end + tolerations: + - key: kubernetes.io/arch + effect: NoSchedule + operator: Equal + value: amd64 #! Allow running on amd64 nodes. + - key: kubernetes.io/arch + effect: NoSchedule + operator: Equal + value: arm64 #! Also allow running on arm64 nodes. #! This will help make sure our multiple pods run on different nodes, making #! our deployment "more" "HA". affinity: diff --git a/hack/Dockerfile_fips b/hack/Dockerfile_fips index bf883a6a..6afe7b7a 100644 --- a/hack/Dockerfile_fips +++ b/hack/Dockerfile_fips @@ -15,22 +15,30 @@ # hidden behind a `GOEXPERIMENT=boringcrypto` env var. # See https://go.googlesource.com/go/+/dev.boringcrypto/README.boringcrypto.md # and https://kupczynski.info/posts/fips-golang/ for details. + +# This is not currently using --platform to prepare to cross-compile because we use gcc below to build +# platform-specific GCO code. This makes multi-arch builds slow due to target platform emulation. FROM golang:1.21.1 as build-env WORKDIR /work -COPY . . + ARG GOPROXY +ARG KUBE_GIT_VERSION +ENV KUBE_GIT_VERSION=$KUBE_GIT_VERSION + +# These will be set by buildkit automatically, e.g. TARGETOS set to "linux" and TARGETARCH set to "amd64" or "arm64". +# Useful for building multi-arch container images. +ARG TARGETOS +ARG TARGETARCH + # Build the executable binary (CGO_ENABLED=1 is required for go boring). # Even though we need cgo to call the boring crypto C functions, these # functions are statically linked into the binary. We also want to statically # link any libc bits hence we pass "-linkmode=external -extldflags -static" # to the ldflags directive. We do not pass "-s" to ldflags because we do # not want to strip symbols - those are used to verify if we compiled correctly. -# We do not pass in GOCACHE (build cache) and GOMODCACHE (module cache) -# because there have been bugs in the Go compiler caching when using cgo -# (it will sometimes use cached artifiacts when it should not). Since we -# use gcc as the C compiler, the following warning is emitted: +# Since we use gcc as the C compiler, the following warning is emitted: # /boring/boringssl/build/../crypto/bio/socket_helper.c:55: warning: # Using 'getaddrinfo' in statically linked applications requires at # runtime the shared libraries from the glibc version used for linking @@ -43,13 +51,11 @@ ARG GOPROXY # The osusergo and netgo tags are used to make sure that the Go implementations of these # standard library packages are used instead of the libc based versions. # We want to have no reliance on any C code other than the boring crypto bits. -# Setting GOOS=linux GOARCH=amd64 is a hard requirment for boring crypto: -# https://github.com/golang/go/blob/9d6ab825f6fe125f7ce630e103b887e580403802/misc/boring/README.md?plain=1#L95 -# Thus trying to compile the pinniped CLI with boring crypto is meaningless -# since we would not be able to ship windows and macOS binaries. RUN \ - mkdir out && \ - export CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GOEXPERIMENT=boringcrypto && \ + --mount=target=. \ + --mount=type=cache,target=/cache/gocache \ + --mount=type=cache,target=/cache/gomodcache \ + export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH GOEXPERIMENT=boringcrypto && \ go build -tags fips_strict,osusergo,netgo -v -trimpath -ldflags "$(hack/get-ldflags.sh) -w -linkmode=external -extldflags -static" -o /usr/local/bin/pinniped-concierge-kube-cert-agent ./cmd/pinniped-concierge-kube-cert-agent/... && \ go build -tags fips_strict,osusergo,netgo -v -trimpath -ldflags "$(hack/get-ldflags.sh) -w -linkmode=external -extldflags -static" -o /usr/local/bin/pinniped-server ./cmd/pinniped-server/... && \ ln -s /usr/local/bin/pinniped-server /usr/local/bin/pinniped-concierge && \ @@ -57,6 +63,9 @@ RUN \ ln -s /usr/local/bin/pinniped-server /usr/local/bin/local-user-authenticator # Use a distroless runtime image with CA certificates, timezone data, and not much else. +# Note that we are not using --platform here, so it will choose the base image for the target platform, not the build platform. +# By using "distroless/static" instead of "distroless/static-debianXX" we can float on the latest stable version of debian. +# See https://github.com/GoogleContainerTools/distroless#base-operating-system FROM gcr.io/distroless/static:nonroot@sha256:2a9e2b4fa771d31fe3346a873be845bfc2159695b9f90ca08e950497006ccc2e # Copy the server binary from the build-env stage. From 826d8236d92b28f0414e522c5ce58ddca7a4d8d5 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 4 Oct 2023 10:11:46 -0700 Subject: [PATCH 2/2] Use bitnami/openldap in integration tests instead of our old fork --- test/deploy/tools/ldap.yaml | 74 ++++++++--------------------------- test/deploy/tools/proxy.yaml | 9 +---- test/deploy/tools/values.yaml | 4 +- 3 files changed, 20 insertions(+), 67 deletions(-) diff --git a/test/deploy/tools/ldap.yaml b/test/deploy/tools/ldap.yaml index 89576507..38a6042d 100644 --- a/test/deploy/tools/ldap.yaml +++ b/test/deploy/tools/ldap.yaml @@ -155,60 +155,32 @@ stringData: #@ ldapLIDIF() apiVersion: v1 kind: Secret metadata: - name: ldap-server-config-before-ldif-files + name: ldap-server-additional-schema-ldif-files namespace: tools type: Opaque stringData: - server-config.ldif: | - # Load the memberof module. + #! From https://github.com/bitnami/containers/issues/982#issuecomment-1220354408 + memberof.ldif: | dn: cn=module,cn=config cn: module objectClass: olcModuleList - objectClass: top olcModulePath: /opt/bitnami/openldap/lib/openldap - olcModuleLoad: memberof + olcModuleLoad: memberof.so + olcModuleLoad: refint.so - dn: olcOverlay={0}memberof,olcDatabase={2}hdb,cn=config - objectClass: olcConfig + dn: olcOverlay=memberof,olcDatabase={2}mdb,cn=config objectClass: olcMemberOf objectClass: olcOverlayConfig - objectClass: top olcOverlay: memberof - olcMemberOfDangling: ignore - olcMemberOfRefInt: TRUE - olcMemberOfGroupOC: groupOfNames - olcMemberOfMemberAD: member - # Load the refint module. - dn: cn=module,cn=config - cn: module - objectclass: olcModuleList - objectclass: top - olcmodulepath: /opt/bitnami/openldap/lib/openldap - olcmoduleload: refint - - dn: olcOverlay={1}refint,olcDatabase={2}hdb,cn=config + dn: olcOverlay=refint,olcDatabase={2}mdb,cn=config objectClass: olcConfig objectClass: olcOverlayConfig objectClass: olcRefintConfig objectClass: top - olcOverlay: {1}refint + olcOverlay: refint olcRefintAttribute: memberof member manager owner --- -apiVersion: v1 -kind: Secret -metadata: - name: ldap-server-config-after-ldif-files - namespace: tools -type: Opaque -stringData: - server-config.ldif: | - # Reject any further connections that do not use TLS or StartTLS - dn: olcDatabase={2}hdb,cn=config - changetype: modify - add: olcSecurity - olcSecurity: tls=1 ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -241,13 +213,6 @@ spec: containerPort: 1389 - name: ldaps containerPort: 1636 - resources: - requests: - cpu: "100m" #! one-tenth of one CPU - memory: "64Mi" - limits: - #! Do not limit CPU because it was causing issues running integration tests on AKS where openldap became very slow. - memory: "64Mi" readinessProbe: tcpSocket: port: ldap @@ -274,6 +239,8 @@ spec: value: "password" #! ok to hardcode: the LDAP server will not be available from outside the cluster - name: LDAP_ENABLE_TLS value: "yes" + - name: LDAP_REQUIRE_TLS + value: "yes" - name: LDAP_TLS_CERT_FILE value: "/var/certs/ldap.pem" - name: LDAP_TLS_KEY_FILE @@ -283,14 +250,12 @@ spec: #! Note that the custom LDIF file is only read at pod start-up time. - name: LDAP_CUSTOM_LDIF_DIR value: "/var/ldifs" - - name: LDAP_SERVER_CONFIG_BEFORE_CUSTOM_LDIF_DIR - value: "/var/server-config-before-ldifs" - - name: LDAP_SERVER_CONFIG_AFTER_CUSTOM_LDIF_DIR - value: "/var/server-config-after-ldifs" #! Seems like LDAP_ROOT is still required when using LDAP_CUSTOM_LDIF_DIR because it effects the admin user. #! Presumably this needs to match the root that we create in the LDIF file. - name: LDAP_ROOT value: "dc=pinniped,dc=dev" + - name: LDAP_EXTRA_SCHEMAS + value: "cosine,inetorgperson,nis,memberof" volumeMounts: - name: certs mountPath: /var/certs @@ -298,11 +263,9 @@ spec: - name: ldifs mountPath: /var/ldifs readOnly: true - - name: server-config-before-ldifs - mountPath: /var/server-config-before-ldifs - readOnly: true - - name: server-config-after-ldifs - mountPath: /var/server-config-after-ldifs + - name: additional-schema + mountPath: /opt/bitnami/openldap/etc/schema/memberof.ldif + subPath: memberof.ldif readOnly: true volumes: - name: certs @@ -311,12 +274,9 @@ spec: - name: ldifs secret: secretName: ldap-ldif-files - - name: server-config-before-ldifs + - name: additional-schema secret: - secretName: ldap-server-config-before-ldif-files - - name: server-config-after-ldifs - secret: - secretName: ldap-server-config-after-ldif-files + secretName: ldap-server-additional-schema-ldif-files --- apiVersion: v1 kind: Service diff --git a/test/deploy/tools/proxy.yaml b/test/deploy/tools/proxy.yaml index 38cdc88e..05617394 100644 --- a/test/deploy/tools/proxy.yaml +++ b/test/deploy/tools/proxy.yaml @@ -1,4 +1,4 @@ -#! Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +#! Copyright 2020-2023 the Pinniped contributors. All Rights Reserved. #! SPDX-License-Identifier: Apache-2.0 #@ load("@ytt:data", "data") @@ -30,13 +30,6 @@ spec: ports: - name: http containerPort: 3128 - resources: - requests: - cpu: "100m" #! one-tenth of one CPU - memory: "64Mi" - limits: - cpu: "100m" #! one-tenth of one CPU - memory: "64Mi" volumeMounts: - name: log-dir mountPath: "/var/log/squid/" diff --git a/test/deploy/tools/values.yaml b/test/deploy/tools/values.yaml index 201dd00f..8f563949 100644 --- a/test/deploy/tools/values.yaml +++ b/test/deploy/tools/values.yaml @@ -1,4 +1,4 @@ -#! Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +#! Copyright 2020-2023 the Pinniped contributors. All Rights Reserved. #! SPDX-License-Identifier: Apache-2.0 #@data/values @@ -28,7 +28,7 @@ pinny_ldap_password: #! Images for each of the deployed test components. dex_image: ghcr.io/pinniped-ci-bot/test-dex:latest -ldap_image: ghcr.io/pinniped-ci-bot/test-ldap:latest +ldap_image: ghcr.io/pinniped-ci-bot/test-bitnami-ldap:latest proxy_image: ghcr.io/pinniped-ci-bot/test-forward-proxy:latest cfssl_image: ghcr.io/pinniped-ci-bot/test-cfssl:latest kubectl_image: ghcr.io/pinniped-ci-bot/test-kubectl:latest