Merge pull request #1699 from vmware-tanzu/arm64

Support building of multi-arch linux amd64 and arm64 container images
2023-10-04 12:17:07 -07:00 · 2023-10-04 12:17:07 -07:00 · a27a355071
commit a27a355071
parent af7d3092a5 826d8236d9
8 changed files with 84 additions and 87 deletions
--- a/23
+++ b/23
@ -3,23 +3,29 @@
 # Copyright 2020-2023 the Pinniped contributors. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0

-FROM golang:1.21.1 as build-env
+# Prepare to cross-compile by always running the build stage in the build platform, not the target platform.
+FROM --platform=$BUILDPLATFORM golang:1.21.1 as build-env

 WORKDIR /work
-COPY . .
+
 ARG GOPROXY

 ARG KUBE_GIT_VERSION
 ENV KUBE_GIT_VERSION=$KUBE_GIT_VERSION

-# Build the executable binary (CGO_ENABLED=0 means static linking)
-# Pass in GOCACHE (build cache) and GOMODCACHE (module cache) so they
-# can be re-used between image builds.
+# These will be set by buildkit automatically, e.g. TARGETOS set to "linux" and TARGETARCH set to "amd64" or "arm64".
+# Useful for building multi-arch container images.
+ARG TARGETOS
+ARG TARGETARCH
+
+# Build the statically linked (CGO_ENABLED=0) binary.
+# Mount source, build cache, and module cache for performance reasons.
+# See https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/
 RUN \
+  --mount=target=. \
  --mount=type=cache,target=/cache/gocache \
  --mount=type=cache,target=/cache/gomodcache \
-  mkdir out && \
-  export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache CGO_ENABLED=0 GOOS=linux GOARCH=amd64 && \
+  export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH && \
  go build -v -trimpath -ldflags "$(hack/get-ldflags.sh) -w -s" -o /usr/local/bin/pinniped-concierge-kube-cert-agent ./cmd/pinniped-concierge-kube-cert-agent/... && \
  go build -v -trimpath -ldflags "$(hack/get-ldflags.sh) -w -s" -o /usr/local/bin/pinniped-server ./cmd/pinniped-server/... && \
  ln -s /usr/local/bin/pinniped-server /usr/local/bin/pinniped-concierge && \
@ -27,6 +33,9 @@ RUN \
  ln -s /usr/local/bin/pinniped-server /usr/local/bin/local-user-authenticator

 # Use a distroless runtime image with CA certificates, timezone data, and not much else.
+# Note that we are not using --platform here, so it will choose the base image for the target platform, not the build platform.
+# By using "distroless/static" instead of "distroless/static-debianXX" we can float on the latest stable version of debian.
+# See https://github.com/GoogleContainerTools/distroless#base-operating-system
 FROM gcr.io/distroless/static:nonroot@sha256:2a9e2b4fa771d31fe3346a873be845bfc2159695b9f90ca08e950497006ccc2e

 # Copy the server binary from the build-env stage.
--- a/deploy/concierge/deployment.yaml
+++ b/deploy/concierge/deployment.yaml
@ -245,6 +245,14 @@ spec:
          effect: NoSchedule
        - key: node-role.kubernetes.io/control-plane #! The new name for these nodes as of Kubernetes 1.24.
          effect: NoSchedule
+        - key: kubernetes.io/arch
+          effect: NoSchedule
+          operator: Equal
+          value: amd64 #! Allow running on amd64 nodes.
+        - key: kubernetes.io/arch
+          effect: NoSchedule
+          operator: Equal
+          value: arm64 #! Also allow running on arm64 nodes.
      #! This will help make sure our multiple pods run on different nodes, making
      #! our deployment "more" "HA".
      affinity:
--- a/deploy/local-user-authenticator/deployment.yaml
+++ b/deploy/local-user-authenticator/deployment.yaml
@ -1,4 +1,4 @@
-#! Copyright 2020-2022 the Pinniped contributors. All Rights Reserved.
+#! Copyright 2020-2023 the Pinniped contributors. All Rights Reserved.
 #! SPDX-License-Identifier: Apache-2.0

 #@ load("@ytt:data", "data")
@ -76,6 +76,15 @@ spec:
            #! `--validate=false` flag. Note that installing via `kapp` does not complain about this validation error.
            seccompProfile:
              type: "RuntimeDefault"
+      tolerations:
+        - key: kubernetes.io/arch
+          effect: NoSchedule
+          operator: Equal
+          value: amd64 #! Allow running on amd64 nodes.
+        - key: kubernetes.io/arch
+          effect: NoSchedule
+          operator: Equal
+          value: arm64 #! Also allow running on arm64 nodes.
 ---
 apiVersion: v1
 kind: Service
--- a/deploy/supervisor/deployment.yaml
+++ b/deploy/supervisor/deployment.yaml
@ -1,4 +1,4 @@
-#! Copyright 2020-2022 the Pinniped contributors. All Rights Reserved.
+#! Copyright 2020-2023 the Pinniped contributors. All Rights Reserved.
 #! SPDX-License-Identifier: Apache-2.0

 #@ load("@ytt:data", "data")
@ -190,6 +190,15 @@ spec:
        - name: socket
          emptyDir: {}
        #@ end
+      tolerations:
+        - key: kubernetes.io/arch
+          effect: NoSchedule
+          operator: Equal
+          value: amd64 #! Allow running on amd64 nodes.
+        - key: kubernetes.io/arch
+          effect: NoSchedule
+          operator: Equal
+          value: arm64 #! Also allow running on arm64 nodes.
      #! This will help make sure our multiple pods run on different nodes, making
      #! our deployment "more" "HA".
      affinity:
--- a/hack/Dockerfile_fips
+++ b/hack/Dockerfile_fips
@ -15,22 +15,30 @@
 # hidden behind a `GOEXPERIMENT=boringcrypto` env var.
 # See https://go.googlesource.com/go/+/dev.boringcrypto/README.boringcrypto.md
 # and https://kupczynski.info/posts/fips-golang/ for details.
+
+# This is not currently using --platform to prepare to cross-compile because we use gcc below to build
+# platform-specific GCO code. This makes multi-arch builds slow due to target platform emulation.
 FROM golang:1.21.1 as build-env

 WORKDIR /work
-COPY . .
+
 ARG GOPROXY

+ARG KUBE_GIT_VERSION
+ENV KUBE_GIT_VERSION=$KUBE_GIT_VERSION
+
+# These will be set by buildkit automatically, e.g. TARGETOS set to "linux" and TARGETARCH set to "amd64" or "arm64".
+# Useful for building multi-arch container images.
+ARG TARGETOS
+ARG TARGETARCH
+
 # Build the executable binary (CGO_ENABLED=1 is required for go boring).
 # Even though we need cgo to call the boring crypto C functions, these
 # functions are statically linked into the binary.  We also want to statically
 # link any libc bits hence we pass "-linkmode=external -extldflags -static"
 # to the ldflags directive.  We do not pass "-s" to ldflags because we do
 # not want to strip symbols - those are used to verify if we compiled correctly.
-# We do not pass in GOCACHE (build cache) and GOMODCACHE (module cache)
-# because there have been bugs in the Go compiler caching when using cgo
-# (it will sometimes use cached artifiacts when it should not).  Since we
-# use gcc as the C compiler, the following warning is emitted:
+# Since we use gcc as the C compiler, the following warning is emitted:
 # /boring/boringssl/build/../crypto/bio/socket_helper.c:55: warning:
 # Using 'getaddrinfo' in statically linked applications requires at
 # runtime the shared libraries from the glibc version used for linking
@ -43,13 +51,11 @@ ARG GOPROXY
 # The osusergo and netgo tags are used to make sure that the Go implementations of these
 # standard library packages are used instead of the libc based versions.
 # We want to have no reliance on any C code other than the boring crypto bits.
-# Setting GOOS=linux GOARCH=amd64 is a hard requirment for boring crypto:
-# https://github.com/golang/go/blob/9d6ab825f6fe125f7ce630e103b887e580403802/misc/boring/README.md?plain=1#L95
-# Thus trying to compile the pinniped CLI with boring crypto is meaningless
-# since we would not be able to ship windows and macOS binaries.
 RUN \
-  mkdir out && \
-  export CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GOEXPERIMENT=boringcrypto && \
+  --mount=target=. \
+  --mount=type=cache,target=/cache/gocache \
+  --mount=type=cache,target=/cache/gomodcache \
+  export GOCACHE=/cache/gocache GOMODCACHE=/cache/gomodcache CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH GOEXPERIMENT=boringcrypto && \
  go build -tags fips_strict,osusergo,netgo -v -trimpath -ldflags "$(hack/get-ldflags.sh) -w -linkmode=external -extldflags -static" -o /usr/local/bin/pinniped-concierge-kube-cert-agent ./cmd/pinniped-concierge-kube-cert-agent/... && \
  go build -tags fips_strict,osusergo,netgo -v -trimpath -ldflags "$(hack/get-ldflags.sh) -w -linkmode=external -extldflags -static" -o /usr/local/bin/pinniped-server ./cmd/pinniped-server/... && \
  ln -s /usr/local/bin/pinniped-server /usr/local/bin/pinniped-concierge && \
@ -57,6 +63,9 @@ RUN \
  ln -s /usr/local/bin/pinniped-server /usr/local/bin/local-user-authenticator

 # Use a distroless runtime image with CA certificates, timezone data, and not much else.
+# Note that we are not using --platform here, so it will choose the base image for the target platform, not the build platform.
+# By using "distroless/static" instead of "distroless/static-debianXX" we can float on the latest stable version of debian.
+# See https://github.com/GoogleContainerTools/distroless#base-operating-system
 FROM gcr.io/distroless/static:nonroot@sha256:2a9e2b4fa771d31fe3346a873be845bfc2159695b9f90ca08e950497006ccc2e

 # Copy the server binary from the build-env stage.
--- a/test/deploy/tools/ldap.yaml
+++ b/test/deploy/tools/ldap.yaml
@ -155,60 +155,32 @@ stringData: #@ ldapLIDIF()
 apiVersion: v1
 kind: Secret
 metadata:
-  name: ldap-server-config-before-ldif-files
+  name: ldap-server-additional-schema-ldif-files
  namespace: tools
 type: Opaque
 stringData:
-  server-config.ldif: |
-    # Load the memberof module.
+  #! From https://github.com/bitnami/containers/issues/982#issuecomment-1220354408
+  memberof.ldif: |
    dn: cn=module,cn=config
    cn: module
    objectClass: olcModuleList
-    objectClass: top
    olcModulePath: /opt/bitnami/openldap/lib/openldap
-    olcModuleLoad: memberof
+    olcModuleLoad: memberof.so
+    olcModuleLoad: refint.so

-    dn: olcOverlay={0}memberof,olcDatabase={2}hdb,cn=config
-    objectClass: olcConfig
+    dn: olcOverlay=memberof,olcDatabase={2}mdb,cn=config
    objectClass: olcMemberOf
    objectClass: olcOverlayConfig
-    objectClass: top
    olcOverlay: memberof
-    olcMemberOfDangling: ignore
-    olcMemberOfRefInt: TRUE
-    olcMemberOfGroupOC: groupOfNames
-    olcMemberOfMemberAD: member

-    # Load the refint module.
-    dn: cn=module,cn=config
-    cn: module
-    objectclass: olcModuleList
-    objectclass: top
-    olcmodulepath: /opt/bitnami/openldap/lib/openldap
-    olcmoduleload: refint
-
-    dn: olcOverlay={1}refint,olcDatabase={2}hdb,cn=config
+    dn: olcOverlay=refint,olcDatabase={2}mdb,cn=config
    objectClass: olcConfig
    objectClass: olcOverlayConfig
    objectClass: olcRefintConfig
    objectClass: top
-    olcOverlay: {1}refint
+    olcOverlay: refint
    olcRefintAttribute: memberof member manager owner
 ---
-apiVersion: v1
-kind: Secret
-metadata:
-  name: ldap-server-config-after-ldif-files
-  namespace: tools
-type: Opaque
-stringData:
-  server-config.ldif: |
-    # Reject any further connections that do not use TLS or StartTLS
-    dn: olcDatabase={2}hdb,cn=config
-    changetype: modify
-    add: olcSecurity
-    olcSecurity: tls=1
---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@ -241,13 +213,6 @@ spec:
              containerPort: 1389
            - name: ldaps
              containerPort: 1636
-          resources:
-            requests:
-              cpu: "100m" #! one-tenth of one CPU
-              memory: "64Mi"
-            limits:
-              #! Do not limit CPU because it was causing issues running integration tests on AKS where openldap became very slow.
-              memory: "64Mi"
          readinessProbe:
            tcpSocket:
              port: ldap
@ -274,6 +239,8 @@ spec:
              value: "password" #! ok to hardcode: the LDAP server will not be available from outside the cluster
            - name: LDAP_ENABLE_TLS
              value: "yes"
+            - name: LDAP_REQUIRE_TLS
+              value: "yes"
            - name: LDAP_TLS_CERT_FILE
              value: "/var/certs/ldap.pem"
            - name: LDAP_TLS_KEY_FILE
@ -283,14 +250,12 @@ spec:
              #! Note that the custom LDIF file is only read at pod start-up time.
            - name: LDAP_CUSTOM_LDIF_DIR
              value: "/var/ldifs"
-            - name: LDAP_SERVER_CONFIG_BEFORE_CUSTOM_LDIF_DIR
-              value: "/var/server-config-before-ldifs"
-            - name: LDAP_SERVER_CONFIG_AFTER_CUSTOM_LDIF_DIR
-              value: "/var/server-config-after-ldifs"
              #! Seems like LDAP_ROOT is still required when using LDAP_CUSTOM_LDIF_DIR because it effects the admin user.
              #! Presumably this needs to match the root that we create in the LDIF file.
            - name: LDAP_ROOT
              value: "dc=pinniped,dc=dev"
+            - name: LDAP_EXTRA_SCHEMAS
+              value: "cosine,inetorgperson,nis,memberof"
          volumeMounts:
            - name: certs
              mountPath: /var/certs
@ -298,11 +263,9 @@ spec:
            - name: ldifs
              mountPath: /var/ldifs
              readOnly: true
-            - name: server-config-before-ldifs
-              mountPath: /var/server-config-before-ldifs
-              readOnly: true
-            - name: server-config-after-ldifs
-              mountPath: /var/server-config-after-ldifs
+            - name: additional-schema
+              mountPath: /opt/bitnami/openldap/etc/schema/memberof.ldif
+              subPath: memberof.ldif
              readOnly: true
      volumes:
        - name: certs
@ -311,12 +274,9 @@ spec:
        - name: ldifs
          secret:
            secretName: ldap-ldif-files
-        - name: server-config-before-ldifs
+        - name: additional-schema
          secret:
-            secretName: ldap-server-config-before-ldif-files
-        - name: server-config-after-ldifs
-          secret:
-            secretName: ldap-server-config-after-ldif-files
+            secretName: ldap-server-additional-schema-ldif-files
 ---
 apiVersion: v1
 kind: Service
--- a/test/deploy/tools/proxy.yaml
+++ b/test/deploy/tools/proxy.yaml
@ -1,4 +1,4 @@
-#! Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
+#! Copyright 2020-2023 the Pinniped contributors. All Rights Reserved.
 #! SPDX-License-Identifier: Apache-2.0

 #@ load("@ytt:data", "data")
@ -30,13 +30,6 @@ spec:
          ports:
          - name: http
            containerPort: 3128
-          resources:
-            requests:
-              cpu: "100m" #! one-tenth of one CPU
-              memory: "64Mi"
-            limits:
-              cpu: "100m" #! one-tenth of one CPU
-              memory: "64Mi"
          volumeMounts:
            - name: log-dir
              mountPath: "/var/log/squid/"
--- a/test/deploy/tools/values.yaml
+++ b/test/deploy/tools/values.yaml
@ -1,4 +1,4 @@
-#! Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
+#! Copyright 2020-2023 the Pinniped contributors. All Rights Reserved.
 #! SPDX-License-Identifier: Apache-2.0

 #@data/values
@ -28,7 +28,7 @@ pinny_ldap_password:

 #! Images for each of the deployed test components.
 dex_image: ghcr.io/pinniped-ci-bot/test-dex:latest
-ldap_image: ghcr.io/pinniped-ci-bot/test-ldap:latest
+ldap_image: ghcr.io/pinniped-ci-bot/test-bitnami-ldap:latest
 proxy_image: ghcr.io/pinniped-ci-bot/test-forward-proxy:latest
 cfssl_image: ghcr.io/pinniped-ci-bot/test-cfssl:latest
 kubectl_image: ghcr.io/pinniped-ci-bot/test-kubectl:latest