Optimize image build using .dockerignore and BuildKit features.

This optimizes our image in a few different ways:

- It adds a bunch of files and directories to the `.dockerignore` file.
  This lets us have a single `COPY . .` but still be very aggressive about pruning what files end up in the build context.

- It adds build-time cache mounts to the `go build` commands using BuildKit's `--mount=type=cache` flag.
  This requires BuildKit-capable Docker, but means that our Go builds can all be incremental builds.
  This replaces the previous flow we had where we needed to split out `go mod download`.

- Instead of letting the full `apt-get install ca-certificates` layer end up in our final image, we copy just the single file we need.

Signed-off-by: Matt Moyer <moyerm@vmware.com>

.dockerignore

@@ -0,0 +1,13 @@
+./.*
+./*.md
+./*.yaml
+./apis
+./deploy
+./Dockerfile
+./generated/1.1*
+./hack/lib/tilt/
+./internal/mocks
+./LICENSE
+./site/
+./test
+**/*_test.go

Dockerfile

@@ -1,36 +1,41 @@
+# syntax = docker/dockerfile:1.0-experimental
+
 # Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
 FROM golang:1.15.7 as build-env
 
 WORKDIR /work
-# Get dependencies first so they can be cached as a layer
+COPY . .
-COPY go.* ./
+ARG GOPROXY
-COPY generated/1.20/apis/go.* ./generated/1.20/apis/
-COPY generated/1.20/client/go.* ./generated/1.20/client/
-RUN go mod download
-
-# Copy only the production source code to avoid cache misses when editing other files
-COPY generated ./generated
-COPY cmd ./cmd
-COPY pkg ./pkg
-COPY internal ./internal
-COPY hack ./hack
 
 # Build the executable binary (CGO_ENABLED=0 means static linking)
-RUN mkdir out \
+# Pass in GOCACHE (build cache) and GOMODCACHE (module cache) so they
-  && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "$(hack/get-ldflags.sh)" -o out ./cmd/pinniped-concierge/... \
+# can be re-used between image builds.
-  && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "$(hack/get-ldflags.sh)" -o out ./cmd/pinniped-supervisor/... \
+RUN \
-  && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o out ./cmd/local-user-authenticator/...
+  --mount=type=cache,target=/cache/gocache \
+  --mount=type=cache,target=/cache/gomodcache \
+  mkdir out && \
+  GOCACHE=/cache/gocache \
+  GOMODCACHE=/cache/gomodcache \
+  CGO_ENABLED=0 \
+  GOOS=linux \
+  GOARCH=amd64 \
+  go build -v -ldflags "$(hack/get-ldflags.sh)" -o out \
+    ./cmd/pinniped-concierge/... \
+    ./cmd/pinniped-supervisor/... \
+    ./cmd/local-user-authenticator/...
 
-# Use a runtime image based on Debian slim
+# Use a Debian slim image to grab a reasonable default CA bundle.
+FROM debian:10.7-slim AS get-ca-bundle-env
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* /var/cache/debconf/*
+
+# Use a runtime image based on Debian slim.
 FROM debian:10.7-slim
-RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
+COPY --from=get-ca-bundle-env /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 
-# Copy the binaries from the build-env stage
+# Copy the binaries from the build-env stage.
-COPY --from=build-env /work/out/pinniped-concierge /usr/local/bin/pinniped-concierge
+COPY --from=build-env /work/out/ /usr/local/bin/
-COPY --from=build-env /work/out/pinniped-supervisor /usr/local/bin/pinniped-supervisor
-COPY --from=build-env /work/out/local-user-authenticator /usr/local/bin/local-user-authenticator
 
 # Document the ports
 EXPOSE 8080 8443