Hacking in a local registry - WIP

deploy_carvel/build.sh

@@ -59,7 +59,8 @@ KAPP_CONTROLLER_GLOBAL_NAMESPACE="kapp-controller-packaging-global"
 # REPO_HOST="1.2.3.4.fake.repo.host:5000"
 # PACKAGE_REPO_HOST="projects.registry.vmware.com/pinniped/pinniped-server"
 # PACKAGE_REPO_HOST="docker.io/benjaminapetersen/pinniped-package-repo"
-PACKAGE_REPO_HOST="benjaminapetersen/pinniped-package-repo"
+# PACKAGE_REPO_HOST="benjaminapetersen/pinniped-package-repo"
+PACKGE_REPO_HOST="localhost:5001/pinniped-package-repo"
 PINNIPED_PACKAGE_VERSION="0.25.0"
 
 # TODO: cp ./deploy/supervisor.... into ./deploy_carvel/supervisor/config...

deploy_carvel/build_and_deploy_for_integration_tests.sh

@@ -34,13 +34,10 @@ echo_blue() {
 # build_and_deploy_for_integration_tests.sh 123455
 echo ""
 echo ""
-tag=${1} # need to manually pass this, use the same from prepare-for-integreation-tests.sh
+app=${1} # throw away, pattern is always to call scripts with 2 args
+tag=${2} # need to manually pass this, use the same from prepare-for-integreation-tests.sh
 echo_yellow "using tag: ${tag}"
-echo_yellow "does this match output tag from prepare-for-integration-test.sh?"
 echo ""
-echo ""
-sleep 3 # just to give enough time to see it for a human
-
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
@@ -69,12 +66,12 @@ KAPP_CONTROLLER_GLOBAL_NAMESPACE="kapp-controller-packaging-global"
 # since we ran prepare-for-integration-test.sh
 # and it does the "kind load" on the
 # pinniped image
-registry="pinniped.local" # Hack, but not what we really want: getpinniped/pinniped-server:latest
+# registry="pinniped.local" # Hack, but not what we really want: getpinniped/pinniped-server:latest
+registry="localhost:5001" # local registry setup via splicing in https://kind.sigs.k8s.io/docs/user/local-registry/
 repo="test/build"
 registry_repo="$registry/$repo"
 
 
-
 api_group_suffix="pinniped.dev"
 # END PINNIPED IMAGE, NOT PACKAGE -------------- >>>>
 
@@ -175,16 +172,16 @@ ls -la "/tmp/${PACKAGE_REPO_HOST}:${PINNIPED_PACKAGE_VERSION}"
 
 
 echo_yellow "deploying PackageRepository..."
-PINNIPED_PACKGE_REPOSITORY_NAME="pinniped-package-repository"
+PINNIPED_PACKAGE_REPOSITORY_NAME="pinniped-package-repository"
-PINNIPED_PACKGE_REPOSITORY_FILE="packagerepository.${PINNIPED_PACKAGE_VERSION}.yml"
+PINNIPED_PACKAGE_REPOSITORY_FILE="packagerepository.${PINNIPED_PACKAGE_VERSION}.yml"
-echo -n "" > "${PINNIPED_PACKGE_REPOSITORY_FILE}"
+echo -n "" > "${PINNIPED_PACKAGE_REPOSITORY_FILE}"
 
-cat <<EOT >> "${PINNIPED_PACKGE_REPOSITORY_FILE}"
+cat <<EOT >> "${PINNIPED_PACKAGE_REPOSITORY_FILE}"
 ---
 apiVersion: packaging.carvel.dev/v1alpha1
 kind: PackageRepository
 metadata:
-  name: "${PINNIPED_PACKGE_REPOSITORY_NAME}"
+  name: "${PINNIPED_PACKAGE_REPOSITORY_NAME}"
 spec:
   fetch:
     imgpkgBundle:
@@ -193,8 +190,8 @@ EOT
 
 
 # Now, gotta make this work.  It'll be interesting if we can...
-kapp deploy --app "${PINNIPED_PACKGE_REPOSITORY_NAME}" --file "${PINNIPED_PACKGE_REPOSITORY_FILE}" -y
+kapp deploy --app "${PINNIPED_PACKAGE_REPOSITORY_NAME}" --file "${PINNIPED_PACKAGE_REPOSITORY_FILE}" -y
-kapp inspect --app "${PINNIPED_PACKGE_REPOSITORY_NAME}" --tree
+kapp inspect --app "${PINNIPED_PACKAGE_REPOSITORY_NAME}" --tree
 
 sleep 2 # TODO: remove
 

deploy_carvel/temp_actual_deploy_resources/pinniped-package-rbac-concierge-concierge-rbac.yml

@@ -1,37 +0,0 @@
-# ---
-# apiVersion: v1
-# kind: Namespace
-# metadata:
-#  name: "concierge-ns" <--- "supervisor-ns" will cause other package install errors.
----
-# ServiceAccount details from the file linked above
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: "pinniped-package-rbac-concierge-sa-superadmin-dangerous"
-  # namespace: "concierge-ns"
-  namespace: default # --> sticking to default for everything for now.
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: "pinniped-package-rbac-concierge-role-superadmin-dangerous"
-rules:
-- apiGroups: ["*"]
-  resources: ["*"]
-  verbs: ["*"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: "pinniped-package-rbac-concierge-role-binding-superadmin-dangerous"
-subjects:
-- kind: ServiceAccount
-  name: "pinniped-package-rbac-concierge-sa-superadmin-dangerous"
-  # namespace: "concierge-ns"
-  namespace: default # --> sticking to default for everything for now.
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: "pinniped-package-rbac-concierge-role-superadmin-dangerous"
-

deploy_carvel/temp_actual_deploy_resources/pinniped-package-rbac-supervisor-supervisor-rbac.yml

@@ -1,37 +0,0 @@
-# ---
-# apiVersion: v1
-# kind: Namespace
-# metadata:
-#  name: "supervisor-ns" <--- "supervisor-ns" will cause other package install errors.
----
-# ServiceAccount details from the file linked above
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: "pinniped-package-rbac-supervisor-sa-superadmin-dangerous"
-  # namespace: "supervisor-ns"
-  namespace: default # --> sticking to default for everything for now.
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: "pinniped-package-rbac-supervisor-role-superadmin-dangerous"
-rules:
-- apiGroups: ["*"]
-  resources: ["*"]
-  verbs: ["*"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: "pinniped-package-rbac-supervisor-role-binding-superadmin-dangerous"
-subjects:
-- kind: ServiceAccount
-  name: "pinniped-package-rbac-supervisor-sa-superadmin-dangerous"
-  # namespace: "supervisor-ns"
-  namespace: default # --> sticking to default for everything for now.
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: "pinniped-package-rbac-supervisor-role-superadmin-dangerous"
-

deploy_carvel/temp_actual_deploy_resources/supervisor-pkginstall.yml

@@ -1,35 +0,0 @@
----
-apiVersion: packaging.carvel.dev/v1alpha1
-kind: PackageInstall
-metadata:
-    # name, does not have to be versioned, versionSelection.constraints below will handle
-    name: "supervisor-package-install"
-    # namespace: "supervisor-ns"
-    namespace: default # --> sticking to default for everything for now.
-spec:
-  serviceAccountName: "pinniped-package-rbac-supervisor-sa-superadmin-dangerous"
-  packageRef:
-    refName: "supervisor.pinniped.dev"
-    versionSelection:
-      constraints: "0.25.0"
-  values:
-  - secretRef:
-      name: "supervisor-package-install-secret"
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: "supervisor-package-install-secret"
-stringData:
-  values.yml: |
-    ---
-    app_name: pinniped-supervisor
-    namespace: supervisor
-    api_group_suffix: pinniped.dev
-    image_repo: pinniped.local/test/build
-    image_tag: 160E1A5F-409E-431A-9CFC-53825C6978C7
-    log_level: debug
-
-    service_https_nodeport_port: 443
-    service_https_nodeport_nodeport: 31243
-    service_https_clusterip_port: 443

hack/kind-up.sh

@@ -5,9 +5,88 @@
 
 set -euo pipefail
 
+#
+# Helper functions
+#
+function log_note() {
+  GREEN='\033[0;32m'
+  NC='\033[0m'
+  if [[ ${COLORTERM:-unknown} =~ ^(truecolor|24bit)$ ]]; then
+    echo -e "${GREEN}$*${NC}"
+  else
+    echo "$*"
+  fi
+}
+
+function log_error() {
+  RED='\033[0;31m'
+  NC='\033[0m'
+  if [[ ${COLORTERM:-unknown} =~ ^(truecolor|24bit)$ ]]; then
+    echo -e "🙁${RED} Error: $* ${NC}"
+  else
+    echo ":( Error: $*"
+  fi
+}
+
+log_note "begin setting up kind cluster with local registry..."
+
 ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 cd "${ROOT}"
 
+log_note "creating local registry..."
+
+# part of the configuration enabling kind access to a local running docker registry
+# this will eventually be replaced by a built-in kind feature:
+# - https://kind.sigs.k8s.io/docs/user/local-registry/
+# - https://github.com/kubernetes-sigs/kind/issues/1213
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
+    registry:2
+  log_note "registry created..."
+fi
+
+KIND_CLUSTER_NAME="pinniped"
+
+log_note "creating kind cluster..."
+
 # To choose a specific version of kube, add this option to the command below: `--image kindest/node:v1.28.0`.
 # To debug the kind config, add this option to the command below: `-v 10`
-kind create cluster --config "hack/lib/kind-config/single-node.yaml" --name pinniped
+kind create cluster --config "hack/lib/kind-config/single-node.yaml" --name "${KIND_CLUSTER_NAME}"
+
+REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
+for node in $(kind get nodes --name "${KIND_CLUSTER_NAME}"); do
+  log_note "setting up node ${node} with registry....."
+  docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
+  cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
+[host."http://${reg_name}:5000"]
+EOF
+  log_note "hosts.toml on node: ${node}....."
+  docker exec "${node}" tail "${REGISTRY_DIR}/hosts.toml"
+done
+
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+  log_note "setting up docker network with kind..."
+  docker network connect "kind" "${reg_name}"
+fi
+
+log_note "documenting registry with configmap..."
+# 5. Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
+
+kubectl get cm local-registry-hosting -n kube-public -o yaml
+
+log_note "finished setting up kind cluster with local registry"

hack/lib/kind-config/single-node.yaml

@@ -44,3 +44,11 @@ kubeadmConfigPatches:
     extraArgs:
       # See comment above.
       enable-aggregator-routing: "true"
+# part of the configuration enabling kind access to a local running docker registry
+# this will eventually be replaced by a built-in kind feature:
+# - https://kind.sigs.k8s.io/docs/user/local-registry/
+# - https://github.com/kubernetes-sigs/kind/issues/1213
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry]
+    config_path = "/etc/containerd/certs.d"

hack/prepare-for-integration-tests.sh

@@ -55,6 +55,7 @@ alternate_deploy="undefined"
 alternate_deploy_supervisor="undefined"
 alternate_deploy_concierge="undefined"
 alternate_deploy_local_user_authenticator="undefined"
+post_install="undefined"
 
 # supported variable style:
 #  --dockerfile-path ./foo.sh
@@ -140,6 +141,15 @@ while (("$#")); do
     alternate_deploy_local_user_authenticator=$1
     shift
     ;;
+  --post-install)
+    shift
+    if [[ "$#" == "0" || "$1" == -* ]]; then
+      log_error "--post-install requires a script path to be specified"
+      exit 1
+    fi
+    post_install=$1
+    shift
+    ;;
   -*)
     log_error "Unsupported flag $1" >&2
     if [[ "$1" == *"active-directory"* ]]; then
@@ -169,6 +179,7 @@ if [[ "$help" == "yes" ]]; then
   log_note "   -p, --alternate-deploy-supervisor:                 specify an alternate deploy script to install Pinniped Supervisor"
   log_note "   -c, --alternate-deploy-concierge:                  specify an alternate deploy script to install Pinniped Concierge"
   log_note "   -l, --alternate-deploy-local-user-authenticator:   specify an alternate deploy script to install Pinniped local-user-authenticator"
+  log_note "   --post-install:                                    specify an post-install script"
   exit 1
 fi
 
@@ -220,7 +231,8 @@ else
   fi
 fi
 
-registry="pinniped.local"
+# registry="pinniped.local"
+registry="localhost:5001" # local registry setup via splicing in https://kind.sigs.k8s.io/docs/user/local-registry/
 repo="test/build"
 registry_repo="$registry/$repo"
 tag=$(uuidgen) # always a new tag to force K8s to reload the image on redeploy
@@ -255,6 +267,9 @@ fi
 
 # Load it into the cluster
 log_note "Loading the app's container image into the kind cluster..."
+# TODO: do I need to change anything here yet?
+#    with this new update that we are making?
+#    probably need to docker push to the new local registry instead of kind side-load?
 kind load docker-image "$registry_repo_tag" --name pinniped
 
 #
@@ -402,6 +417,17 @@ fi
 #
 test_ca_bundle_pem="$(kubectl get secrets -n tools certs -o go-template='{{index .data "ca.pem"}}')"
 
+
+#
+# Call a post-install script
+# simplifies passing the $tag which may be necessary if the current local build is to be
+# referenced, for example, deploying via a Carvel package rather than our ytt mechanism
+if [ "$post_install" != "undefined" ] ; then
+  log_note "The post-install script will be called with $tag..."
+  $post_install post-install-script $tag
+fi
+
+
 #
 # Create the environment file.
 #
@@ -501,11 +527,3 @@ log_note
 log_note "To delete the deployments, run:"
 log_note "  kapp delete -a local-user-authenticator -y && kapp delete -a $concierge_app_name -y &&  kapp delete -a $supervisor_app_name -y"
 log_note "When you're finished, use './hack/kind-down.sh' to tear down the cluster."
-
-log_note "--------------------"
-log_note "tag deployed: ${tag}"
-log_note "tag deployed: ${tag}"
-log_note "tag deployed: ${tag}"
-log_note "pass tag ${tag} to script: "
-log_note "    ./deploy_carvel/build_and_deploy_for_integration_tests.sh ${tag}"
-log_note "--------------------"