This simplifies the stand-up of a sandbox:

Only 2 main Vagrant calls are now needed (`vagrant up` and `vagrant up machine1`).
This PR only updates the Vagrant Virtualbox setup. The Vagrant Libvirt and Terraform
still need to be updated.

This uses docker-compose as the entry point for standing up the stack and makes the stand-up
of the sandbox more portal. Vagrant and Terraform are only responsible for standing up infrastructure
and then running docker-compose, not for running any glue scripts.

The docker-compose calls out to single-shot services to do all the glue required to get the fully
functional Tinkerbell stack up and running. All the single-shot services are idempotent.
This increases portability and the development iteration loop. This also simplifies the required
steps needed to get a fully functioning sandbox up and running.

This is intended to help people looking to get started by getting them to a provisioned
machine quicker and more easily.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>

releases/cmd/bump-version/envfile/envfile.go

@@ -0,0 +1,48 @@
+package envfile
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/joho/godotenv"
+)
+
+type EnvFile map[string]string
+
+func ReadEnvFile(f string) (EnvFile, error) {
+	myEnv, err := godotenv.Read(f)
+	if err != nil {
+		return nil, err
+	}
+	return myEnv, nil
+}
+
+// Copied and modified from https://github.com/joho/godotenv
+
+const doubleQuoteSpecialChars = "\\\n\r\"!$`"
+
+func Marshal(envMap map[string]string) (string, error) {
+	lines := make([]string, 0, len(envMap))
+	for k, v := range envMap {
+		lines = append(lines, fmt.Sprintf(`export %s="%s"`, k, doubleQuoteEscape(v)))
+	}
+	sort.Strings(lines)
+	return strings.Join(lines, "\n"), nil
+}
+
+func doubleQuoteEscape(line string) string {
+	for _, c := range doubleQuoteSpecialChars {
+		toReplace := "\\" + string(c)
+		if c == '\n' {
+			toReplace = `\n`
+		}
+		if c == '\r' {
+			toReplace = `\r`
+		}
+		line = strings.Replace(line, string(c), toReplace, -1)
+	}
+	return line
+}
+
+// End of the part copied from https://github.com/joho/godotenv

releases/cmd/bump-version/git/git.go

@@ -0,0 +1,30 @@
+package git
+
+import (
+	g "github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/storage/memory"
+)
+
+type Repository struct {
+	repo *g.Repository
+}
+
+func Clone(remote string) (*Repository, error) {
+	repo := &Repository{}
+	r, err := g.Clone(memory.NewStorage(), nil, &g.CloneOptions{
+		URL: remote,
+	})
+	if err != nil {
+		return nil, err
+	}
+	repo.repo = r
+	return repo, nil
+}
+
+func (r *Repository) GetHeadHash() (string, error) {
+	ref, err := r.repo.Head()
+	if err != nil {
+		return "", err
+	}
+	return ref.Hash().String(), nil
+}

releases/cmd/bump-version/image/image.go

@@ -0,0 +1,24 @@
+package image
+
+import "fmt"
+
+var ErrCommitTooShort = fmt.Errorf("commit to short, it needs at least 8 characters")
+
+type Image struct {
+	name string
+}
+
+// TagFromSha returns an image tag from a commit sha following the
+// convention we have in Tinkerbell
+// Commit: a7e947efc194fb0375f88cccc67f2fde5e0c85c1
+//    -> Tag: sha-a7e947ef
+func TagFromSha(commit string) (string, error) {
+	if len(commit) < 8 {
+		return "", ErrCommitTooShort
+	}
+	return "sha-" + commit[0:8], nil
+}
+
+func NewImage(name string) *Image {
+	return &Image{name: name}
+}

releases/cmd/bump-version/image/image_test.go

@@ -0,0 +1,39 @@
+package image_test
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/tinkerbell/sandbox/cmd/bump-version/image"
+)
+
+func TestTagFromSha(t *testing.T) {
+	s := []struct {
+		Err    error
+		Commit string
+		Tag    string
+	}{
+		{
+			Commit: "a7e947efc194fb0375f88cccc67f2fde5e0c85c1",
+			Tag:    "sha-a7e947ef",
+		},
+		{
+			Commit: "0",
+			Err:    image.ErrCommitTooShort,
+		},
+	}
+
+	for _, v := range s {
+		t.Run(fmt.Sprintf("%s -> %s", v.Commit, v.Tag), func(t *testing.T) {
+			tag, err := image.TagFromSha(v.Commit)
+			errors.Is(err, v.Err)
+			if v.Err != err {
+				t.Error(err)
+			}
+			if tag != v.Tag {
+				t.Errorf("expected %s got %s", v.Tag, tag)
+			}
+		})
+	}
+}

releases/cmd/bump-version/main.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/tinkerbell/sandbox/cmd/bump-version/envfile"
+	"github.com/tinkerbell/sandbox/cmd/bump-version/git"
+	"github.com/tinkerbell/sandbox/cmd/bump-version/image"
+)
+
+// Config represents the flags that you can configure
+// when running bump-version
+type Config struct {
+	// CurrentVersionFile is the path of the envfile that
+	// contains the current version of Tinkerbell stack.
+	// In sandbod it is called current_versions.sh
+	CurrentVersionFile string
+	// Overwrite switched the output of this program from
+	// standard output to the current_versions.sh file.
+	Overwrite bool
+	// Help prints the help command and exists
+	Help bool
+}
+
+const help = `
+bump-version is an utility that reads and update the current_version.sh file.
+It is designed to be used in CI/CD but it works locally as well.
+
+Note:
+    * Osie is currently not supported
+
+Examples:
+    Prints this -help
+
+    $ bump-version -help
+
+    This is how you will normally run this command.
+    bump-version will read the specified current-version-file
+    (./current_version.sh), it will clone the various components in memory
+    checking the latest commit available in the default branch.
+
+    The command will get and validate that a docker image is available for each
+    component and it will print the updated current_version.sh to stdout.
+
+    $ bump-version -current-version-file ./current_versions.sh
+
+    This command does the same as the previous one but it overwrite the content
+    of the original current_version file
+
+    $ bump-version -overwrite
+`
+
+const headerFile = `#!/bin/bash
+
+# This file is generated by an utility called bump-version in
+# tinkerbell/sandbox.
+# This file gets used from generate-env.sh but it is also used standalone by
+# automation that wants to get the version of the programs currently supported
+# in sandbox
+
+`
+
+func main() {
+	config := Config{}
+	fs := flag.NewFlagSet("bump-version", flag.ContinueOnError)
+	fs.BoolVar(&config.Help, "help", false, "Display help command and exit")
+	fs.BoolVar(&config.Overwrite, "overwrite", false, "This commands overwrite the current_versions file instead of printing to stdout")
+	fs.StringVar(&config.CurrentVersionFile, "current-version-file", "./current_versions.sh", "Load the current_version.sh file where the current Tinkerbell component versions are specified.")
+	err := fs.Parse(os.Args[1:])
+	if err != nil {
+		fmt.Fprint(os.Stderr, help)
+		log.Fatalf("Err: %s\n\r", err.Error())
+	}
+	if config.Help {
+		fmt.Fprint(os.Stderr, help)
+		os.Exit(0)
+	}
+	// Read the current_versions envfile
+	myEnv, err := envfile.ReadEnvFile(config.CurrentVersionFile)
+	if err != nil {
+		log.Fatal(errors.Wrap(err, "Impossible to read current_file.sh"))
+	}
+
+	tag, err := getImageTagFromRepo("https://github.com/tinkerbell/tink")
+	if err != nil {
+		log.Fatal(errors.Wrap(err, "Impossible to get image tag from tinkerbell/tink repo"))
+	}
+
+	// Set the new versions to the current_versions.sh
+	myEnv["TINKERBELL_TINK_SERVER_IMAGE"] = "quay.io/tinkerbell/tink:" + tag
+	myEnv["TINKERBELL_TINK_WORKER_IMAGE"] = "quay.io/tinkerbell/tink-worker:" + tag
+	myEnv["TINKERBELL_TINK_CLI_IMAGE"] = "quay.io/tinkerbell/tink-cli:" + tag
+
+	tag, err = getImageTagFromRepo("https://github.com/tinkerbell/hegel")
+	if err != nil {
+		log.Fatal(errors.Wrap(err, "Impossible to get image tag from tinkerbell/hegel repo"))
+	}
+
+	myEnv["TINKERBELL_TINK_HEGEL_IMAGE"] = "quay.io/tinkerbell/hegel:" + tag
+
+	tag, err = getImageTagFromRepo("https://github.com/tinkerbell/boots")
+	if err != nil {
+		log.Fatal(errors.Wrap(err, "Impossible to get image tag from tinkerbell/hegel repo"))
+	}
+
+	myEnv["TINKERBELL_TINK_BOOTS_IMAGE"] = "quay.io/tinkerbell/boots:" + tag
+
+	out, err := envfile.Marshal(myEnv)
+	if err != nil {
+		log.Fatal(errors.Wrap(err, "Impossible to render new current_version envfile"))
+	}
+
+	destination := os.Stdout
+	// if the overwrite flag is set to true change the output of the command
+	// from stdout to the original file
+	if config.Overwrite {
+		f, err := os.OpenFile(config.CurrentVersionFile, os.O_WRONLY|os.O_TRUNC, 0)
+		if err != nil {
+			log.Fatal(errors.Wrap(err, "Error overwriting current versions envfile"))
+		}
+		destination = f
+	}
+
+	io.WriteString(destination, headerFile+out+"\n")
+}
+
+// getImageTagFromRepo is an utility function that wraps a couple of things
+// together that technically we need to repeat for each repo.
+func getImageTagFromRepo(repoName string) (string, error) {
+	// Clone the specified repository in memory
+	repo, err := git.Clone(repoName)
+	if err != nil {
+		return "", errors.Wrap(err, "Impossible to clone the repository")
+	}
+
+	// Get the latest commit from the defautl branch
+	latestCommit, err := repo.GetHeadHash()
+	if err != nil {
+		return "", errors.Wrap(err, "Impossible to get commit from HEAD")
+	}
+
+	// Calculate the image tag from a commit sha
+	tag, err := image.TagFromSha(latestCommit)
+	if err != nil {
+		return "", errors.Wrap(err, "Impossible to get image tag from commit")
+	}
+
+	return tag, nil
+
+}

releases/cmd/getbinariesfromquay/docker/image.go

@@ -0,0 +1,76 @@
+package docker
+
+import (
+	"context"
+
+	"github.com/containers/image/v5/copy"
+	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/signature"
+	"github.com/containers/image/v5/transports/alltransports"
+	"github.com/containers/image/v5/types"
+)
+
+type Image struct {
+	src types.ImageSource
+	ref types.ImageReference
+	sys *types.SystemContext
+}
+
+func ImageFromName(ctx context.Context, sys *types.SystemContext, name string) (*Image, error) {
+	imageRef, err := alltransports.ParseImageName(name)
+	if err != nil {
+		return nil, err
+	}
+	src, err := imageRef.NewImageSource(ctx, sys)
+	if err != nil {
+		return nil, err
+	}
+	return &Image{
+		src: src,
+		ref: imageRef,
+		sys: sys,
+	}, nil
+}
+
+func (img *Image) GetManifest(ctx context.Context) ([]byte, string, error) {
+	rawManifest, _, err := img.src.GetManifest(ctx, nil)
+	if err != nil {
+		return nil, "", err
+	}
+	return rawManifest, manifest.GuessMIMEType(rawManifest), nil
+}
+
+func (img *Image) Copy(ctx context.Context, dst string) error {
+	destRef, err := alltransports.ParseImageName(dst)
+	if err != nil {
+		return err
+	}
+	pc, err := signature.NewPolicyContext(&signature.Policy{
+		Default: []signature.PolicyRequirement{
+			signature.NewPRInsecureAcceptAnything(),
+		},
+	})
+	if err != nil {
+		return err
+	}
+	_, err = copy.Image(ctx, pc, destRef, img.ref, &copy.Options{SourceCtx: img.sys})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+type SchemaV2List struct {
+	MediaType     string `json:"mediaType"`
+	SchemaVersion int    `json:"schemaVersion"`
+	Manifests     []struct {
+		MediaType string `json:"mediaType"`
+		Digest    string `json:"digest"`
+		Size      int    `json:"size"`
+		Platform  struct {
+			Architecture string `json:"architecture"`
+			Os           string `json:"os"`
+			Variant      string `json:"variant"`
+		} `json:"platform,omitempty"`
+	} `json:"manifests"`
+}

releases/cmd/getbinariesfromquay/main.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+
+	"github.com/containers/image/manifest"
+	"github.com/containers/image/v5/types"
+	"github.com/tinkerbell/sandbox/cmd/getbinariesfromquay/docker"
+	"github.com/tinkerbell/sandbox/cmd/getbinariesfromquay/tar"
+)
+
+type Config struct {
+	ProgramName  string
+	OutDirectory string
+	Image        string
+	Binary       string
+}
+
+var config = Config{}
+
+func init() {
+	flag.StringVar(&config.ProgramName, "program", "hegel", "The name of the program you are extracing binaries for. (eg tink-worker, hegel, tink-server, tink, boots)")
+	flag.StringVar(&config.OutDirectory, "out", "./out", "The directory that will be used to store the release binaries")
+	flag.StringVar(&config.Image, "image", "docker://quay.io/tinkerbell/hegel", "The image you want to download binaries from. It has to be a multi stage image.")
+	flag.StringVar(&config.Binary, "binary-to-copy", "/usr/bin/hegel", "The location of the binary you want to copy from inside the image.")
+}
+
+func main() {
+	flag.Parse()
+
+	println("Extracing binary: " + config.Binary)
+	println("From Image: " + config.Image)
+
+	imageR := config.Image
+	binaryToCopy := config.Binary
+	baseDir, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	programName := config.ProgramName
+	outDir := path.Join(baseDir, config.OutDirectory)
+	releaseDir := path.Join(outDir, "release")
+	err = os.MkdirAll(releaseDir, 0755)
+	if err != nil {
+		log.Fatal(err)
+	}
+	println("Binaries will be located in: " + releaseDir)
+	imgsDir := path.Join(outDir, "imgs")
+	err = os.MkdirAll(imgsDir, 0755)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	ctx := context.Background()
+	img, err := docker.ImageFromName(ctx, &types.SystemContext{}, imageR)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	rawManifest, mt, err := img.GetManifest(ctx)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if mt != manifest.DockerV2ListMediaType {
+		log.Fatal("manifest not supported, it is not a multi arch image")
+	}
+
+	archList := docker.SchemaV2List{}
+	err = json.Unmarshal(rawManifest, &archList)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, arch := range archList.Manifests {
+		imgDir := fmt.Sprintf("%s-%s-%s", programName, arch.Platform.Os, arch.Platform.Architecture)
+		println("Extracting ", imgDir)
+		syss := &types.SystemContext{
+			ArchitectureChoice: arch.Platform.Architecture,
+			OSChoice:           arch.Platform.Os,
+		}
+		if arch.Platform.Variant != "" {
+			syss.VariantChoice = arch.Platform.Variant
+			imgDir = imgDir + arch.Platform.Variant
+		}
+		archImg, err := docker.ImageFromName(ctx, syss, imageR)
+		if err != nil {
+			log.Fatal(err)
+		}
+		err = archImg.Copy(ctx, fmt.Sprintf("dir:%s", path.Join(imgsDir, imgDir)))
+		if err != nil {
+			log.Fatal(err)
+		}
+		err = untarLayers(path.Join(imgsDir, imgDir), path.Join(releaseDir, imgDir), binaryToCopy)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func untarLayers(src, dest, binaryPath string) error {
+	b, err := ioutil.ReadFile(path.Join(src, "manifest.json"))
+	if err != nil {
+		return err
+	}
+	man, err := manifest.FromBlob(b, manifest.DockerV2Schema2MediaType)
+	if err != nil {
+		return err
+	}
+	for _, l := range man.LayerInfos() {
+		layerTar, err := os.Open(path.Join(src, l.Digest.String()[7:]))
+		if err != nil {
+			return err
+		}
+		err = tar.Untar(src, layerTar)
+		if err != nil {
+			return err
+		}
+	}
+
+	input, err := ioutil.ReadFile(path.Join(src, binaryPath))
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(dest, input, 0755)
+	if err != nil {
+		return err
+	}
+	return nil
+}

releases/cmd/getbinariesfromquay/tar/untar.go

@@ -0,0 +1,48 @@
+package tar
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func Untar(dst string, r io.Reader) error {
+	gzr, err := gzip.NewReader(r)
+	if err != nil {
+		return err
+	}
+	defer gzr.Close()
+	tr := tar.NewReader(gzr)
+
+	for {
+		header, err := tr.Next()
+		switch {
+		case err == io.EOF:
+			return nil
+		case err != nil:
+			return err
+		case header == nil:
+			continue
+		}
+		target := filepath.Join(dst, header.Name)
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				if err := os.MkdirAll(target, 0755); err != nil {
+					return err
+				}
+			}
+		case tar.TypeReg:
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(f, tr); err != nil {
+				return err
+			}
+			f.Close()
+		}
+	}
+}