diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b990498..b1d0ba1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,7 +4,7 @@ on: pull_request: jobs: - validation: + build: runs-on: ubuntu-latest steps: - name: Checkout code diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml new file mode 100644 index 0000000..26e2a20 --- /dev/null +++ b/.github/workflows/tags.yaml @@ -0,0 +1,33 @@ +on: + push: + tags: + - "v*" +name: Create release +jobs: + release: + runs-on: ubuntu-latest + steps: + - run: sudo apt install -y libgpgme-dev libassuan-dev libdevmapper-dev + - name: Checkout code + uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: "1.15.5" + - run: ./script/release-binaries.sh + name: get binaries + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: false + prerelease: false + - name: Upload release binaries + uses: alexellis/upload-assets@0.2.2 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + asset_paths: '["./out/release/*"]' diff --git a/.gitignore b/.gitignore index 0c744de..f13af75 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ envrc +out diff --git a/README.md b/README.md index 987484e..9bfa249 100644 --- a/README.md +++ b/README.md @@ -16,3 +16,44 @@ that will may change a bit how Tinkerbell works. We are keeping the number of bc break as low as possible but in the current state they are expected. + +## Binary release + +As part of a new release for sandbox we want to push binaries to GitHub Release +in this way the community will be able to use them if needed. + +We build Docker images across many architectures, each of them in its own +repository: boots, hegel, tink and so on. + +Sandbox is just a collection of those services and we follow the same pattern +for getting binaries as well. + +There is a go program available in `./cmd/getbinariesfromquay/main.go`. You can +run it with `go run` or build it with `go build`: + +```terminal +$ go run cmd/getbinariesfromquay/main.go -h + -binary-to-copy string + The location of the binary you want to copy from inside the image. (default "/usr/bin/hegel") + -image string + The image you want to download binaries from. It has to be a multi stage image. (default "docker://quay.io/tinkerbell/hegel") + -out string + The directory that will be used to store the release binaries (default "./out") + -program string + The name of the program you are extracing binaries for. (eg tink-worker, hegel, tink-server, tink, boots) (default "hegel") +``` + +By default it uses the image running on Quay for Hegel and it gets the binary +`/usr/bin/hegel` from there. The directory `./out` is used to store images and +binaries inside `./out/releases`. + +To get the binaries for example for boots you can run: + +```terminal +$ go run cmd/getbinariesfromquay/main.go \ + -binary-to-copy /usr/bin/boots \ + -image docker://quay.io/tinkerbell/boots:sha-9625559b \ + -program boots +``` + +You will find them in `./out/release` diff --git a/cmd/getbinariesfromquay/docker/image.go b/cmd/getbinariesfromquay/docker/image.go new file mode 100644 index 0000000..d8e729b --- /dev/null +++ b/cmd/getbinariesfromquay/docker/image.go @@ -0,0 +1,76 @@ +package docker + +import ( + "context" + + "github.com/containers/image/v5/copy" + "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/signature" + "github.com/containers/image/v5/transports/alltransports" + "github.com/containers/image/v5/types" +) + +type Image struct { + src types.ImageSource + ref types.ImageReference + sys *types.SystemContext +} + +func ImageFromName(ctx context.Context, sys *types.SystemContext, name string) (*Image, error) { + imageRef, err := alltransports.ParseImageName(name) + if err != nil { + return nil, err + } + src, err := imageRef.NewImageSource(ctx, sys) + if err != nil { + return nil, err + } + return &Image{ + src: src, + ref: imageRef, + sys: sys, + }, nil +} + +func (img *Image) GetManifest(ctx context.Context) ([]byte, string, error) { + rawManifest, _, err := img.src.GetManifest(ctx, nil) + if err != nil { + return nil, "", err + } + return rawManifest, manifest.GuessMIMEType(rawManifest), nil +} + +func (img *Image) Copy(ctx context.Context, dst string) error { + destRef, err := alltransports.ParseImageName(dst) + if err != nil { + return err + } + pc, err := signature.NewPolicyContext(&signature.Policy{ + Default: []signature.PolicyRequirement{ + signature.NewPRInsecureAcceptAnything(), + }, + }) + if err != nil { + return err + } + _, err = copy.Image(ctx, pc, destRef, img.ref, ©.Options{SourceCtx: img.sys}) + if err != nil { + return err + } + return nil +} + +type SchemaV2List struct { + MediaType string `json:"mediaType"` + SchemaVersion int `json:"schemaVersion"` + Manifests []struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int `json:"size"` + Platform struct { + Architecture string `json:"architecture"` + Os string `json:"os"` + Variant string `json:"variant"` + } `json:"platform,omitempty"` + } `json:"manifests"` +} diff --git a/cmd/getbinariesfromquay/main.go b/cmd/getbinariesfromquay/main.go new file mode 100644 index 0000000..f1add7b --- /dev/null +++ b/cmd/getbinariesfromquay/main.go @@ -0,0 +1,137 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "path" + + "github.com/containers/image/manifest" + "github.com/containers/image/v5/types" + "github.com/tinkerbell/sandbox/cmd/getbinariesfromquay/docker" + "github.com/tinkerbell/sandbox/cmd/getbinariesfromquay/tar" +) + +type Config struct { + ProgramName string + OutDirectory string + Image string + Binary string +} + +var config = Config{} + +func init() { + flag.StringVar(&config.ProgramName, "program", "hegel", "The name of the program you are extracing binaries for. (eg tink-worker, hegel, tink-server, tink, boots)") + flag.StringVar(&config.OutDirectory, "out", "./out", "The directory that will be used to store the release binaries") + flag.StringVar(&config.Image, "image", "docker://quay.io/tinkerbell/hegel", "The image you want to download binaries from. It has to be a multi stage image.") + flag.StringVar(&config.Binary, "binary-to-copy", "/usr/bin/hegel", "The location of the binary you want to copy from inside the image.") +} + +func main() { + flag.Parse() + + println("Extracing binary: " + config.Binary) + println("From Image: " + config.Image) + + imageR := config.Image + binaryToCopy := config.Binary + baseDir, err := os.Getwd() + if err != nil { + log.Fatal(err) + } + programName := config.ProgramName + outDir := path.Join(baseDir, config.OutDirectory) + releaseDir := path.Join(outDir, "release") + err = os.MkdirAll(releaseDir, 0755) + if err != nil { + log.Fatal(err) + } + println("Binaries will be located in: " + releaseDir) + imgsDir := path.Join(outDir, "imgs") + err = os.MkdirAll(imgsDir, 0755) + if err != nil { + log.Fatal(err) + } + + ctx := context.Background() + img, err := docker.ImageFromName(ctx, &types.SystemContext{}, imageR) + if err != nil { + log.Fatal(err) + } + + rawManifest, mt, err := img.GetManifest(ctx) + if err != nil { + log.Fatal(err) + } + + if mt != manifest.DockerV2ListMediaType { + log.Fatal("manifest not supported, it is not a multi arch image") + } + + archList := docker.SchemaV2List{} + err = json.Unmarshal(rawManifest, &archList) + if err != nil { + log.Fatal(err) + } + + for _, arch := range archList.Manifests { + imgDir := fmt.Sprintf("%s-%s-%s", programName, arch.Platform.Os, arch.Platform.Architecture) + println("Extracting ", imgDir) + syss := &types.SystemContext{ + ArchitectureChoice: arch.Platform.Architecture, + OSChoice: arch.Platform.Os, + } + if arch.Platform.Variant != "" { + syss.VariantChoice = arch.Platform.Variant + imgDir = imgDir + "-" + arch.Platform.Variant + } + archImg, err := docker.ImageFromName(ctx, syss, imageR) + if err != nil { + log.Fatal(err) + } + err = archImg.Copy(ctx, fmt.Sprintf("dir:%s", path.Join(imgsDir, imgDir))) + if err != nil { + log.Fatal(err) + } + err = copyBinaryFromLastLayer(path.Join(imgsDir, imgDir), path.Join(releaseDir, imgDir), binaryToCopy) + if err != nil { + log.Fatal(err) + } + } +} + +func copyBinaryFromLastLayer(src, dest, binaryPath string) error { + b, err := ioutil.ReadFile(path.Join(src, "manifest.json")) + if err != nil { + return err + } + man, err := manifest.FromBlob(b, manifest.DockerV2Schema2MediaType) + if err != nil { + return err + } + last := man.LayerInfos()[len(man.LayerInfos())-1] + layerTar, err := os.Open(path.Join(src, last.Digest.String()[7:])) + if err != nil { + return err + } + err = tar.Untar(src, layerTar) + if err != nil { + return err + } + + input, err := ioutil.ReadFile(path.Join(src, binaryPath)) + if err != nil { + return err + } + + err = ioutil.WriteFile(dest, input, 0755) + if err != nil { + return err + } + return nil +} diff --git a/cmd/getbinariesfromquay/tar/untar.go b/cmd/getbinariesfromquay/tar/untar.go new file mode 100644 index 0000000..d0313e0 --- /dev/null +++ b/cmd/getbinariesfromquay/tar/untar.go @@ -0,0 +1,48 @@ +package tar + +import ( + "archive/tar" + "compress/gzip" + "io" + "os" + "path/filepath" +) + +func Untar(dst string, r io.Reader) error { + gzr, err := gzip.NewReader(r) + if err != nil { + return err + } + defer gzr.Close() + tr := tar.NewReader(gzr) + + for { + header, err := tr.Next() + switch { + case err == io.EOF: + return nil + case err != nil: + return err + case header == nil: + continue + } + target := filepath.Join(dst, header.Name) + switch header.Typeflag { + case tar.TypeDir: + if _, err := os.Stat(target); err != nil { + if err := os.MkdirAll(target, 0755); err != nil { + return err + } + } + case tar.TypeReg: + f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { + return err + } + f.Close() + } + } +} diff --git a/current_versions.sh b/current_versions.sh new file mode 100644 index 0000000..4d7651c --- /dev/null +++ b/current_versions.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# This file gets used from generate-envrc.sh but it is also used standalone by +# automation that needs to get the version of the programs currently supported +# in sandbox + +export OSIE_DOWNLOAD_LINK=https://tinkerbell-oss.s3.amazonaws.com/osie-uploads/osie-v0-n=366,c=1aec189,b=master.tar.gz +export TINKERBELL_TINK_SERVER_IMAGE=quay.io/tinkerbell/tink:sha-0e8e5733 +export TINKERBELL_TINK_CLI_IMAGE=quay.io/tinkerbell/tink-cli:sha-0e8e5733 +export TINKERBELL_TINK_BOOTS_IMAGE=quay.io/tinkerbell/boots:sha-e81a291c +export TINKERBELL_TINK_HEGEL_IMAGE=quay.io/tinkerbell/hegel:sha-c17b512f +export TINKERBELL_TINK_WORKER_IMAGE=quay.io/tinkerbell/tink-worker:sha-0e8e5733 diff --git a/generate-envrc.sh b/generate-envrc.sh index 83fa7c2..49d7064 100755 --- a/generate-envrc.sh +++ b/generate-envrc.sh @@ -11,6 +11,8 @@ fi ERR="${RED:-}ERROR:${RESET:-}" +source ./current_versions.sh + err() ( if [ -z "${1:-}" ]; then cat >&2 @@ -54,12 +56,12 @@ generate_envrc() ( cat < { }; -in { pkgs ? import (_pkgs.fetchFromGitHub { - owner = "NixOS"; - repo = "nixpkgs"; - #branch@date: nixpkgs-unstable@2020-11-04 - rev = "dfea4e4951a3cee4d1807d8d4590189cf16f366b"; - sha256 = "02j7f5l2p08144b2fb7pg6sbni5km5y72k3nk3i7irddx8j2s04i"; -}) { } }: +in +{ pkgs ? + import + (_pkgs.fetchFromGitHub { + owner = "NixOS"; + repo = "nixpkgs"; + #branch@date: nixpkgs-unstable@2020-11-19 + rev = "4f3475b113c93d204992838aecafa89b1b3ccfde"; + sha256 = "158iik656ds6i6pc672w54cnph4d44d0a218dkq6npzrbhd3vvbg"; + }) { } +}: with pkgs; mkShell { - buildInputs = [ go nodePackages.prettier shellcheck shfmt terraform ]; + buildInputs = [ go nodePackages.prettier shellcheck shfmt terraform gpgme ]; }