From cffaeda0f191a1140f843984a2f86a774bb59320 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 11 Dec 2019 23:06:53 +0200 Subject: [PATCH] all: publish Docker images for the following GOARCH: amd64, arm, arm64, ppc64le and 386 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/258 --- app/vmbackup/Makefile | 49 +++++++++---- app/vmbackup/deployment/Dockerfile | 8 ++- app/vminsert/Makefile | 10 ++- app/vminsert/deployment/Dockerfile | 7 +- app/vmrestore/Makefile | 49 +++++++++---- app/vmrestore/deployment/Dockerfile | 8 ++- app/vmselect/Makefile | 6 +- app/vmselect/deployment/Dockerfile | 7 +- app/vmstorage/Makefile | 6 +- app/vmstorage/deployment/Dockerfile | 7 +- deployment/docker/Makefile | 105 +++++++++++++++++++++++++--- 11 files changed, 200 insertions(+), 62 deletions(-) diff --git a/app/vmbackup/Makefile b/app/vmbackup/Makefile index 0e21a574cf..27735cee4b 100644 --- a/app/vmbackup/Makefile +++ b/app/vmbackup/Makefile @@ -6,32 +6,53 @@ vmbackup: vmbackup-prod: APP_NAME=vmbackup $(MAKE) app-via-docker +vmbackup-pure-prod: + APP_NAME=vmbackup $(MAKE) app-via-docker-pure + +vmbackup--arm-prod: + APP_NAME=vmbackup $(MAKE) app-via-docker-arm + +vmbackup-arm64-prod: + APP_NAME=vmbackup $(MAKE) app-via-docker-arm64 + +vmbackup-ppc64le-prod: + APP_NAME=vmbackup $(MAKE) app-via-docker-ppc64le + +vmbackup-386-prod: + APP_NAME=vmbackup $(MAKE) app-via-docker-386 + package-vmbackup: APP_NAME=vmbackup $(MAKE) package-via-docker +package-vmbackup-pure: + APP_NAME=vmbackup $(MAKE) package-via-docker-pure + +package-vmbackup-arm: + APP_NAME=vmbackup $(MAKE) package-via-docker-arm + +package-vmbackup-arm64: + APP_NAME=vmbackup $(MAKE) package-via-docker-arm64 + +package-vmbackup-ppc64le: + APP_NAME=vmbackup $(MAKE) package-via-docker-ppc64le + +package-vmbackup-386: + APP_NAME=vmbackup $(MAKE) package-via-docker-386 + publish-vmbackup: APP_NAME=vmbackup $(MAKE) publish-via-docker +vmbackup-pure: + APP_NAME=vmbackup $(MAKE) app-local-pure + vmbackup-arm: CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm ./app/vmbackup -vmbackup-arm-prod: - APP_NAME=vmbackup APP_SUFFIX='-arm' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=arm' $(MAKE) app-via-docker - vmbackup-arm64: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm64 ./app/vmbackup -vmbackup-arm64-prod: - APP_NAME=vmbackup APP_SUFFIX='-arm64' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=arm64' $(MAKE) app-via-docker +vmbackup-ppc64le: + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-ppc64le ./app/vmbackup vmbackup-386: CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-386 ./app/vmbackup - -vmbackup-386-prod: - APP_NAME=vmbackup APP_SUFFIX='-386' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=386' $(MAKE) app-via-docker - -vmbackup-pure: - APP_NAME=vmbackup $(MAKE) app-local-pure - -vmbackup-pure-prod: - APP_NAME=vmbackup APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker diff --git a/app/vmbackup/deployment/Dockerfile b/app/vmbackup/deployment/Dockerfile index eb78fa7864..7e84910049 100644 --- a/app/vmbackup/deployment/Dockerfile +++ b/app/vmbackup/deployment/Dockerfile @@ -1,5 +1,7 @@ +ARG certs_image +FROM $certs_image AS certs FROM scratch -COPY --from=local/certs:1.0.3 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY bin/vmbackup-prod . -EXPOSE 8428 +COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +ARG src_binary +COPY $src_binary ./vmbackup-prod ENTRYPOINT ["/vmbackup-prod"] diff --git a/app/vminsert/Makefile b/app/vminsert/Makefile index 8a2ebc0ffa..c43b12e6be 100644 --- a/app/vminsert/Makefile +++ b/app/vminsert/Makefile @@ -1,9 +1,7 @@ # All these commands must run from repository root. run-vminsert: - APP_NAME=vminsert \ - ARGS='-storageNode=localhost:8400' \ - $(MAKE) run-via-docker + APP_NAME=vminsert ARGS='-storageNode=localhost:8400' $(MAKE) run-via-docker vminsert: APP_NAME=vminsert $(MAKE) app-local @@ -14,15 +12,15 @@ vminsert-race: vminsert-prod: APP_NAME=vminsert $(MAKE) app-via-docker +vminsert-pure-prod: + APP_NAME=vminsert $(MAKE) app-via-docker-pure + vminsert-prod-race: APP_NAME=vminsert RACE=-race $(MAKE) app-via-docker vminsert-pure: APP_NAME=vminsert $(MAKE) app-local-pure -vminsert-pure-prod: - APP_NAME=vminsert APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker - package-vminsert: APP_NAME=vminsert $(MAKE) package-via-docker diff --git a/app/vminsert/deployment/Dockerfile b/app/vminsert/deployment/Dockerfile index cb56ebe764..abd89a8767 100644 --- a/app/vminsert/deployment/Dockerfile +++ b/app/vminsert/deployment/Dockerfile @@ -1,5 +1,8 @@ +ARG certs_image +FROM $certs_image AS certs FROM scratch -COPY --from=local/certs:1.0.3 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY bin/vminsert-prod . +COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +ARG src_binary +COPY $src_binary ./vminsert-prod EXPOSE 8480 ENTRYPOINT ["/vminsert-prod"] diff --git a/app/vmrestore/Makefile b/app/vmrestore/Makefile index 9fcbb98c1a..fd5e8acc36 100644 --- a/app/vmrestore/Makefile +++ b/app/vmrestore/Makefile @@ -6,32 +6,53 @@ vmrestore: vmrestore-prod: APP_NAME=vmrestore $(MAKE) app-via-docker +vmrestore-pure-prod: + APP_NAME=vmrestore $(MAKE) app-via-docker-pure + +vmrestore--arm-prod: + APP_NAME=vmrestore $(MAKE) app-via-docker-arm + +vmrestore-arm64-prod: + APP_NAME=vmrestore $(MAKE) app-via-docker-arm64 + +vmrestore-ppc64le-prod: + APP_NAME=vmrestore $(MAKE) app-via-docker-ppc64le + +vmrestore-386-prod: + APP_NAME=vmrestore $(MAKE) app-via-docker-386 + package-vmrestore: APP_NAME=vmrestore $(MAKE) package-via-docker +package-vmrestore-pure: + APP_NAME=vmrestore $(MAKE) package-via-docker-pure + +package-vmrestore-arm: + APP_NAME=vmrestore $(MAKE) package-via-docker-arm + +package-vmrestore-arm64: + APP_NAME=vmrestore $(MAKE) package-via-docker-arm64 + +package-vmrestore-ppc64le: + APP_NAME=vmrestore $(MAKE) package-via-docker-ppc64le + +package-vmrestore-386: + APP_NAME=vmrestore $(MAKE) package-via-docker-386 + publish-vmrestore: APP_NAME=vmrestore $(MAKE) publish-via-docker +vmrestore-pure: + APP_NAME=vmrestore $(MAKE) app-local-pure + vmrestore-arm: CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-arm ./app/vmrestore -vmrestore-arm-prod: - APP_NAME=vmrestore APP_SUFFIX='-arm' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=arm' $(MAKE) app-via-docker - vmrestore-arm64: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-arm64 ./app/vmrestore -vmrestore-arm64-prod: - APP_NAME=vmrestore APP_SUFFIX='-arm64' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=arm64' $(MAKE) app-via-docker +vmrestore-ppc64le: + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-ppc64le ./app/vmrestore vmrestore-386: CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-386 ./app/vmrestore - -vmrestore-386-prod: - APP_NAME=vmrestore APP_SUFFIX='-386' DOCKER_OPTS='--env CGO_ENABLED=0 --env GOARCH=386' $(MAKE) app-via-docker - -vmrestore-pure: - APP_NAME=vmrestore $(MAKE) app-local-pure - -vmrestore-pure-prod: - APP_NAME=vmrestore APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker diff --git a/app/vmrestore/deployment/Dockerfile b/app/vmrestore/deployment/Dockerfile index 31967ae520..1affee1845 100644 --- a/app/vmrestore/deployment/Dockerfile +++ b/app/vmrestore/deployment/Dockerfile @@ -1,5 +1,7 @@ +ARG certs_image +FROM $certs_image AS certs FROM scratch -COPY --from=local/certs:1.0.3 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY bin/vmrestore-prod . -EXPOSE 8428 +COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +ARG src_binary +COPY $src_binary ./vmrestore-prod ENTRYPOINT ["/vmrestore-prod"] diff --git a/app/vmselect/Makefile b/app/vmselect/Makefile index 41de873a70..1ed4e3f0af 100644 --- a/app/vmselect/Makefile +++ b/app/vmselect/Makefile @@ -16,15 +16,15 @@ vmselect-race: vmselect-prod: APP_NAME=vmselect $(MAKE) app-via-docker +vmselect-pure-prod: + APP_NAME=vmselect $(MAKE) app-via-docker-pure + vmselect-prod-race: APP_NAME=vmselect RACE=-race $(MAKE) app-via-docker vmselect-pure: APP_NAME=vmselect $(MAKE) app-local-pure -vmselect-pure-prod: - APP_NAME=vmselect APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker - package-vmselect: APP_NAME=vmselect $(MAKE) package-via-docker diff --git a/app/vmselect/deployment/Dockerfile b/app/vmselect/deployment/Dockerfile index 117233b85e..4c57f455a4 100644 --- a/app/vmselect/deployment/Dockerfile +++ b/app/vmselect/deployment/Dockerfile @@ -1,5 +1,8 @@ +ARG certs_image +FROM $certs_image AS certs FROM scratch -COPY --from=local/certs:1.0.3 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY bin/vmselect-prod . +COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +ARG src_binary +COPY $src_binary ./vmselect-prod EXPOSE 8481 ENTRYPOINT ["/vmselect-prod"] diff --git a/app/vmstorage/Makefile b/app/vmstorage/Makefile index 94f23aca39..e17e51d5b5 100644 --- a/app/vmstorage/Makefile +++ b/app/vmstorage/Makefile @@ -16,15 +16,15 @@ vmstorage-race: vmstorage-prod: APP_NAME=vmstorage $(MAKE) app-via-docker +vmstorage-pure-prod: + APP_NAME=vmstorage $(MAKE) app-via-docker-pure + vmstorage-prod-race: APP_NAME=vmstorage RACE=-race $(MAKE) app-via-docker vmstorage-pure: APP_NAME=vmstorage $(MAKE) app-local-pure -vmstorage-pure-prod: - APP_NAME=vmstorage APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker - package-vmstorage: APP_NAME=vmstorage $(MAKE) package-via-docker diff --git a/app/vmstorage/deployment/Dockerfile b/app/vmstorage/deployment/Dockerfile index b32611fd0b..e316977a18 100644 --- a/app/vmstorage/deployment/Dockerfile +++ b/app/vmstorage/deployment/Dockerfile @@ -1,6 +1,9 @@ +ARG certs_image +FROM $certs_image AS certs FROM scratch -COPY --from=local/certs:1.0.3 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY bin/vmstorage-prod . +COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +ARG src_binary +COPY $src_binary ./vmstorage-prod EXPOSE 8482 EXPOSE 8400 EXPOSE 8401 diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index f889678507..bae7a7886e 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -1,15 +1,15 @@ # All these commands must run from repository root. -DOCKER_NAMESPACE := victoriametrics +DOCKER_NAMESPACE := docker.io/victoriametrics BUILDER_IMAGE := local/builder:go1.13.5 CERTS_IMAGE := local/certs:1.0.3 package-certs: - (docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(CERTS_IMAGE)') \ + (docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(CERTS_IMAGE)$$') \ || docker build -t $(CERTS_IMAGE) deployment/docker/certs package-builder: - (docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(BUILDER_IMAGE)') \ + (docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(BUILDER_IMAGE)$$') \ || docker build -t $(BUILDER_IMAGE) deployment/docker/builder app-via-docker: package-certs package-builder @@ -27,21 +27,106 @@ app-via-docker: package-certs package-builder -o bin/$(APP_NAME)$(APP_SUFFIX)-prod $(PKG_PREFIX)/app/$(APP_NAME) package-via-docker: - (docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)') || (\ + (docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE)$$') || (\ $(MAKE) app-via-docker && \ - docker build -t $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) -f app/$(APP_NAME)/deployment/Dockerfile .) + docker build \ + --build-arg src_binary=$(APP_NAME)$(APP_SUFFIX)-prod \ + --build-arg certs_image=$(CERTS_IMAGE) \ + -t $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) \ + -f app/$(APP_NAME)/deployment/Dockerfile bin) -publish-via-docker: package-via-docker - docker push $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) - docker tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) $(DOCKER_NAMESPACE)/$(APP_NAME):latest - docker push $(DOCKER_NAMESPACE)/$(APP_NAME):latest +package-manifest: \ + package-via-docker-amd64 \ + package-via-docker-arm \ + package-via-docker-arm64 \ + package-via-docker-ppc64le \ + package-via-docker-386 + $(MAKE) package-manifest-internal + +package-manifest-internal: + docker push $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-amd64$(RACE) + docker push $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-arm$(RACE) + docker push $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-arm64$(RACE) + docker push $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-ppc64le$(RACE) + docker push $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-386$(RACE) + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create --amend $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \ + $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-amd64$(RACE) \ + $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-arm$(RACE) \ + $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-arm64$(RACE) \ + $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-ppc64le$(RACE) \ + $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-386$(RACE) + GOARCH=amd64 $(MAKE) package-manifest-annotate-goarch + GOARCH=arm $(MAKE) package-manifest-annotate-goarch + GOARCH=arm64 $(MAKE) package-manifest-annotate-goarch + GOARCH=ppc64le $(MAKE) package-manifest-annotate-goarch + GOARCH=386 $(MAKE) package-manifest-annotate-goarch + +package-manifest-annotate-goarch: + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest annotate $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \ + $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-$(GOARCH)$(RACE) --os linux --arch $(GOARCH) + +publish-via-docker: package-manifest + docker tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-amd64$(RACE) $(DOCKER_NAMESPACE)/$(APP_NAME):latest-amd64$(RACE) + docker tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-arm$(RACE) $(DOCKER_NAMESPACE)/$(APP_NAME):latest-arm$(RACE) + docker tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-arm64$(RACE) $(DOCKER_NAMESPACE)/$(APP_NAME):latest-arm64$(RACE) + docker tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-ppc64le$(RACE) $(DOCKER_NAMESPACE)/$(APP_NAME):latest-ppc64le$(RACE) + docker tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)-386$(RACE) $(DOCKER_NAMESPACE)/$(APP_NAME):latest-386$(RACE) + PKG_TAG=latest $(MAKE) package-manifest-internal + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push --purge $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) + DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push --purge $(DOCKER_NAMESPACE)/$(APP_NAME):latest$(RACE) run-via-docker: package-via-docker docker run -it --rm \ --user $(shell id -u):$(shell id -g) \ --net host \ $(DOCKER_OPTS) \ - $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) $(ARGS) + $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) $(ARGS) + +app-via-docker-goarch: + APP_SUFFIX='-$(GOARCH)' \ + DOCKER_OPTS='--env CGO_ENABLED=0 --env GOOS=linux --env GOARCH=$(GOARCH)' \ + $(MAKE) app-via-docker + +app-via-docker-pure: + APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker + +app-via-docker-amd64: + GOARCH=amd64 $(MAKE) app-via-docker-goarch + +app-via-docker-arm: + GOARCH=arm $(MAKE) app-via-docker-goarch + +app-via-docker-arm64: + GOARCH=arm64 $(MAKE) app-via-docker-goarch + +app-via-docker-ppc64le: + GOARCH=ppc64le $(MAKE) app-via-docker-goarch + +app-via-docker-386: + GOARCH=386 $(MAKE) app-via-docker-goarch + +package-via-docker-goarch: + APP_SUFFIX='-$(GOARCH)' \ + DOCKER_OPTS='--env CGO_ENABLED=0 --env GOOS=linux --env GOARCH=$(GOARCH)' \ + $(MAKE) package-via-docker + +package-via-docker-pure: + APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) package-via-docker + +package-via-docker-amd64: + GOARCH=amd64 $(MAKE) package-via-docker-goarch + +package-via-docker-arm: + GOARCH=arm $(MAKE) package-via-docker-goarch + +package-via-docker-arm64: + GOARCH=arm64 $(MAKE) package-via-docker-goarch + +package-via-docker-ppc64le: + GOARCH=ppc64le $(MAKE) package-via-docker-goarch + +package-via-docker-386: + GOARCH=386 $(MAKE) package-via-docker-goarch remove-docker-images: docker image ls --format '{{.Repository}}\t{{.ID}}' | grep $(DOCKER_NAMESPACE)/ | grep -v /builder | awk '{print $$2}' | xargs docker image rm -f