deployment: build image for vmagent streamaggr benchmark (#6515)

### Describe Your Changes optionally build vmagent image for benchmark needed for https://github.com/VictoriaMetrics/ops/pull/1297 ### Checklist The following checks are **mandatory**: - [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).
2024-11-21 14:44:00 +00:00 · 2024-06-24 17:28:50 +03:00 · 2024-06-24 17:28:50 +03:00 · 6b128da811
commit 6b128da811
parent 1af13208c2
13 changed files with 98 additions and 40 deletions
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@ -0,0 +1,56 @@
+name: benchmark
+on:
+  push:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "dashboards/**"
+      - "deployment/**.yml"
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "dashboards/**"
+      - "deployment/**.yml"
+permissions:
+  contents: read
+  packages: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-streamaggr-benchmark-image:
+    name: build
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'streamaggr-benchmark')
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Package VMAgent Docker image for benchmark
+        run: |
+          SKIP_SCRATCH_BUILD=true \
+          DOCKER_BUILD_OPTS='--cache-to type=gha,mode=max --cache-from type=gha' \
+          PKG_TAG=${{ github.event.pull_request.head.sha }} \
+          DOCKER_REGISTRY=ghcr.io \
+          TARGET_PLATFORM=linux/amd64 make publish-vmagent
--- a/app/victoria-logs/multiarch/Dockerfile
+++ b/app/victoria-logs/multiarch/Dockerfile
@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates

 FROM $root_image
--- a/app/victoria-metrics/multiarch/Dockerfile
+++ b/app/victoria-metrics/multiarch/Dockerfile
@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates

 FROM $root_image
--- a/app/vmagent/multiarch/Dockerfile
+++ b/app/vmagent/multiarch/Dockerfile
@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates

 FROM $root_image
--- a/app/vmalert-tool/multiarch/Dockerfile
+++ b/app/vmalert-tool/multiarch/Dockerfile
@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates

 FROM $root_image
--- a/app/vmalert/multiarch/Dockerfile
+++ b/app/vmalert/multiarch/Dockerfile
@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates

 FROM $root_image
--- a/app/vmauth/multiarch/Dockerfile
+++ b/app/vmauth/multiarch/Dockerfile
@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates

 FROM $root_image
--- a/app/vmbackup/multiarch/Dockerfile
+++ b/app/vmbackup/multiarch/Dockerfile
@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates

 FROM $root_image
--- a/app/vmctl/multiarch/Dockerfile
+++ b/app/vmctl/multiarch/Dockerfile
@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates

 FROM $root_image
--- a/app/vmrestore/multiarch/Dockerfile
+++ b/app/vmrestore/multiarch/Dockerfile
@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates

 FROM $root_image
--- a/app/vmui/Dockerfile-web
+++ b/app/vmui/Dockerfile-web
@ -1,4 +1,4 @@
-FROM golang:1.22.4 as build-web-stage
+FROM golang:1.22.4 AS build-web-stage
 COPY build /build

 WORKDIR /build
--- a/deployment/docker/Makefile
+++ b/deployment/docker/Makefile
@ -1,19 +1,24 @@
 # All these commands must run from repository root.

+DOCKER_REGISTRY ?= docker.io
 DOCKER_NAMESPACE ?= victoriametrics

 ROOT_IMAGE ?= alpine:3.20.0
 ROOT_IMAGE_SCRATCH ?= scratch
+SKIP_SCRATCH_BUILD ?= false
 CERTS_IMAGE := alpine:3.20.0

 GO_BUILDER_IMAGE := golang:1.22.4-alpine
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
 DOCKER ?= docker
+DOCKER_BUILD_OPTS ?=
 DOCKER_RUN ?= $(DOCKER) run
 DOCKER_BUILD ?= $(DOCKER) build
 DOCKER_COMPOSE ?= $(DOCKER) compose
 DOCKER_IMAGE_LS ?= $(DOCKER) image ls --format '{{.Repository}}:{{.Tag}}'
+TARGET_PLATFORM ?= linux/amd64,linux/arm,linux/arm64,linux/ppc64le,linux/386
+COMMA := ,

 package-base:
 	($(DOCKER_IMAGE_LS) | grep -q '$(BASE_IMAGE)$$') \
@ -21,6 +26,7 @@ package-base:
 			--build-arg root_image=$(ROOT_IMAGE) \
 			--build-arg certs_image=$(CERTS_IMAGE) \
 			--tag $(BASE_IMAGE) \
+			$(DOCKER_BUILD_OPTS) \
 			deployment/docker/base

 package-builder:
@ -28,6 +34,7 @@ package-builder:
 		|| $(DOCKER_BUILD) \
 			--build-arg go_builder_image=$(GO_BUILDER_IMAGE) \
 			--tag $(BUILDER_IMAGE) \
+			$(DOCKER_BUILD_OPTS) \
 			deployment/docker/builder

 app-via-docker: package-builder
@ -61,57 +68,52 @@ app-via-docker-windows: package-builder
 			-o bin/$(APP_NAME)-windows$(APP_SUFFIX)-prod.exe $(PKG_PREFIX)/app/$(APP_NAME)

 package-via-docker: package-base
-	($(DOCKER_IMAGE_LS) | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE)$$') || (\
+	($(DOCKER_IMAGE_LS) | grep -q '$(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE)$$') || (\
 		$(MAKE) app-via-docker && \
 		$(DOCKER_BUILD) \
 			--build-arg src_binary=$(APP_NAME)$(APP_SUFFIX)-prod \
 			--build-arg base_image=$(BASE_IMAGE) \
-			--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) \
+			--tag $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) \
 			-f app/$(APP_NAME)/deployment/Dockerfile bin)

 publish-via-docker:
-	$(MAKE_PARALLEL) app-via-docker-linux-amd64 \
-		app-via-docker-linux-arm \
-		app-via-docker-linux-arm64 \
-		app-via-docker-linux-ppc64le \
-		app-via-docker-linux-386
+	$(eval TARGET_PLATFORMS := $(subst $(COMMA), ,$(TARGET_PLATFORM)))
+	$(MAKE_PARALLEL) $(foreach PLATFORM,$(TARGET_PLATFORMS),app-via-docker-$(subst /,-,$(PLATFORM)))
 	$(DOCKER) buildx build \
-		--platform=linux/amd64,linux/arm,linux/arm64,linux/ppc64le,linux/386 \
+		--platform=$(TARGET_PLATFORM) \
 		--build-arg certs_image=$(CERTS_IMAGE) \
 		--build-arg root_image=$(ROOT_IMAGE) \
 		--build-arg APP_NAME=$(APP_NAME) \
-		--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \
-		--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE) \
+		--tag $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \
+		--tag $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE) \
 		-o type=image \
 		--provenance=false \
+		$(DOCKER_BUILD_OPTS) \
 		-f app/$(APP_NAME)/multiarch/Dockerfile \
 		--push \
 		bin
-	$(DOCKER) buildx build \
-		--platform=linux/amd64,linux/arm,linux/arm64,linux/ppc64le,linux/386 \
-		--build-arg certs_image=$(CERTS_IMAGE) \
-		--build-arg root_image=$(ROOT_IMAGE_SCRATCH) \
-		--build-arg APP_NAME=$(APP_NAME) \
-		--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)-scratch \
-		--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE)-scratch \
-		-o type=image \
-		--provenance=false \
-		-f app/$(APP_NAME)/multiarch/Dockerfile \
-		--push \
-		bin
-	cd bin && rm -rf \
-		$(APP_NAME)-linux-amd64-prod \
-		$(APP_NAME)-linux-arm-prod \
-		$(APP_NAME)-linux-arm64-prod \
-		$(APP_NAME)-linux-ppc64le-prod \
-		$(APP_NAME)-linux-386-prod
+	$(if $(findstring $(SKIP_SCRATCH_BUILD),true),, \
+		$(DOCKER) buildx build \
+			--platform=$(TARGET_PLATFORM) \
+			--build-arg certs_image=$(CERTS_IMAGE) \
+			--build-arg root_image=$(ROOT_IMAGE_SCRATCH) \
+			--build-arg APP_NAME=$(APP_NAME) \
+			--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)-scratch \
+			--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE)-scratch \
+			-o type=image \
+			--provenance=false \
+			$(DOCKER_BUILD_OPTS) \
+			-f app/$(APP_NAME)/multiarch/Dockerfile \
+			--push \
+			bin)
+	cd bin && rm -rf $(foreach PLATFORM,$(TARGET_PLATFORMS),$(APP_NAME)-$(subst /,-,$(PLATFORM))-prod)

 run-via-docker: package-via-docker
 	$(DOCKER_RUN) -it --rm \
 		--user $(shell id -u):$(shell id -g) \
 		--net host \
 		$(DOCKER_OPTS) \
-		$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) $(ARGS)
+		$(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) $(ARGS)

 app-via-docker-goos-goarch:
 	APP_SUFFIX='-$(GOOS)-$(GOARCH)' \
--- a/deployment/docker/base/Dockerfile
+++ b/deployment/docker/base/Dockerfile
@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image as certs
+FROM $certs_image AS certs

 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates