diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..eb6d89d52 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,56 @@ +name: benchmark +on: + push: + branches: + - master + - cluster + paths-ignore: + - "docs/**" + - "**.md" + - "dashboards/**" + - "deployment/**.yml" + pull_request: + types: + - opened + - synchronize + - reopened + - labeled + branches: + - master + - cluster + paths-ignore: + - "docs/**" + - "**.md" + - "dashboards/**" + - "deployment/**.yml" +permissions: + contents: read + packages: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build-streamaggr-benchmark-image: + name: build + runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'streamaggr-benchmark') + steps: + - name: Code checkout + uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Package VMAgent Docker image for benchmark + run: | + SKIP_SCRATCH_BUILD=true \ + DOCKER_BUILD_OPTS='--cache-to type=gha,mode=max --cache-from type=gha' \ + PKG_TAG=${{ github.event.pull_request.head.sha }} \ + DOCKER_REGISTRY=ghcr.io \ + TARGET_PLATFORM=linux/amd64 make publish-vmagent diff --git a/app/victoria-logs/multiarch/Dockerfile b/app/victoria-logs/multiarch/Dockerfile index 17375a2ed..cb63c1ee8 100644 --- a/app/victoria-logs/multiarch/Dockerfile +++ b/app/victoria-logs/multiarch/Dockerfile @@ -1,7 +1,7 @@ # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b ARG certs_image ARG root_image -FROM $certs_image as certs +FROM $certs_image AS certs RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates FROM $root_image diff --git a/app/victoria-metrics/multiarch/Dockerfile b/app/victoria-metrics/multiarch/Dockerfile index a52c7ccf1..d14169665 100644 --- a/app/victoria-metrics/multiarch/Dockerfile +++ b/app/victoria-metrics/multiarch/Dockerfile @@ -1,7 +1,7 @@ # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b ARG certs_image ARG root_image -FROM $certs_image as certs +FROM $certs_image AS certs RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates FROM $root_image diff --git a/app/vmagent/multiarch/Dockerfile b/app/vmagent/multiarch/Dockerfile index 47370c9cd..7db8037a6 100644 --- a/app/vmagent/multiarch/Dockerfile +++ b/app/vmagent/multiarch/Dockerfile @@ -1,7 +1,7 @@ # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b ARG certs_image ARG root_image -FROM $certs_image as certs +FROM $certs_image AS certs RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates FROM $root_image diff --git a/app/vmalert-tool/multiarch/Dockerfile b/app/vmalert-tool/multiarch/Dockerfile index e16c94d84..ef1635ec7 100644 --- a/app/vmalert-tool/multiarch/Dockerfile +++ b/app/vmalert-tool/multiarch/Dockerfile @@ -1,7 +1,7 @@ # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b ARG certs_image ARG root_image -FROM $certs_image as certs +FROM $certs_image AS certs RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates FROM $root_image diff --git a/app/vmalert/multiarch/Dockerfile b/app/vmalert/multiarch/Dockerfile index 500b9bcb1..e2f272391 100644 --- a/app/vmalert/multiarch/Dockerfile +++ b/app/vmalert/multiarch/Dockerfile @@ -1,7 +1,7 @@ # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b ARG certs_image ARG root_image -FROM $certs_image as certs +FROM $certs_image AS certs RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates FROM $root_image diff --git a/app/vmauth/multiarch/Dockerfile b/app/vmauth/multiarch/Dockerfile index 96aa589b2..61322beec 100644 --- a/app/vmauth/multiarch/Dockerfile +++ b/app/vmauth/multiarch/Dockerfile @@ -1,7 +1,7 @@ # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b ARG certs_image ARG root_image -FROM $certs_image as certs +FROM $certs_image AS certs RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates FROM $root_image diff --git a/app/vmbackup/multiarch/Dockerfile b/app/vmbackup/multiarch/Dockerfile index 747c6b7b4..157333d36 100644 --- a/app/vmbackup/multiarch/Dockerfile +++ b/app/vmbackup/multiarch/Dockerfile @@ -1,7 +1,7 @@ # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b ARG certs_image ARG root_image -FROM $certs_image as certs +FROM $certs_image AS certs RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates FROM $root_image diff --git a/app/vmctl/multiarch/Dockerfile b/app/vmctl/multiarch/Dockerfile index d48338309..ac0963199 100644 --- a/app/vmctl/multiarch/Dockerfile +++ b/app/vmctl/multiarch/Dockerfile @@ -1,7 +1,7 @@ # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b ARG certs_image ARG root_image -FROM $certs_image as certs +FROM $certs_image AS certs RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates FROM $root_image diff --git a/app/vmrestore/multiarch/Dockerfile b/app/vmrestore/multiarch/Dockerfile index 504656508..108274007 100644 --- a/app/vmrestore/multiarch/Dockerfile +++ b/app/vmrestore/multiarch/Dockerfile @@ -1,7 +1,7 @@ # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b ARG certs_image ARG root_image -FROM $certs_image as certs +FROM $certs_image AS certs RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates FROM $root_image diff --git a/app/vmui/Dockerfile-web b/app/vmui/Dockerfile-web index 52dfe7a7e..80dfe8df6 100644 --- a/app/vmui/Dockerfile-web +++ b/app/vmui/Dockerfile-web @@ -1,4 +1,4 @@ -FROM golang:1.22.4 as build-web-stage +FROM golang:1.22.4 AS build-web-stage COPY build /build WORKDIR /build diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index 6f961a9d1..3029ee8d1 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -1,19 +1,24 @@ # All these commands must run from repository root. +DOCKER_REGISTRY ?= docker.io DOCKER_NAMESPACE ?= victoriametrics ROOT_IMAGE ?= alpine:3.20.0 ROOT_IMAGE_SCRATCH ?= scratch +SKIP_SCRATCH_BUILD ?= false CERTS_IMAGE := alpine:3.20.0 GO_BUILDER_IMAGE := golang:1.22.4-alpine BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __) DOCKER ?= docker +DOCKER_BUILD_OPTS ?= DOCKER_RUN ?= $(DOCKER) run DOCKER_BUILD ?= $(DOCKER) build DOCKER_COMPOSE ?= $(DOCKER) compose DOCKER_IMAGE_LS ?= $(DOCKER) image ls --format '{{.Repository}}:{{.Tag}}' +TARGET_PLATFORM ?= linux/amd64,linux/arm,linux/arm64,linux/ppc64le,linux/386 +COMMA := , package-base: ($(DOCKER_IMAGE_LS) | grep -q '$(BASE_IMAGE)$$') \ @@ -21,6 +26,7 @@ package-base: --build-arg root_image=$(ROOT_IMAGE) \ --build-arg certs_image=$(CERTS_IMAGE) \ --tag $(BASE_IMAGE) \ + $(DOCKER_BUILD_OPTS) \ deployment/docker/base package-builder: @@ -28,6 +34,7 @@ package-builder: || $(DOCKER_BUILD) \ --build-arg go_builder_image=$(GO_BUILDER_IMAGE) \ --tag $(BUILDER_IMAGE) \ + $(DOCKER_BUILD_OPTS) \ deployment/docker/builder app-via-docker: package-builder @@ -61,57 +68,52 @@ app-via-docker-windows: package-builder -o bin/$(APP_NAME)-windows$(APP_SUFFIX)-prod.exe $(PKG_PREFIX)/app/$(APP_NAME) package-via-docker: package-base - ($(DOCKER_IMAGE_LS) | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE)$$') || (\ + ($(DOCKER_IMAGE_LS) | grep -q '$(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE)$$') || (\ $(MAKE) app-via-docker && \ $(DOCKER_BUILD) \ --build-arg src_binary=$(APP_NAME)$(APP_SUFFIX)-prod \ --build-arg base_image=$(BASE_IMAGE) \ - --tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) \ + --tag $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) \ -f app/$(APP_NAME)/deployment/Dockerfile bin) publish-via-docker: - $(MAKE_PARALLEL) app-via-docker-linux-amd64 \ - app-via-docker-linux-arm \ - app-via-docker-linux-arm64 \ - app-via-docker-linux-ppc64le \ - app-via-docker-linux-386 + $(eval TARGET_PLATFORMS := $(subst $(COMMA), ,$(TARGET_PLATFORM))) + $(MAKE_PARALLEL) $(foreach PLATFORM,$(TARGET_PLATFORMS),app-via-docker-$(subst /,-,$(PLATFORM))) $(DOCKER) buildx build \ - --platform=linux/amd64,linux/arm,linux/arm64,linux/ppc64le,linux/386 \ + --platform=$(TARGET_PLATFORM) \ --build-arg certs_image=$(CERTS_IMAGE) \ --build-arg root_image=$(ROOT_IMAGE) \ --build-arg APP_NAME=$(APP_NAME) \ - --tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \ - --tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE) \ + --tag $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \ + --tag $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE) \ -o type=image \ --provenance=false \ + $(DOCKER_BUILD_OPTS) \ -f app/$(APP_NAME)/multiarch/Dockerfile \ --push \ bin - $(DOCKER) buildx build \ - --platform=linux/amd64,linux/arm,linux/arm64,linux/ppc64le,linux/386 \ - --build-arg certs_image=$(CERTS_IMAGE) \ - --build-arg root_image=$(ROOT_IMAGE_SCRATCH) \ - --build-arg APP_NAME=$(APP_NAME) \ - --tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)-scratch \ - --tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE)-scratch \ - -o type=image \ - --provenance=false \ - -f app/$(APP_NAME)/multiarch/Dockerfile \ - --push \ - bin - cd bin && rm -rf \ - $(APP_NAME)-linux-amd64-prod \ - $(APP_NAME)-linux-arm-prod \ - $(APP_NAME)-linux-arm64-prod \ - $(APP_NAME)-linux-ppc64le-prod \ - $(APP_NAME)-linux-386-prod + $(if $(findstring $(SKIP_SCRATCH_BUILD),true),, \ + $(DOCKER) buildx build \ + --platform=$(TARGET_PLATFORM) \ + --build-arg certs_image=$(CERTS_IMAGE) \ + --build-arg root_image=$(ROOT_IMAGE_SCRATCH) \ + --build-arg APP_NAME=$(APP_NAME) \ + --tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)-scratch \ + --tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE)-scratch \ + -o type=image \ + --provenance=false \ + $(DOCKER_BUILD_OPTS) \ + -f app/$(APP_NAME)/multiarch/Dockerfile \ + --push \ + bin) + cd bin && rm -rf $(foreach PLATFORM,$(TARGET_PLATFORMS),$(APP_NAME)-$(subst /,-,$(PLATFORM))-prod) run-via-docker: package-via-docker $(DOCKER_RUN) -it --rm \ --user $(shell id -u):$(shell id -g) \ --net host \ $(DOCKER_OPTS) \ - $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) $(ARGS) + $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) $(ARGS) app-via-docker-goos-goarch: APP_SUFFIX='-$(GOOS)-$(GOARCH)' \ diff --git a/deployment/docker/base/Dockerfile b/deployment/docker/base/Dockerfile index 6d06e75ce..7cadd9e25 100644 --- a/deployment/docker/base/Dockerfile +++ b/deployment/docker/base/Dockerfile @@ -1,7 +1,7 @@ # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b ARG certs_image ARG root_image -FROM $certs_image as certs +FROM $certs_image AS certs RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates