deployment: build image for vmagent streamaggr benchmark (#6515)

### Describe Your Changes

optionally build vmagent image for benchmark
needed for https://github.com/VictoriaMetrics/ops/pull/1297

### Checklist

The following checks are **mandatory**:

- [ ] My change adheres [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/contributing/).
This commit is contained in:
Andrii Chubatiuk 2024-06-24 17:28:50 +03:00 committed by GitHub
parent 1af13208c2
commit 6b128da811
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 98 additions and 40 deletions

56
.github/workflows/benchmark.yml vendored Normal file
View file

@ -0,0 +1,56 @@
name: benchmark
on:
push:
branches:
- master
- cluster
paths-ignore:
- "docs/**"
- "**.md"
- "dashboards/**"
- "deployment/**.yml"
pull_request:
types:
- opened
- synchronize
- reopened
- labeled
branches:
- master
- cluster
paths-ignore:
- "docs/**"
- "**.md"
- "dashboards/**"
- "deployment/**.yml"
permissions:
contents: read
packages: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build-streamaggr-benchmark-image:
name: build
runs-on: ubuntu-latest
if: contains(github.event.pull_request.labels.*.name, 'streamaggr-benchmark')
steps:
- name: Code checkout
uses: actions/checkout@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Package VMAgent Docker image for benchmark
run: |
SKIP_SCRATCH_BUILD=true \
DOCKER_BUILD_OPTS='--cache-to type=gha,mode=max --cache-from type=gha' \
PKG_TAG=${{ github.event.pull_request.head.sha }} \
DOCKER_REGISTRY=ghcr.io \
TARGET_PLATFORM=linux/amd64 make publish-vmagent

View file

@ -1,7 +1,7 @@
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
ARG certs_image ARG certs_image
ARG root_image ARG root_image
FROM $certs_image as certs FROM $certs_image AS certs
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
FROM $root_image FROM $root_image

View file

@ -1,7 +1,7 @@
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
ARG certs_image ARG certs_image
ARG root_image ARG root_image
FROM $certs_image as certs FROM $certs_image AS certs
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
FROM $root_image FROM $root_image

View file

@ -1,7 +1,7 @@
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
ARG certs_image ARG certs_image
ARG root_image ARG root_image
FROM $certs_image as certs FROM $certs_image AS certs
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
FROM $root_image FROM $root_image

View file

@ -1,7 +1,7 @@
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
ARG certs_image ARG certs_image
ARG root_image ARG root_image
FROM $certs_image as certs FROM $certs_image AS certs
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
FROM $root_image FROM $root_image

View file

@ -1,7 +1,7 @@
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
ARG certs_image ARG certs_image
ARG root_image ARG root_image
FROM $certs_image as certs FROM $certs_image AS certs
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
FROM $root_image FROM $root_image

View file

@ -1,7 +1,7 @@
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
ARG certs_image ARG certs_image
ARG root_image ARG root_image
FROM $certs_image as certs FROM $certs_image AS certs
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
FROM $root_image FROM $root_image

View file

@ -1,7 +1,7 @@
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
ARG certs_image ARG certs_image
ARG root_image ARG root_image
FROM $certs_image as certs FROM $certs_image AS certs
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
FROM $root_image FROM $root_image

View file

@ -1,7 +1,7 @@
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
ARG certs_image ARG certs_image
ARG root_image ARG root_image
FROM $certs_image as certs FROM $certs_image AS certs
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
FROM $root_image FROM $root_image

View file

@ -1,7 +1,7 @@
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
ARG certs_image ARG certs_image
ARG root_image ARG root_image
FROM $certs_image as certs FROM $certs_image AS certs
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
FROM $root_image FROM $root_image

View file

@ -1,4 +1,4 @@
FROM golang:1.22.4 as build-web-stage FROM golang:1.22.4 AS build-web-stage
COPY build /build COPY build /build
WORKDIR /build WORKDIR /build

View file

@ -1,19 +1,24 @@
# All these commands must run from repository root. # All these commands must run from repository root.
DOCKER_REGISTRY ?= docker.io
DOCKER_NAMESPACE ?= victoriametrics DOCKER_NAMESPACE ?= victoriametrics
ROOT_IMAGE ?= alpine:3.20.0 ROOT_IMAGE ?= alpine:3.20.0
ROOT_IMAGE_SCRATCH ?= scratch ROOT_IMAGE_SCRATCH ?= scratch
SKIP_SCRATCH_BUILD ?= false
CERTS_IMAGE := alpine:3.20.0 CERTS_IMAGE := alpine:3.20.0
GO_BUILDER_IMAGE := golang:1.22.4-alpine GO_BUILDER_IMAGE := golang:1.22.4-alpine
BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __) BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
DOCKER ?= docker DOCKER ?= docker
DOCKER_BUILD_OPTS ?=
DOCKER_RUN ?= $(DOCKER) run DOCKER_RUN ?= $(DOCKER) run
DOCKER_BUILD ?= $(DOCKER) build DOCKER_BUILD ?= $(DOCKER) build
DOCKER_COMPOSE ?= $(DOCKER) compose DOCKER_COMPOSE ?= $(DOCKER) compose
DOCKER_IMAGE_LS ?= $(DOCKER) image ls --format '{{.Repository}}:{{.Tag}}' DOCKER_IMAGE_LS ?= $(DOCKER) image ls --format '{{.Repository}}:{{.Tag}}'
TARGET_PLATFORM ?= linux/amd64,linux/arm,linux/arm64,linux/ppc64le,linux/386
COMMA := ,
package-base: package-base:
($(DOCKER_IMAGE_LS) | grep -q '$(BASE_IMAGE)$$') \ ($(DOCKER_IMAGE_LS) | grep -q '$(BASE_IMAGE)$$') \
@ -21,6 +26,7 @@ package-base:
--build-arg root_image=$(ROOT_IMAGE) \ --build-arg root_image=$(ROOT_IMAGE) \
--build-arg certs_image=$(CERTS_IMAGE) \ --build-arg certs_image=$(CERTS_IMAGE) \
--tag $(BASE_IMAGE) \ --tag $(BASE_IMAGE) \
$(DOCKER_BUILD_OPTS) \
deployment/docker/base deployment/docker/base
package-builder: package-builder:
@ -28,6 +34,7 @@ package-builder:
|| $(DOCKER_BUILD) \ || $(DOCKER_BUILD) \
--build-arg go_builder_image=$(GO_BUILDER_IMAGE) \ --build-arg go_builder_image=$(GO_BUILDER_IMAGE) \
--tag $(BUILDER_IMAGE) \ --tag $(BUILDER_IMAGE) \
$(DOCKER_BUILD_OPTS) \
deployment/docker/builder deployment/docker/builder
app-via-docker: package-builder app-via-docker: package-builder
@ -61,57 +68,52 @@ app-via-docker-windows: package-builder
-o bin/$(APP_NAME)-windows$(APP_SUFFIX)-prod.exe $(PKG_PREFIX)/app/$(APP_NAME) -o bin/$(APP_NAME)-windows$(APP_SUFFIX)-prod.exe $(PKG_PREFIX)/app/$(APP_NAME)
package-via-docker: package-base package-via-docker: package-base
($(DOCKER_IMAGE_LS) | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE)$$') || (\ ($(DOCKER_IMAGE_LS) | grep -q '$(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE)$$') || (\
$(MAKE) app-via-docker && \ $(MAKE) app-via-docker && \
$(DOCKER_BUILD) \ $(DOCKER_BUILD) \
--build-arg src_binary=$(APP_NAME)$(APP_SUFFIX)-prod \ --build-arg src_binary=$(APP_NAME)$(APP_SUFFIX)-prod \
--build-arg base_image=$(BASE_IMAGE) \ --build-arg base_image=$(BASE_IMAGE) \
--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) \ --tag $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) \
-f app/$(APP_NAME)/deployment/Dockerfile bin) -f app/$(APP_NAME)/deployment/Dockerfile bin)
publish-via-docker: publish-via-docker:
$(MAKE_PARALLEL) app-via-docker-linux-amd64 \ $(eval TARGET_PLATFORMS := $(subst $(COMMA), ,$(TARGET_PLATFORM)))
app-via-docker-linux-arm \ $(MAKE_PARALLEL) $(foreach PLATFORM,$(TARGET_PLATFORMS),app-via-docker-$(subst /,-,$(PLATFORM)))
app-via-docker-linux-arm64 \
app-via-docker-linux-ppc64le \
app-via-docker-linux-386
$(DOCKER) buildx build \ $(DOCKER) buildx build \
--platform=linux/amd64,linux/arm,linux/arm64,linux/ppc64le,linux/386 \ --platform=$(TARGET_PLATFORM) \
--build-arg certs_image=$(CERTS_IMAGE) \ --build-arg certs_image=$(CERTS_IMAGE) \
--build-arg root_image=$(ROOT_IMAGE) \ --build-arg root_image=$(ROOT_IMAGE) \
--build-arg APP_NAME=$(APP_NAME) \ --build-arg APP_NAME=$(APP_NAME) \
--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \ --tag $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \
--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE) \ --tag $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE) \
-o type=image \ -o type=image \
--provenance=false \ --provenance=false \
$(DOCKER_BUILD_OPTS) \
-f app/$(APP_NAME)/multiarch/Dockerfile \ -f app/$(APP_NAME)/multiarch/Dockerfile \
--push \ --push \
bin bin
$(DOCKER) buildx build \ $(if $(findstring $(SKIP_SCRATCH_BUILD),true),, \
--platform=linux/amd64,linux/arm,linux/arm64,linux/ppc64le,linux/386 \ $(DOCKER) buildx build \
--build-arg certs_image=$(CERTS_IMAGE) \ --platform=$(TARGET_PLATFORM) \
--build-arg root_image=$(ROOT_IMAGE_SCRATCH) \ --build-arg certs_image=$(CERTS_IMAGE) \
--build-arg APP_NAME=$(APP_NAME) \ --build-arg root_image=$(ROOT_IMAGE_SCRATCH) \
--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)-scratch \ --build-arg APP_NAME=$(APP_NAME) \
--tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE)-scratch \ --tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)-scratch \
-o type=image \ --tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(LATEST_TAG)$(RACE)-scratch \
--provenance=false \ -o type=image \
-f app/$(APP_NAME)/multiarch/Dockerfile \ --provenance=false \
--push \ $(DOCKER_BUILD_OPTS) \
bin -f app/$(APP_NAME)/multiarch/Dockerfile \
cd bin && rm -rf \ --push \
$(APP_NAME)-linux-amd64-prod \ bin)
$(APP_NAME)-linux-arm-prod \ cd bin && rm -rf $(foreach PLATFORM,$(TARGET_PLATFORMS),$(APP_NAME)-$(subst /,-,$(PLATFORM))-prod)
$(APP_NAME)-linux-arm64-prod \
$(APP_NAME)-linux-ppc64le-prod \
$(APP_NAME)-linux-386-prod
run-via-docker: package-via-docker run-via-docker: package-via-docker
$(DOCKER_RUN) -it --rm \ $(DOCKER_RUN) -it --rm \
--user $(shell id -u):$(shell id -g) \ --user $(shell id -u):$(shell id -g) \
--net host \ --net host \
$(DOCKER_OPTS) \ $(DOCKER_OPTS) \
$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) $(ARGS) $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) $(ARGS)
app-via-docker-goos-goarch: app-via-docker-goos-goarch:
APP_SUFFIX='-$(GOOS)-$(GOARCH)' \ APP_SUFFIX='-$(GOOS)-$(GOARCH)' \

View file

@ -1,7 +1,7 @@
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
ARG certs_image ARG certs_image
ARG root_image ARG root_image
FROM $certs_image as certs FROM $certs_image AS certs
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates