simplify release process (#3012)

* simplify release process

* address comments

* address comments

* wip

* wip

* wip

Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>

Makefile

@@ -16,6 +16,7 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TA
 include app/*/Makefile
 include deployment/*/Makefile
 include snap/local/Makefile
+include package/release/Makefile
 
 all: \
 	victoria-metrics-prod \
@@ -168,7 +169,9 @@ publish-release:
 	git checkout $(TAG) && $(MAKE) release publish && \
 		git checkout $(TAG)-cluster && $(MAKE) release publish && \
 		git checkout $(TAG)-enterprise && $(MAKE) release publish && \
-		git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish
+		git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish && \
+		$(MAKE) github-create-release && \
+		$(MAKE) github-upload-assets
 
 release: \
 	release-victoria-metrics \

docs/Release-Guide.md

@@ -4,22 +4,56 @@ sort: 18
 
 # Release process guidance
 
+## Prereqs
+1. Make sure you have enterprise remote configured
+```
+git remote add enterprise <url>
+```
+2. Make sure you have singing key configured
+3. Make sure you have github token with at least `read:org, repo, write:packages` permissions exported under `GITHUB_TOKEN` env variable.
+   You can create token [here](https://github.com/settings/tokens)
+
 ## Release version and Docker images
 
 0. Make sure that the release commits have no security issues.
 1a. Document all the changes for new release in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md).
-1b. Add `(available in v1.xx.y)` line to feature docs introduced in the upcoming release.
+1b. Add `(available starting from v1.xx.y)` line to feature docs introduced in the upcoming release.
 2. Create the following release tags:
    * `git tag -s v1.xx.y` in `master` branch
    * `git tag -s v1.xx.y-cluster` in `cluster` branch
    * `git tag -s v1.xx.y-enterprise` in `enterprise` branch
    * `git tag -s v1.xx.y-enterprise-cluster` in `enterprise-cluster` branch
-3. Run `TAG=v1.xx.y make publish-release`. It will create `*.tar.gz` release archives with the corresponding `_checksums.txt` files inside `bin` directory and publish Docker images for the given `TAG`, `TAG-cluster`, `TAG-enterprise` and `TAG-enterprise-cluster`.
+3. Run `TAG=v1.xx.y make publish-release`. This command performs the following tasks:
-4. Push release tags to <https://github.com/VictoriaMetrics/VictoriaMetrics> : `git push origin v1.xx.y` and `git push origin v1.xx.y-cluster`. Do not push `-enterprise` tags to public repository.
+   a) Build and package binaries in `*.tar.gz` release archives with the corresponding `_checksums.txt` files inside `bin` directory.
-5. Go to <https://github.com/VictoriaMetrics/VictoriaMetrics/releases> , create new release from the pushed tag on step 4 and upload `*.tar.gz` archive with the corresponding `_checksums.txt` from step 3.
+      This step can be run manually with the command `make release` from the needed git tag.
-6. Copy the [CHANGELOG](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md) for this release to [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page.
+   b) Build and publish [multi-platform Docker images](https://docs.docker.com/build/buildx/multiplatform-images/)
-7. Bump version of the VictoriaMetrics cluster setup in for [sandbox environment](https://github.com/VictoriaMetrics/ops/blob/main/sandbox/manifests/benchmark-vm/vmcluster.yaml)
+      for the given `TAG`, `TAG-cluster`, `TAG-enterprise` and `TAG-enterprise-cluster`.
-by [opening and merging PR](https://github.com/VictoriaMetrics/ops/pull/58).
+      The multi-platform Docker image is built for the following platforms:
+      * linux/amd64
+      * linux/arm64
+      * linux/arm
+      * linux/ppc64le
+      * linux/386
+      This step can be run manually with the command `make publish` from the needed git tag.
+   c) Create draft GitHub release with the name `TAG`. This step can be run manually
+      with the command `TAG=v1.xx.y make github-create-release`.
+      The release id is stored at `/tmp/vm-github-release` file.
+   d) Upload all the binaries and checksums created at step `a` to that release.
+      This step can be run manually with the command `make github-upload-assets`.
+      It is expected that the needed release id is stored at `/tmp/vm-github-release` file,
+      which must be created at the step `c`.
+      If the upload process is interrupted by any reason, then the following recovery steps must be performed:
+      - To delete the created draft release by running the command `make github-delete-release`.
+        This command expects that the id of the release to delete is located at `/tmp/vm-github-release`
+        file created at the step `c`.
+      - To run the command `TAG=v1.xx.y make github-create-release github-upload-assets`, so new release is created
+        and all the needed assets are re-uploaded to it.
+5. Go to <https://github.com/VictoriaMetrics/VictoriaMetrics/releases> and verify that draft release with the name `TAG` has been created
+   and this release contains all the needed binaries and checksums.
+6. Update the release description with the [CHANGELOG](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md) for this release.
+7. Remove the `draft` checkbox for the `TAG` release and manually publish it.
+8. Bump version of the VictoriaMetrics cluster in the [sandbox environment](https://github.com/VictoriaMetrics/ops/blob/main/sandbox/manifests/benchmark-vm/vmcluster.yaml)
+   by [opening and merging PR](https://github.com/VictoriaMetrics/ops/pull/58).
 
 ## Building snap package
 

package/release/Makefile

@@ -0,0 +1,71 @@
+GITHUB_RELEASE_SPEC_FILE="/tmp/vm-github-release"
+GITHUB_DEBUG_FILE="/tmp/vm-github-debug"
+
+github-token-check:
+ifndef GITHUB_TOKEN
+	$(error missing GITHUB_TOKEN env var. It must contain github token for VictoriaMetrics project obtained from https://github.com/settings/tokens)
+endif
+
+github-tag-check:
+ifndef TAG
+	$(error missing TAG env var. It must contain github release tag to create)
+endif
+
+github-create-release: github-token-check github-tag-check
+	@result=$$(curl -o $(GITHUB_RELEASE_SPEC_FILE) -s -w "%{http_code}" \
+		-X POST \
+		-H "Accept: application/vnd.github+json" \
+		-H "Authorization: token $(GITHUB_TOKEN)" \
+		https://api.github.com/repos/VictoriaMetrics/VictoriaMetrics/releases \
+		-d '{"tag_name":"$(TAG)","name":"$(TAG)","body":"TODO: put here the changelog for $(TAG) release from docs/CHANGELOG.md","draft":true,"prerelease":false,"generate_release_notes":false}'); \
+		if [ $${result} = 201 ]; then \
+			release_id=$$(cat $(GITHUB_RELEASE_SPEC_FILE) | grep '"id"' -m 1 | sed -E 's/.* ([[:digit:]]+)\,/\1/'); \
+			printf "Created release $(TAG) with id=$${release_id}\n"; \
+		else \
+			printf "Failed to create release $(TAG)\n"; \
+			cat $(GITHUB_RELEASE_SPEC_FILE); \
+			exit 1; \
+		fi
+
+github-upload-assets:
+	@release_id=$$(cat $(GITHUB_RELEASE_SPEC_FILE) | grep '"id"' -m 1 | sed -E 's/.* ([[:digit:]]+)\,/\1/'); \
+	$(foreach file, $(wildcard bin/*.zip), FILE=$(file) RELEASE_ID=$${release_id} CONTENT_TYPE="application/zip" $(MAKE) github-upload-asset || exit 1;) \
+	$(foreach file, $(wildcard bin/*.tar.gz), FILE=$(file) RELEASE_ID=$${release_id} CONTENT_TYPE="application/x-gzip" $(MAKE) github-upload-asset || exit 1;) \
+	$(foreach file, $(wildcard bin/*_checksums.txt), FILE=$(file) RELEASE_ID=$${release_id} CONTENT_TYPE="text/plain" $(MAKE) github-upload-asset || exit 1;) 
+
+github-upload-asset: github-token-check
+ifndef FILE
+	$(error missing FILE env var. It must contain path to file to upload to github release)
+endif
+	@printf "Uploading $(FILE)\n"
+	@result=$$(curl -o $(GITHUB_DEBUG_FILE) -w "%{http_code}" \
+		-X POST \
+		-H "Accept: application/vnd.github+json" \
+		-H "Authorization: token $(GITHUB_TOKEN)" \
+		-H "Content-Type: $(CONTENT_TYPE)" \
+		--data-binary "@$(FILE)" \
+		https://uploads.github.com/repos/VictoriaMetrics/VictoriaMetrics/releases/$(RELEASE_ID)/assets?name=$(notdir $(FILE))); \
+		if [ $${result} = 201 ]; then \
+			printf "Upload OK: $${result}\n"; \
+		elif [ $${result} = 422 ]; then \
+			printf "Asset already uploaded, you need to delete it from UI if you want to re-upload it\n"; \
+		else \
+			printf "Upload failed: $${result}\n"; \
+			cat $(GITHUB_DEBUG_FILE); \
+			exit 1; \
+		fi
+
+github-delete-release: github-token-check
+	@release_id=$$(cat $(GITHUB_RELEASE_SPEC_FILE) | grep '"id"' -m 1 | sed -E 's/.* ([[:digit:]]+)\,/\1/'); \
+	result=$$(curl -o $(GITHUB_DEBUG_FILE) -s -w "%{http_code}" \
+		-X DELETE \
+		-H "Accept: application/vnd.github+json" \
+		-H "Authorization: token $(GITHUB_TOKEN)" \
+		https://api.github.com/repos/VictoriaMetrics/VictoriaMetrics/releases/$${release_id}); \
+		if [ $${result} = 204 ]; then \
+			printf "Deleted release with id=$${release_id}\n"; \
+		else \
+			printf "Failed to delete release with id=$${release_id}\n"; \
+			cat $(GITHUB_DEBUG_FILE); \
+			exit 1; \
+		fi