Implement SBOM generation

Signed-off-by: Arkadii Yakovets <ark@victoriametrics.com>

Makefile

@@ -14,6 +14,10 @@ endif
 
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 
+VICTORIA_LOGS_COMPONENTS = victoria-logs
+VICTORIA_METRICS_COMPONENTS = victoria-metrics
+VICTORIA_METRICS_UTILS_COMPONENTS = vmagent vmalert vmalert-tool vmauth vmbackup vmrestore vmctl
+
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
@@ -23,6 +27,73 @@ include deployment/*/Makefile
 include dashboards/Makefile
 include package/release/Makefile
 
+define RELEASE_GOOS_GOARCH
+	$(eval PKG_NAME := $(1))
+	$(eval PKG_COMPONENTS := $(2))
+
+	# Build
+	$(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
+
+	# Generate SBOM
+	mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom"
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cyclonedx-gomod app -assert-licenses -json -licenses -packages \
+			-main app/$(pkg_component) \
+			-output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json)
+	
+	# Pack and compress
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cd bin && tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar \
+			$(pkg_component)-$(GOOS)-$(GOARCH)-prod)
+	cd bin && gzip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar
+	cd bin && tar -czf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
+
+	# Generate checksums
+	cd bin && \
+		sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \
+		sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod | sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ >> \
+			$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt)
+
+	# Clean up
+	cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
+	$(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
+endef
+
+define RELEASE_WINDOWS_GOARCH
+	$(eval PKG_NAME := $(1))
+	$(eval PKG_COMPONENTS := $(2))
+
+	# Build
+	$(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
+
+	# Generate SBOM
+	mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom"
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cyclonedx-gomod app -assert-licenses -json -licenses -packages \
+			-main app/$(pkg_component) \
+			-output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json)
+	
+	# Pack and compress
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cd bin && zip -u $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip \
+			$(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe)
+	cd bin && zip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip -r $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
+
+	# Generate checksums
+	cd bin && \
+		sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \
+		sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	$(foreach pkg_component, $(PKG_COMPONENTS),
+		cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt)
+
+	# Clean up
+	cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
+	$(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe)
+endef
+
+
 all: \
 	victoria-metrics-prod \
 	victoria-logs-prod \
@@ -241,26 +312,13 @@ release-victoria-metrics-openbsd-amd64:
 	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
 release-victoria-metrics-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
+	GOOS=windows GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
 
-release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
+release-victoria-metrics-goos-goarch:
-	cd bin && \
+	$(call RELEASE_GOOS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS))
-		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
-		&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
-			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod
 
-release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
+release-victoria-metrics-windows-goarch:
-	cd bin && \
+	$(call RELEASE_WINDOWS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS))
-		zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			victoria-metrics-windows-$(GOARCH)-prod.exe \
-		&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			victoria-metrics-windows-$(GOARCH)-prod.exe \
-			> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		victoria-metrics-windows-$(GOARCH)-prod.exe
 
 release-victoria-logs:
 	$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
@@ -355,77 +413,13 @@ release-vmutils-openbsd-amd64:
 	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
 release-vmutils-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
+	GOOS=windows GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
 
-release-vmutils-goos-goarch: \
+release-vmutils-goos-goarch:
-	vmagent-$(GOOS)-$(GOARCH)-prod \
+	$(call RELEASE_GOOS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS))
-	vmalert-$(GOOS)-$(GOARCH)-prod \
-	vmalert-tool-$(GOOS)-$(GOARCH)-prod \
-	vmauth-$(GOOS)-$(GOARCH)-prod \
-	vmbackup-$(GOOS)-$(GOARCH)-prod \
-	vmrestore-$(GOOS)-$(GOARCH)-prod \
-	vmctl-$(GOOS)-$(GOARCH)-prod
-	cd bin && \
-		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(GOOS)-$(GOARCH)-prod \
-			vmalert-$(GOOS)-$(GOARCH)-prod \
-			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
-			vmauth-$(GOOS)-$(GOARCH)-prod \
-			vmbackup-$(GOOS)-$(GOARCH)-prod \
-			vmrestore-$(GOOS)-$(GOARCH)-prod \
-			vmctl-$(GOOS)-$(GOARCH)-prod \
-		&& sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			vmagent-$(GOOS)-$(GOARCH)-prod \
-			vmalert-$(GOOS)-$(GOARCH)-prod \
-			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
-			vmauth-$(GOOS)-$(GOARCH)-prod \
-			vmbackup-$(GOOS)-$(GOARCH)-prod \
-			vmrestore-$(GOOS)-$(GOARCH)-prod \
-			vmctl-$(GOOS)-$(GOARCH)-prod \
-			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vmagent-$(GOOS)-$(GOARCH)-prod \
-		vmalert-$(GOOS)-$(GOARCH)-prod \
-		vmalert-tool-$(GOOS)-$(GOARCH)-prod \
-		vmauth-$(GOOS)-$(GOARCH)-prod \
-		vmbackup-$(GOOS)-$(GOARCH)-prod \
-		vmrestore-$(GOOS)-$(GOARCH)-prod \
-		vmctl-$(GOOS)-$(GOARCH)-prod
 
-release-vmutils-windows-goarch: \
+release-vmutils-windows-goarch:
-	vmagent-windows-$(GOARCH)-prod \
+	$(call RELEASE_WINDOWS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS))
-	vmalert-windows-$(GOARCH)-prod \
-	vmalert-tool-windows-$(GOARCH)-prod \
-	vmauth-windows-$(GOARCH)-prod \
-	vmbackup-windows-$(GOARCH)-prod \
-	vmrestore-windows-$(GOARCH)-prod \
-	vmctl-windows-$(GOARCH)-prod
-	cd bin && \
-		zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vmagent-windows-$(GOARCH)-prod.exe \
-			vmalert-windows-$(GOARCH)-prod.exe \
-			vmalert-tool-windows-$(GOARCH)-prod.exe \
-			vmauth-windows-$(GOARCH)-prod.exe \
-			vmbackup-windows-$(GOARCH)-prod.exe \
-			vmrestore-windows-$(GOARCH)-prod.exe \
-			vmctl-windows-$(GOARCH)-prod.exe \
-		&& sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vmagent-windows-$(GOARCH)-prod.exe \
-			vmalert-windows-$(GOARCH)-prod.exe \
-			vmalert-tool-windows-$(GOARCH)-prod.exe \
-			vmauth-windows-$(GOARCH)-prod.exe \
-			vmbackup-windows-$(GOARCH)-prod.exe \
-			vmrestore-windows-$(GOARCH)-prod.exe \
-			vmctl-windows-$(GOARCH)-prod.exe \
-			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vmagent-windows-$(GOARCH)-prod.exe \
-		vmalert-windows-$(GOARCH)-prod.exe \
-		vmalert-tool-windows-$(GOARCH)-prod.exe \
-		vmauth-windows-$(GOARCH)-prod.exe \
-		vmbackup-windows-$(GOARCH)-prod.exe \
-		vmrestore-windows-$(GOARCH)-prod.exe \
-		vmctl-windows-$(GOARCH)-prod.exe
 
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
@@ -514,6 +508,9 @@ install-wwhrd:
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
 
+cyclonedx-gomod-install:
+	which cyclonedx-gomod || go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@latest
+
 copy-docs:
 # The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
 # For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n

package/release/asset_test.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"sort"
 	"strings"
 	"testing"
 )
@@ -21,7 +22,7 @@ const unixExecSuffix = "-prod"
 const windowsExecSuffix = "-prod.exe"
 const zipExt = ".zip"
 
-func assertArchiveFile(t *testing.T, extension string, path string, expectedPrefixes []string) {
+func assertArchiveFile(t *testing.T, path string, expectedFiles []string) {
 	// Check if file exists
 	archiveFileInfo, err := getFileInfo(path)
 	if err != nil {
@@ -31,10 +32,7 @@ func assertArchiveFile(t *testing.T, extension string, path string, expectedPref
 	}
 
 	var archiveFiles []string
-	var binaryFileSuffix string
+	if strings.HasSuffix(path, tarGzExt) { // Unix-like stuff
-	if extension == tarGzExt { // Unix-like stuff
-		binaryFileSuffix = unixExecSuffix
-
 		// Get file handler
 		tarGzFile, err := os.Open(path)
 		if err != nil {
@@ -59,15 +57,15 @@ func assertArchiveFile(t *testing.T, extension string, path string, expectedPref
 			if err != nil {
 				t.Fatalf("Failed to read tar header: %s", err)
 			}
-			if header.Size == 0 {
+			if header.FileInfo().IsDir() {
-				t.Fatalf("Archive file is empty: %s (%s)", header.Name, path)
+				continue
+			} else if header.Size == 0 {
+				t.Fatalf("Archived file is empty: %s (%s)", header.Name, path)
 			}
 			archiveFiles = append(archiveFiles, header.Name)
 		}
 
-	} else if extension == zipExt { // Windows stuff
+	} else if strings.HasSuffix(path, zipExt) { // Windows stuff
-		binaryFileSuffix = "-windows-amd64" + windowsExecSuffix
-
 		// Get file handler
 		zipFile, err := os.Open(path)
 		if err != nil {
@@ -87,26 +85,24 @@ func assertArchiveFile(t *testing.T, extension string, path string, expectedPref
 		}
 
 		for _, file := range zipReader.File {
-			if file.CompressedSize64 == 0 {
+			if file.FileInfo().IsDir() {
-				t.Fatalf("Archive file is empty: %s (%s)", file.Name, path)
+				continue
+			} else if file.CompressedSize64 == 0 {
+				t.Fatalf("Archived file is empty: %s (%s)", file.Name, path)
 			}
 			archiveFiles = append(archiveFiles, file.Name)
 		}
 
 	} else { // Unexpected stuff.
-		t.Fatalf("Unknown archive type: %s", extension)
+		t.Fatalf("Unknown archive type: %s", path)
 	}
 
-	var expectedFiles []string
-	for _, expectedFilePrefix := range expectedPrefixes {
-		expectedFiles = append(expectedFiles, strings.Join([]string{expectedFilePrefix, binaryFileSuffix}, ""))
-	}
 	if !compareSlices(archiveFiles, expectedFiles) {
 		t.Fatalf("Archive contents `%s` doesn't match the expected one: `%s`", archiveFiles, expectedFiles)
 	}
 }
 
-func assertChecksumsFile(t *testing.T, extension string, path string, expectedPrefixes []string) {
+func assertChecksumsFile(t *testing.T, path string, expectedFiles []string) {
 	// Check if file exists
 	checksumsFileInfo, err := getFileInfo(path)
 	if err != nil {
@@ -130,20 +126,6 @@ func assertChecksumsFile(t *testing.T, extension string, path string, expectedPr
 		t.Fatalf("Failed to  read file: %s", err)
 	}
 
-	var binaryFileSuffix string
-	if extension == tarGzExt { // Unix-like stuff
-		binaryFileSuffix = unixExecSuffix
-	} else if extension == zipExt { // Windows stuff
-		binaryFileSuffix = "-windows-amd64" + windowsExecSuffix
-	} else { // Unexpected stuff.
-		t.Fatalf("Unknown archive type: %s", extension)
-	}
-
-	archiveFileName := strings.ReplaceAll(filepath.Base(path), "_checksums.txt", extension)
-	expectedFiles := []string{archiveFileName}
-	for _, expectedFilePrefix := range expectedPrefixes {
-		expectedFiles = append(expectedFiles, strings.Join([]string{expectedFilePrefix, binaryFileSuffix}, ""))
-	}
 	if !compareSlices(checksumsFiles, expectedFiles) {
 		t.Fatalf("Archive contents `%s` doesn't match the expected one: `%s`", checksumsFiles, expectedFiles)
 	}
@@ -153,6 +135,9 @@ func compareSlices(slice1, slice2 []string) bool {
 	if len(slice1) != len(slice2) {
 		return false
 	}
+
+	sort.Strings(slice1)
+	sort.Strings(slice2)
 	for i := range slice1 {
 		if slice1[i] != slice2[i] {
 			return false
@@ -239,24 +224,44 @@ func testReleaseAssets(t *testing.T, componentNames []string) {
 	for _, componentName := range componentNames {
 		for osName, archNames := range getArchOsMap() {
 			var archiveFileExtension string
+			var binaryFileSuffix string
 			if osName == "windows" {
 				archiveFileExtension = zipExt
+				binaryFileSuffix = "-windows-amd64" + windowsExecSuffix
 			} else {
 				archiveFileExtension = tarGzExt
+				binaryFileSuffix = unixExecSuffix
 			}
 
 			for _, archName := range archNames {
-				fileNamePrefix := strings.Join([]string{componentName, osName, archName, gitTag}, "-")
+				componentPrefix := strings.Join([]string{componentName, osName, archName, gitTag}, "-")
 
-				// Check archive file.
+				// Check binaries.
-				archiveFileName := strings.Join([]string{fileNamePrefix, archiveFileExtension}, "")
+				expectedBinaryFiles := []string{}
-				archiveFilePath := filepath.Join(binPath, archiveFileName)
+				for _, componentFile := range getComponentFileMap()[componentName] {
-				assertArchiveFile(t, archiveFileExtension, archiveFilePath, getComponentFileMap()[componentName])
+					expectedBinaryFiles = append(expectedBinaryFiles, strings.Join([]string{componentFile, binaryFileSuffix}, ""))
+				}
+				archiveFile := strings.Join([]string{componentPrefix, archiveFileExtension}, "")
+				assertArchiveFile(t, filepath.Join(binPath, archiveFile), expectedBinaryFiles)
 
-				// Check checksums file.
+				// Check checksums.
-				checksumsFileName := strings.Join([]string{fileNamePrefix, "_checksums.txt"}, "")
+				bomFile := strings.Join([]string{componentPrefix, "_bom", archiveFileExtension}, "")
-				checksumsFilePath := filepath.Join(binPath, checksumsFileName)
+				expectedChecksumsFiles := []string{archiveFile, bomFile}
-				assertChecksumsFile(t, archiveFileExtension, checksumsFilePath, getComponentFileMap()[componentName])
+				for _, componentFile := range getComponentFileMap()[componentName] {
+					expectedChecksumsFiles = append(expectedChecksumsFiles, strings.Join([]string{componentFile, binaryFileSuffix}, ""))
+				}
+				checksumsFile := strings.Join([]string{componentPrefix, "_checksums.txt"}, "")
+				assertChecksumsFile(t, filepath.Join(binPath, checksumsFile), expectedChecksumsFiles)
+
+				// Check SBOMs.
+				expectedSbomFiles := []string{}
+				for _, componentFile := range getComponentFileMap()[componentName] {
+					sbomFilePrefix := strings.Join([]string{componentFile, osName, archName, gitTag}, "-")
+					sbomDirectory := strings.Join([]string{componentPrefix, "_bom"}, "")
+					expectedSbomFiles = append(expectedSbomFiles, filepath.Join(sbomDirectory, strings.Join([]string{sbomFilePrefix, "_bom.json"}, "")))
+				}
+				sbomFile := strings.Join([]string{componentPrefix, "_bom", archiveFileExtension}, "")
+				assertArchiveFile(t, filepath.Join(binPath, sbomFile), expectedSbomFiles)
 			}
 		}
 	}