From 9b10e6d26b446b4c0b1b63fb475dbc71b0c6c866 Mon Sep 17 00:00:00 2001 From: Arkadii Yakovets Date: Tue, 16 Jul 2024 17:28:24 -0700 Subject: [PATCH] Implement SBOM generation Signed-off-by: Arkadii Yakovets --- Makefile | 171 +++++++++++++++++----------------- package/release/asset_test.go | 87 +++++++++-------- 2 files changed, 130 insertions(+), 128 deletions(-) diff --git a/Makefile b/Makefile index 5afc5f2d9..ec42eb387 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,10 @@ endif GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)' +VICTORIA_LOGS_COMPONENTS = victoria-logs +VICTORIA_METRICS_COMPONENTS = victoria-metrics +VICTORIA_METRICS_UTILS_COMPONENTS = vmagent vmalert vmalert-tool vmauth vmbackup vmrestore vmctl + .PHONY: $(MAKECMDGOALS) include app/*/Makefile @@ -23,6 +27,73 @@ include deployment/*/Makefile include dashboards/Makefile include package/release/Makefile +define RELEASE_GOOS_GOARCH + $(eval PKG_NAME := $(1)) + $(eval PKG_COMPONENTS := $(2)) + + # Build + $(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod) + + # Generate SBOM + mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom" + $(foreach pkg_component, $(PKG_COMPONENTS), + cyclonedx-gomod app -assert-licenses -json -licenses -packages \ + -main app/$(pkg_component) \ + -output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json) + + # Pack and compress + $(foreach pkg_component, $(PKG_COMPONENTS), + cd bin && tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar \ + $(pkg_component)-$(GOOS)-$(GOARCH)-prod) + cd bin && gzip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar + cd bin && tar -czf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom + + # Generate checksums + cd bin && \ + sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \ + sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt + $(foreach pkg_component, $(PKG_COMPONENTS), + cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod | sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ >> \ + $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt) + + # Clean up + cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom + $(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod) +endef + +define RELEASE_WINDOWS_GOARCH + $(eval PKG_NAME := $(1)) + $(eval PKG_COMPONENTS := $(2)) + + # Build + $(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod) + + # Generate SBOM + mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom" + $(foreach pkg_component, $(PKG_COMPONENTS), + cyclonedx-gomod app -assert-licenses -json -licenses -packages \ + -main app/$(pkg_component) \ + -output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json) + + # Pack and compress + $(foreach pkg_component, $(PKG_COMPONENTS), + cd bin && zip -u $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip \ + $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe) + cd bin && zip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip -r $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom + + # Generate checksums + cd bin && \ + sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \ + sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt + $(foreach pkg_component, $(PKG_COMPONENTS), + cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt) + + # Clean up + cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom + $(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe) +endef + + all: \ victoria-metrics-prod \ victoria-logs-prod \ @@ -241,26 +312,13 @@ release-victoria-metrics-openbsd-amd64: GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch release-victoria-metrics-windows-amd64: - GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch + GOOS=windows GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch -release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod - cd bin && \ - tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \ - victoria-metrics-$(GOOS)-$(GOARCH)-prod \ - && sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \ - victoria-metrics-$(GOOS)-$(GOARCH)-prod \ - | sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt - cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod +release-victoria-metrics-goos-goarch: + $(call RELEASE_GOOS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS)) -release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod - cd bin && \ - zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \ - victoria-metrics-windows-$(GOARCH)-prod.exe \ - && sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \ - victoria-metrics-windows-$(GOARCH)-prod.exe \ - > victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt - cd bin && rm -rf \ - victoria-metrics-windows-$(GOARCH)-prod.exe +release-victoria-metrics-windows-goarch: + $(call RELEASE_WINDOWS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS)) release-victoria-logs: $(MAKE_PARALLEL) release-victoria-logs-linux-386 \ @@ -355,77 +413,13 @@ release-vmutils-openbsd-amd64: GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch release-vmutils-windows-amd64: - GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch + GOOS=windows GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch -release-vmutils-goos-goarch: \ - vmagent-$(GOOS)-$(GOARCH)-prod \ - vmalert-$(GOOS)-$(GOARCH)-prod \ - vmalert-tool-$(GOOS)-$(GOARCH)-prod \ - vmauth-$(GOOS)-$(GOARCH)-prod \ - vmbackup-$(GOOS)-$(GOARCH)-prod \ - vmrestore-$(GOOS)-$(GOARCH)-prod \ - vmctl-$(GOOS)-$(GOARCH)-prod - cd bin && \ - tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \ - vmagent-$(GOOS)-$(GOARCH)-prod \ - vmalert-$(GOOS)-$(GOARCH)-prod \ - vmalert-tool-$(GOOS)-$(GOARCH)-prod \ - vmauth-$(GOOS)-$(GOARCH)-prod \ - vmbackup-$(GOOS)-$(GOARCH)-prod \ - vmrestore-$(GOOS)-$(GOARCH)-prod \ - vmctl-$(GOOS)-$(GOARCH)-prod \ - && sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \ - vmagent-$(GOOS)-$(GOARCH)-prod \ - vmalert-$(GOOS)-$(GOARCH)-prod \ - vmalert-tool-$(GOOS)-$(GOARCH)-prod \ - vmauth-$(GOOS)-$(GOARCH)-prod \ - vmbackup-$(GOOS)-$(GOARCH)-prod \ - vmrestore-$(GOOS)-$(GOARCH)-prod \ - vmctl-$(GOOS)-$(GOARCH)-prod \ - | sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt - cd bin && rm -rf \ - vmagent-$(GOOS)-$(GOARCH)-prod \ - vmalert-$(GOOS)-$(GOARCH)-prod \ - vmalert-tool-$(GOOS)-$(GOARCH)-prod \ - vmauth-$(GOOS)-$(GOARCH)-prod \ - vmbackup-$(GOOS)-$(GOARCH)-prod \ - vmrestore-$(GOOS)-$(GOARCH)-prod \ - vmctl-$(GOOS)-$(GOARCH)-prod +release-vmutils-goos-goarch: + $(call RELEASE_GOOS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS)) -release-vmutils-windows-goarch: \ - vmagent-windows-$(GOARCH)-prod \ - vmalert-windows-$(GOARCH)-prod \ - vmalert-tool-windows-$(GOARCH)-prod \ - vmauth-windows-$(GOARCH)-prod \ - vmbackup-windows-$(GOARCH)-prod \ - vmrestore-windows-$(GOARCH)-prod \ - vmctl-windows-$(GOARCH)-prod - cd bin && \ - zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \ - vmagent-windows-$(GOARCH)-prod.exe \ - vmalert-windows-$(GOARCH)-prod.exe \ - vmalert-tool-windows-$(GOARCH)-prod.exe \ - vmauth-windows-$(GOARCH)-prod.exe \ - vmbackup-windows-$(GOARCH)-prod.exe \ - vmrestore-windows-$(GOARCH)-prod.exe \ - vmctl-windows-$(GOARCH)-prod.exe \ - && sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \ - vmagent-windows-$(GOARCH)-prod.exe \ - vmalert-windows-$(GOARCH)-prod.exe \ - vmalert-tool-windows-$(GOARCH)-prod.exe \ - vmauth-windows-$(GOARCH)-prod.exe \ - vmbackup-windows-$(GOARCH)-prod.exe \ - vmrestore-windows-$(GOARCH)-prod.exe \ - vmctl-windows-$(GOARCH)-prod.exe \ - > vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt - cd bin && rm -rf \ - vmagent-windows-$(GOARCH)-prod.exe \ - vmalert-windows-$(GOARCH)-prod.exe \ - vmalert-tool-windows-$(GOARCH)-prod.exe \ - vmauth-windows-$(GOARCH)-prod.exe \ - vmbackup-windows-$(GOARCH)-prod.exe \ - vmrestore-windows-$(GOARCH)-prod.exe \ - vmctl-windows-$(GOARCH)-prod.exe +release-vmutils-windows-goarch: + $(call RELEASE_WINDOWS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS)) pprof-cpu: go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE) @@ -514,6 +508,9 @@ install-wwhrd: check-licenses: install-wwhrd wwhrd check -f .wwhrd.yml +cyclonedx-gomod-install: + which cyclonedx-gomod || go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@latest + copy-docs: # The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish). # For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n diff --git a/package/release/asset_test.go b/package/release/asset_test.go index d6cd29bcf..68508f9de 100644 --- a/package/release/asset_test.go +++ b/package/release/asset_test.go @@ -12,6 +12,7 @@ import ( "os" "os/exec" "path/filepath" + "sort" "strings" "testing" ) @@ -21,7 +22,7 @@ const unixExecSuffix = "-prod" const windowsExecSuffix = "-prod.exe" const zipExt = ".zip" -func assertArchiveFile(t *testing.T, extension string, path string, expectedPrefixes []string) { +func assertArchiveFile(t *testing.T, path string, expectedFiles []string) { // Check if file exists archiveFileInfo, err := getFileInfo(path) if err != nil { @@ -31,10 +32,7 @@ func assertArchiveFile(t *testing.T, extension string, path string, expectedPref } var archiveFiles []string - var binaryFileSuffix string - if extension == tarGzExt { // Unix-like stuff - binaryFileSuffix = unixExecSuffix - + if strings.HasSuffix(path, tarGzExt) { // Unix-like stuff // Get file handler tarGzFile, err := os.Open(path) if err != nil { @@ -59,15 +57,15 @@ func assertArchiveFile(t *testing.T, extension string, path string, expectedPref if err != nil { t.Fatalf("Failed to read tar header: %s", err) } - if header.Size == 0 { - t.Fatalf("Archive file is empty: %s (%s)", header.Name, path) + if header.FileInfo().IsDir() { + continue + } else if header.Size == 0 { + t.Fatalf("Archived file is empty: %s (%s)", header.Name, path) } archiveFiles = append(archiveFiles, header.Name) } - } else if extension == zipExt { // Windows stuff - binaryFileSuffix = "-windows-amd64" + windowsExecSuffix - + } else if strings.HasSuffix(path, zipExt) { // Windows stuff // Get file handler zipFile, err := os.Open(path) if err != nil { @@ -87,26 +85,24 @@ func assertArchiveFile(t *testing.T, extension string, path string, expectedPref } for _, file := range zipReader.File { - if file.CompressedSize64 == 0 { - t.Fatalf("Archive file is empty: %s (%s)", file.Name, path) + if file.FileInfo().IsDir() { + continue + } else if file.CompressedSize64 == 0 { + t.Fatalf("Archived file is empty: %s (%s)", file.Name, path) } archiveFiles = append(archiveFiles, file.Name) } } else { // Unexpected stuff. - t.Fatalf("Unknown archive type: %s", extension) + t.Fatalf("Unknown archive type: %s", path) } - var expectedFiles []string - for _, expectedFilePrefix := range expectedPrefixes { - expectedFiles = append(expectedFiles, strings.Join([]string{expectedFilePrefix, binaryFileSuffix}, "")) - } if !compareSlices(archiveFiles, expectedFiles) { t.Fatalf("Archive contents `%s` doesn't match the expected one: `%s`", archiveFiles, expectedFiles) } } -func assertChecksumsFile(t *testing.T, extension string, path string, expectedPrefixes []string) { +func assertChecksumsFile(t *testing.T, path string, expectedFiles []string) { // Check if file exists checksumsFileInfo, err := getFileInfo(path) if err != nil { @@ -130,20 +126,6 @@ func assertChecksumsFile(t *testing.T, extension string, path string, expectedPr t.Fatalf("Failed to read file: %s", err) } - var binaryFileSuffix string - if extension == tarGzExt { // Unix-like stuff - binaryFileSuffix = unixExecSuffix - } else if extension == zipExt { // Windows stuff - binaryFileSuffix = "-windows-amd64" + windowsExecSuffix - } else { // Unexpected stuff. - t.Fatalf("Unknown archive type: %s", extension) - } - - archiveFileName := strings.ReplaceAll(filepath.Base(path), "_checksums.txt", extension) - expectedFiles := []string{archiveFileName} - for _, expectedFilePrefix := range expectedPrefixes { - expectedFiles = append(expectedFiles, strings.Join([]string{expectedFilePrefix, binaryFileSuffix}, "")) - } if !compareSlices(checksumsFiles, expectedFiles) { t.Fatalf("Archive contents `%s` doesn't match the expected one: `%s`", checksumsFiles, expectedFiles) } @@ -153,6 +135,9 @@ func compareSlices(slice1, slice2 []string) bool { if len(slice1) != len(slice2) { return false } + + sort.Strings(slice1) + sort.Strings(slice2) for i := range slice1 { if slice1[i] != slice2[i] { return false @@ -239,24 +224,44 @@ func testReleaseAssets(t *testing.T, componentNames []string) { for _, componentName := range componentNames { for osName, archNames := range getArchOsMap() { var archiveFileExtension string + var binaryFileSuffix string if osName == "windows" { archiveFileExtension = zipExt + binaryFileSuffix = "-windows-amd64" + windowsExecSuffix } else { archiveFileExtension = tarGzExt + binaryFileSuffix = unixExecSuffix } for _, archName := range archNames { - fileNamePrefix := strings.Join([]string{componentName, osName, archName, gitTag}, "-") + componentPrefix := strings.Join([]string{componentName, osName, archName, gitTag}, "-") - // Check archive file. - archiveFileName := strings.Join([]string{fileNamePrefix, archiveFileExtension}, "") - archiveFilePath := filepath.Join(binPath, archiveFileName) - assertArchiveFile(t, archiveFileExtension, archiveFilePath, getComponentFileMap()[componentName]) + // Check binaries. + expectedBinaryFiles := []string{} + for _, componentFile := range getComponentFileMap()[componentName] { + expectedBinaryFiles = append(expectedBinaryFiles, strings.Join([]string{componentFile, binaryFileSuffix}, "")) + } + archiveFile := strings.Join([]string{componentPrefix, archiveFileExtension}, "") + assertArchiveFile(t, filepath.Join(binPath, archiveFile), expectedBinaryFiles) - // Check checksums file. - checksumsFileName := strings.Join([]string{fileNamePrefix, "_checksums.txt"}, "") - checksumsFilePath := filepath.Join(binPath, checksumsFileName) - assertChecksumsFile(t, archiveFileExtension, checksumsFilePath, getComponentFileMap()[componentName]) + // Check checksums. + bomFile := strings.Join([]string{componentPrefix, "_bom", archiveFileExtension}, "") + expectedChecksumsFiles := []string{archiveFile, bomFile} + for _, componentFile := range getComponentFileMap()[componentName] { + expectedChecksumsFiles = append(expectedChecksumsFiles, strings.Join([]string{componentFile, binaryFileSuffix}, "")) + } + checksumsFile := strings.Join([]string{componentPrefix, "_checksums.txt"}, "") + assertChecksumsFile(t, filepath.Join(binPath, checksumsFile), expectedChecksumsFiles) + + // Check SBOMs. + expectedSbomFiles := []string{} + for _, componentFile := range getComponentFileMap()[componentName] { + sbomFilePrefix := strings.Join([]string{componentFile, osName, archName, gitTag}, "-") + sbomDirectory := strings.Join([]string{componentPrefix, "_bom"}, "") + expectedSbomFiles = append(expectedSbomFiles, filepath.Join(sbomDirectory, strings.Join([]string{sbomFilePrefix, "_bom.json"}, ""))) + } + sbomFile := strings.Join([]string{componentPrefix, "_bom", archiveFileExtension}, "") + assertArchiveFile(t, filepath.Join(binPath, sbomFile), expectedSbomFiles) } } }