Implement SBOM generation

Signed-off-by: Arkadii Yakovets <ark@victoriametrics.com>
This commit is contained in:
Arkadii Yakovets 2024-07-16 17:28:24 -07:00
parent 042444f3ea
commit 9b10e6d26b
No known key found for this signature in database
GPG key ID: 7AEAA43DE7C15E41
2 changed files with 130 additions and 128 deletions

171
Makefile
View file

@ -14,6 +14,10 @@ endif
GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
VICTORIA_LOGS_COMPONENTS = victoria-logs
VICTORIA_METRICS_COMPONENTS = victoria-metrics
VICTORIA_METRICS_UTILS_COMPONENTS = vmagent vmalert vmalert-tool vmauth vmbackup vmrestore vmctl
.PHONY: $(MAKECMDGOALS)
include app/*/Makefile
@ -23,6 +27,73 @@ include deployment/*/Makefile
include dashboards/Makefile
include package/release/Makefile
define RELEASE_GOOS_GOARCH
$(eval PKG_NAME := $(1))
$(eval PKG_COMPONENTS := $(2))
# Build
$(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
# Generate SBOM
mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom"
$(foreach pkg_component, $(PKG_COMPONENTS),
cyclonedx-gomod app -assert-licenses -json -licenses -packages \
-main app/$(pkg_component) \
-output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json)
# Pack and compress
$(foreach pkg_component, $(PKG_COMPONENTS),
cd bin && tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar \
$(pkg_component)-$(GOOS)-$(GOARCH)-prod)
cd bin && gzip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar
cd bin && tar -czf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
# Generate checksums
cd bin && \
sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \
sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
$(foreach pkg_component, $(PKG_COMPONENTS),
cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod | sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ >> \
$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt)
# Clean up
cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
$(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
endef
define RELEASE_WINDOWS_GOARCH
$(eval PKG_NAME := $(1))
$(eval PKG_COMPONENTS := $(2))
# Build
$(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
# Generate SBOM
mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom"
$(foreach pkg_component, $(PKG_COMPONENTS),
cyclonedx-gomod app -assert-licenses -json -licenses -packages \
-main app/$(pkg_component) \
-output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json)
# Pack and compress
$(foreach pkg_component, $(PKG_COMPONENTS),
cd bin && zip -u $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip \
$(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe)
cd bin && zip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip -r $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
# Generate checksums
cd bin && \
sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \
sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
$(foreach pkg_component, $(PKG_COMPONENTS),
cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt)
# Clean up
cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
$(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe)
endef
all: \
victoria-metrics-prod \
victoria-logs-prod \
@ -241,26 +312,13 @@ release-victoria-metrics-openbsd-amd64:
GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
release-victoria-metrics-windows-amd64:
GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
GOOS=windows GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
cd bin && \
tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
victoria-metrics-$(GOOS)-$(GOARCH)-prod \
&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
victoria-metrics-$(GOOS)-$(GOARCH)-prod \
| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod
release-victoria-metrics-goos-goarch:
$(call RELEASE_GOOS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS))
release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
cd bin && \
zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
victoria-metrics-windows-$(GOARCH)-prod.exe \
&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
victoria-metrics-windows-$(GOARCH)-prod.exe \
> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
cd bin && rm -rf \
victoria-metrics-windows-$(GOARCH)-prod.exe
release-victoria-metrics-windows-goarch:
$(call RELEASE_WINDOWS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS))
release-victoria-logs:
$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
@ -355,77 +413,13 @@ release-vmutils-openbsd-amd64:
GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
release-vmutils-windows-amd64:
GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
GOOS=windows GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
release-vmutils-goos-goarch: \
vmagent-$(GOOS)-$(GOARCH)-prod \
vmalert-$(GOOS)-$(GOARCH)-prod \
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
vmauth-$(GOOS)-$(GOARCH)-prod \
vmbackup-$(GOOS)-$(GOARCH)-prod \
vmrestore-$(GOOS)-$(GOARCH)-prod \
vmctl-$(GOOS)-$(GOARCH)-prod
cd bin && \
tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
vmagent-$(GOOS)-$(GOARCH)-prod \
vmalert-$(GOOS)-$(GOARCH)-prod \
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
vmauth-$(GOOS)-$(GOARCH)-prod \
vmbackup-$(GOOS)-$(GOARCH)-prod \
vmrestore-$(GOOS)-$(GOARCH)-prod \
vmctl-$(GOOS)-$(GOARCH)-prod \
&& sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
vmagent-$(GOOS)-$(GOARCH)-prod \
vmalert-$(GOOS)-$(GOARCH)-prod \
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
vmauth-$(GOOS)-$(GOARCH)-prod \
vmbackup-$(GOOS)-$(GOARCH)-prod \
vmrestore-$(GOOS)-$(GOARCH)-prod \
vmctl-$(GOOS)-$(GOARCH)-prod \
| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
cd bin && rm -rf \
vmagent-$(GOOS)-$(GOARCH)-prod \
vmalert-$(GOOS)-$(GOARCH)-prod \
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
vmauth-$(GOOS)-$(GOARCH)-prod \
vmbackup-$(GOOS)-$(GOARCH)-prod \
vmrestore-$(GOOS)-$(GOARCH)-prod \
vmctl-$(GOOS)-$(GOARCH)-prod
release-vmutils-goos-goarch:
$(call RELEASE_GOOS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS))
release-vmutils-windows-goarch: \
vmagent-windows-$(GOARCH)-prod \
vmalert-windows-$(GOARCH)-prod \
vmalert-tool-windows-$(GOARCH)-prod \
vmauth-windows-$(GOARCH)-prod \
vmbackup-windows-$(GOARCH)-prod \
vmrestore-windows-$(GOARCH)-prod \
vmctl-windows-$(GOARCH)-prod
cd bin && \
zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
vmagent-windows-$(GOARCH)-prod.exe \
vmalert-windows-$(GOARCH)-prod.exe \
vmalert-tool-windows-$(GOARCH)-prod.exe \
vmauth-windows-$(GOARCH)-prod.exe \
vmbackup-windows-$(GOARCH)-prod.exe \
vmrestore-windows-$(GOARCH)-prod.exe \
vmctl-windows-$(GOARCH)-prod.exe \
&& sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
vmagent-windows-$(GOARCH)-prod.exe \
vmalert-windows-$(GOARCH)-prod.exe \
vmalert-tool-windows-$(GOARCH)-prod.exe \
vmauth-windows-$(GOARCH)-prod.exe \
vmbackup-windows-$(GOARCH)-prod.exe \
vmrestore-windows-$(GOARCH)-prod.exe \
vmctl-windows-$(GOARCH)-prod.exe \
> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
cd bin && rm -rf \
vmagent-windows-$(GOARCH)-prod.exe \
vmalert-windows-$(GOARCH)-prod.exe \
vmalert-tool-windows-$(GOARCH)-prod.exe \
vmauth-windows-$(GOARCH)-prod.exe \
vmbackup-windows-$(GOARCH)-prod.exe \
vmrestore-windows-$(GOARCH)-prod.exe \
vmctl-windows-$(GOARCH)-prod.exe
release-vmutils-windows-goarch:
$(call RELEASE_WINDOWS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS))
pprof-cpu:
go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
@ -514,6 +508,9 @@ install-wwhrd:
check-licenses: install-wwhrd
wwhrd check -f .wwhrd.yml
cyclonedx-gomod-install:
which cyclonedx-gomod || go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@latest
copy-docs:
# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n

View file

@ -12,6 +12,7 @@ import (
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"testing"
)
@ -21,7 +22,7 @@ const unixExecSuffix = "-prod"
const windowsExecSuffix = "-prod.exe"
const zipExt = ".zip"
func assertArchiveFile(t *testing.T, extension string, path string, expectedPrefixes []string) {
func assertArchiveFile(t *testing.T, path string, expectedFiles []string) {
// Check if file exists
archiveFileInfo, err := getFileInfo(path)
if err != nil {
@ -31,10 +32,7 @@ func assertArchiveFile(t *testing.T, extension string, path string, expectedPref
}
var archiveFiles []string
var binaryFileSuffix string
if extension == tarGzExt { // Unix-like stuff
binaryFileSuffix = unixExecSuffix
if strings.HasSuffix(path, tarGzExt) { // Unix-like stuff
// Get file handler
tarGzFile, err := os.Open(path)
if err != nil {
@ -59,15 +57,15 @@ func assertArchiveFile(t *testing.T, extension string, path string, expectedPref
if err != nil {
t.Fatalf("Failed to read tar header: %s", err)
}
if header.Size == 0 {
t.Fatalf("Archive file is empty: %s (%s)", header.Name, path)
if header.FileInfo().IsDir() {
continue
} else if header.Size == 0 {
t.Fatalf("Archived file is empty: %s (%s)", header.Name, path)
}
archiveFiles = append(archiveFiles, header.Name)
}
} else if extension == zipExt { // Windows stuff
binaryFileSuffix = "-windows-amd64" + windowsExecSuffix
} else if strings.HasSuffix(path, zipExt) { // Windows stuff
// Get file handler
zipFile, err := os.Open(path)
if err != nil {
@ -87,26 +85,24 @@ func assertArchiveFile(t *testing.T, extension string, path string, expectedPref
}
for _, file := range zipReader.File {
if file.CompressedSize64 == 0 {
t.Fatalf("Archive file is empty: %s (%s)", file.Name, path)
if file.FileInfo().IsDir() {
continue
} else if file.CompressedSize64 == 0 {
t.Fatalf("Archived file is empty: %s (%s)", file.Name, path)
}
archiveFiles = append(archiveFiles, file.Name)
}
} else { // Unexpected stuff.
t.Fatalf("Unknown archive type: %s", extension)
t.Fatalf("Unknown archive type: %s", path)
}
var expectedFiles []string
for _, expectedFilePrefix := range expectedPrefixes {
expectedFiles = append(expectedFiles, strings.Join([]string{expectedFilePrefix, binaryFileSuffix}, ""))
}
if !compareSlices(archiveFiles, expectedFiles) {
t.Fatalf("Archive contents `%s` doesn't match the expected one: `%s`", archiveFiles, expectedFiles)
}
}
func assertChecksumsFile(t *testing.T, extension string, path string, expectedPrefixes []string) {
func assertChecksumsFile(t *testing.T, path string, expectedFiles []string) {
// Check if file exists
checksumsFileInfo, err := getFileInfo(path)
if err != nil {
@ -130,20 +126,6 @@ func assertChecksumsFile(t *testing.T, extension string, path string, expectedPr
t.Fatalf("Failed to read file: %s", err)
}
var binaryFileSuffix string
if extension == tarGzExt { // Unix-like stuff
binaryFileSuffix = unixExecSuffix
} else if extension == zipExt { // Windows stuff
binaryFileSuffix = "-windows-amd64" + windowsExecSuffix
} else { // Unexpected stuff.
t.Fatalf("Unknown archive type: %s", extension)
}
archiveFileName := strings.ReplaceAll(filepath.Base(path), "_checksums.txt", extension)
expectedFiles := []string{archiveFileName}
for _, expectedFilePrefix := range expectedPrefixes {
expectedFiles = append(expectedFiles, strings.Join([]string{expectedFilePrefix, binaryFileSuffix}, ""))
}
if !compareSlices(checksumsFiles, expectedFiles) {
t.Fatalf("Archive contents `%s` doesn't match the expected one: `%s`", checksumsFiles, expectedFiles)
}
@ -153,6 +135,9 @@ func compareSlices(slice1, slice2 []string) bool {
if len(slice1) != len(slice2) {
return false
}
sort.Strings(slice1)
sort.Strings(slice2)
for i := range slice1 {
if slice1[i] != slice2[i] {
return false
@ -239,24 +224,44 @@ func testReleaseAssets(t *testing.T, componentNames []string) {
for _, componentName := range componentNames {
for osName, archNames := range getArchOsMap() {
var archiveFileExtension string
var binaryFileSuffix string
if osName == "windows" {
archiveFileExtension = zipExt
binaryFileSuffix = "-windows-amd64" + windowsExecSuffix
} else {
archiveFileExtension = tarGzExt
binaryFileSuffix = unixExecSuffix
}
for _, archName := range archNames {
fileNamePrefix := strings.Join([]string{componentName, osName, archName, gitTag}, "-")
componentPrefix := strings.Join([]string{componentName, osName, archName, gitTag}, "-")
// Check archive file.
archiveFileName := strings.Join([]string{fileNamePrefix, archiveFileExtension}, "")
archiveFilePath := filepath.Join(binPath, archiveFileName)
assertArchiveFile(t, archiveFileExtension, archiveFilePath, getComponentFileMap()[componentName])
// Check binaries.
expectedBinaryFiles := []string{}
for _, componentFile := range getComponentFileMap()[componentName] {
expectedBinaryFiles = append(expectedBinaryFiles, strings.Join([]string{componentFile, binaryFileSuffix}, ""))
}
archiveFile := strings.Join([]string{componentPrefix, archiveFileExtension}, "")
assertArchiveFile(t, filepath.Join(binPath, archiveFile), expectedBinaryFiles)
// Check checksums file.
checksumsFileName := strings.Join([]string{fileNamePrefix, "_checksums.txt"}, "")
checksumsFilePath := filepath.Join(binPath, checksumsFileName)
assertChecksumsFile(t, archiveFileExtension, checksumsFilePath, getComponentFileMap()[componentName])
// Check checksums.
bomFile := strings.Join([]string{componentPrefix, "_bom", archiveFileExtension}, "")
expectedChecksumsFiles := []string{archiveFile, bomFile}
for _, componentFile := range getComponentFileMap()[componentName] {
expectedChecksumsFiles = append(expectedChecksumsFiles, strings.Join([]string{componentFile, binaryFileSuffix}, ""))
}
checksumsFile := strings.Join([]string{componentPrefix, "_checksums.txt"}, "")
assertChecksumsFile(t, filepath.Join(binPath, checksumsFile), expectedChecksumsFiles)
// Check SBOMs.
expectedSbomFiles := []string{}
for _, componentFile := range getComponentFileMap()[componentName] {
sbomFilePrefix := strings.Join([]string{componentFile, osName, archName, gitTag}, "-")
sbomDirectory := strings.Join([]string{componentPrefix, "_bom"}, "")
expectedSbomFiles = append(expectedSbomFiles, filepath.Join(sbomDirectory, strings.Join([]string{sbomFilePrefix, "_bom.json"}, "")))
}
sbomFile := strings.Join([]string{componentPrefix, "_bom", archiveFileExtension}, "")
assertArchiveFile(t, filepath.Join(binPath, sbomFile), expectedSbomFiles)
}
}
}