From 54f035d4ce617bcff38917e2e8bacd5e7d869534 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Tue, 23 Jul 2019 19:54:50 +0300
Subject: [PATCH] all: small updates after PR #114
---
Makefile | 22 ++++++----
app/vminsert/Makefile | 6 +++
app/vmselect/Makefile | 6 +++
app/vmstorage/Makefile | 6 +++
deployment/docker/Makefile | 3 +-
lib/encoding/zstd/stream_cgo.go | 25 ++++++++++++
lib/encoding/zstd/stream_pure.go | 70 ++++++++++++++++++++++++++++++++
lib/handshake/buffered_conn.go | 10 ++---
8 files changed, 134 insertions(+), 14 deletions(-)
create mode 100644 lib/encoding/zstd/stream_cgo.go
create mode 100644 lib/encoding/zstd/stream_pure.go
diff --git a/Makefile b/Makefile
index 3518c47c9e..d3573589ee 100644
--- a/Makefile
+++ b/Makefile
@@ -15,6 +15,11 @@ all: \
vmselect \
vmstorage
+all-pure: \
+ vminsert-pure \
+ vmselect-pure \
+ vmstorage-pure
+
include app/*/Makefile
include deployment/*/Makefile
include deployment/*/helm/Makefile
@@ -59,15 +64,13 @@ install-errcheck:
check_all: fmt vet lint errcheck golangci-lint
test:
- GO111MODULE=on go test -mod=vendor ./lib/...
- GO111MODULE=on go test -mod=vendor ./app/...
-
-test_full:
- GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+ GO111MODULE=on go test -mod=vendor ./lib/... ./app/...
test-pure:
- GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/...
- GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./app/...
+ GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/...
+
+test-full:
+ GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
benchmark:
GO111MODULE=on go test -mod=vendor -bench=. ./lib/...
@@ -84,7 +87,10 @@ vendor-update:
GO111MODULE=on go mod vendor
app-local:
- GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+ CGO_ENABLED=1 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+
+app-local-pure:
+ CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
quicktemplate-gen: install-qtc
qtc
diff --git a/app/vminsert/Makefile b/app/vminsert/Makefile
index 5f836d9144..8a2ebc0ffa 100644
--- a/app/vminsert/Makefile
+++ b/app/vminsert/Makefile
@@ -17,6 +17,12 @@ vminsert-prod:
vminsert-prod-race:
APP_NAME=vminsert RACE=-race $(MAKE) app-via-docker
+vminsert-pure:
+ APP_NAME=vminsert $(MAKE) app-local-pure
+
+vminsert-pure-prod:
+ APP_NAME=vminsert APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker
+
package-vminsert:
APP_NAME=vminsert $(MAKE) package-via-docker
diff --git a/app/vmselect/Makefile b/app/vmselect/Makefile
index 8822a69f7c..41de873a70 100644
--- a/app/vmselect/Makefile
+++ b/app/vmselect/Makefile
@@ -19,6 +19,12 @@ vmselect-prod:
vmselect-prod-race:
APP_NAME=vmselect RACE=-race $(MAKE) app-via-docker
+vmselect-pure:
+ APP_NAME=vmselect $(MAKE) app-local-pure
+
+vmselect-pure-prod:
+ APP_NAME=vmselect APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker
+
package-vmselect:
APP_NAME=vmselect $(MAKE) package-via-docker
diff --git a/app/vmstorage/Makefile b/app/vmstorage/Makefile
index 670e2420f2..94f23aca39 100644
--- a/app/vmstorage/Makefile
+++ b/app/vmstorage/Makefile
@@ -19,6 +19,12 @@ vmstorage-prod:
vmstorage-prod-race:
APP_NAME=vmstorage RACE=-race $(MAKE) app-via-docker
+vmstorage-pure:
+ APP_NAME=vmstorage $(MAKE) app-local-pure
+
+vmstorage-pure-prod:
+ APP_NAME=vmstorage APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker
+
package-vmstorage:
APP_NAME=vmstorage $(MAKE) package-via-docker
diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile
index 587f8145f4..e476772911 100644
--- a/deployment/docker/Makefile
+++ b/deployment/docker/Makefile
@@ -21,8 +21,9 @@ app-via-docker: package-certs package-builder
--mount type=bind,src="$(shell pwd)/gocache-for-docker",dst=/gocache \
--env GOCACHE=/gocache \
--env GO111MODULE=on \
+ $(DOCKER_OPTS) \
$(BUILDER_IMAGE) \
- go build $(RACE) -mod=vendor -ldflags "-s -w -extldflags '-static' $(GO_BUILDINFO)" -tags 'netgo osusergo' -o bin/$(APP_NAME)-prod $(PKG_PREFIX)/app/$(APP_NAME)
+ go build $(RACE) -mod=vendor -ldflags "-s -w -extldflags '-static' $(GO_BUILDINFO)" -tags 'netgo osusergo' -o bin/$(APP_NAME)$(APP_SUFFIX)-prod $(PKG_PREFIX)/app/$(APP_NAME)
package-via-docker:
(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)') || (\
diff --git a/lib/encoding/zstd/stream_cgo.go b/lib/encoding/zstd/stream_cgo.go
new file mode 100644
index 0000000000..ca6e4f4474
--- /dev/null
+++ b/lib/encoding/zstd/stream_cgo.go
@@ -0,0 +1,25 @@
+// +build cgo
+
+package zstd
+
+import (
+ "io"
+
+ "github.com/valyala/gozstd"
+)
+
+// Reader is zstd reader
+type Reader = gozstd.Reader
+
+// NewReader returns zstd reader for the given r.
+func NewReader(r io.Reader) *Reader {
+ return gozstd.NewReader(r)
+}
+
+// Writer is zstd writer
+type Writer = gozstd.Writer
+
+// NewWriterLevel returns zstd writer for the given w and level.
+func NewWriterLevel(w io.Writer, level int) *Writer {
+ return gozstd.NewWriterLevel(w, level)
+}
diff --git a/lib/encoding/zstd/stream_pure.go b/lib/encoding/zstd/stream_pure.go
new file mode 100644
index 0000000000..50afb7ccf4
--- /dev/null
+++ b/lib/encoding/zstd/stream_pure.go
@@ -0,0 +1,70 @@
+// +build !cgo
+
+package zstd
+
+import (
+ "io"
+
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+ "github.com/klauspost/compress/zstd"
+)
+
+// Reader is zstd reader
+type Reader struct {
+ d *zstd.Decoder
+}
+
+// NewReader returns zstd reader for the given r.
+func NewReader(r io.Reader) *Reader {
+ d, err := zstd.NewReader(r)
+ if err != nil {
+ logger.Panicf("BUG: failed to create ZSTD reader: %s", err)
+ }
+ return &Reader{
+ d: d,
+ }
+}
+
+// Read reads up to len(p) bytes to p from r.
+func (r *Reader) Read(p []byte) (int, error) {
+ return r.d.Read(p)
+}
+
+// Release releases r.
+func (r *Reader) Release() {
+ r.d.Reset(nil)
+ r.d = nil
+}
+
+// Writer is zstd writer
+type Writer struct {
+ e *zstd.Encoder
+}
+
+// NewWriterLevel returns zstd writer for the given w and level.
+func NewWriterLevel(w io.Writer, level int) *Writer {
+ l := zstd.EncoderLevelFromZstd(level)
+ e, err := zstd.NewWriter(w, zstd.WithEncoderLevel(l))
+ if err != nil {
+ logger.Panicf("BUG: failed to create ZSTD writer: %s", err)
+ }
+ return &Writer{
+ e: e,
+ }
+}
+
+// Write writes p to w.
+func (w *Writer) Write(p []byte) (int, error) {
+ return w.e.Write(p)
+}
+
+// Flush flushes all the pending data from w to the underlying writer.
+func (w *Writer) Flush() error {
+ return w.e.Flush()
+}
+
+// Release releases w.
+func (w *Writer) Release() {
+ w.e.Reset(nil)
+ w.e = nil
+}
diff --git a/lib/handshake/buffered_conn.go b/lib/handshake/buffered_conn.go
index 774adf0366..50fbafb4b4 100644
--- a/lib/handshake/buffered_conn.go
+++ b/lib/handshake/buffered_conn.go
@@ -5,7 +5,7 @@ import (
"io"
"net"
- "github.com/valyala/gozstd"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
)
type bufferedWriter interface {
@@ -31,12 +31,12 @@ func newBufferedConn(c net.Conn, compressionLevel int, isReadCompressed bool) *B
if compressionLevel <= 0 {
bc.bw = bufio.NewWriterSize(c, bufferSize)
} else {
- bc.bw = gozstd.NewWriterLevel(c, compressionLevel)
+ bc.bw = zstd.NewWriterLevel(c, compressionLevel)
}
if !isReadCompressed {
bc.br = bufio.NewReaderSize(c, bufferSize)
} else {
- bc.br = gozstd.NewReader(c)
+ bc.br = zstd.NewReader(c)
}
return bc
}
@@ -60,12 +60,12 @@ func (bc *BufferedConn) Close() error {
err := bc.Conn.Close()
bc.Conn = nil
- if zr, ok := bc.br.(*gozstd.Reader); ok {
+ if zr, ok := bc.br.(*zstd.Reader); ok {
zr.Release()
}
bc.br = nil
- if zw, ok := bc.bw.(*gozstd.Writer); ok {
+ if zw, ok := bc.bw.(*zstd.Writer); ok {
// Do not call zw.Close(), since we already closed the underlying conn.
zw.Release()
}