github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-03-21 15:45:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/logstorage: work-in-progress

Browse source
This commit is contained in:
Aliaksandr Valialkin 2024-06-11 17:50:32 +02:00
parent 65a97317e4
commit 8f5dc966f6
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
25 changed files with 136 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/vlselect/logsql/logsql.go
View file

@ -326,8 +326,8 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
} }
q.AddPipeLimit(uint64(limit)) q.AddPipeLimit(uint64(limit))
q.Optimize()
} }
q.Optimize()
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) { writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
if len(columns) == 0 || len(columns[0].Values) == 0 { if len(columns) == 0 || len(columns[0].Values) == 0 {

8
app/vlselect/vmui/asset-manifest.json
View file

@ -1,13 +1,13 @@
{ {
"files": { "files": {
"main.css": "./static/css/main.7e9644cd.css", "main.css": "./static/css/main.2fa7c03f.css",
"main.js": "./static/js/main.bfdd2931.js", "main.js": "./static/js/main.68f1bd69.js",
"static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js", "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
"static/media/MetricsQL.md": "./static/media/MetricsQL.cb83d071da309a358bc0.md", "static/media/MetricsQL.md": "./static/media/MetricsQL.cb83d071da309a358bc0.md",
"index.html": "./index.html" "index.html": "./index.html"
}, },
"entrypoints": [ "entrypoints": [
"static/css/main.7e9644cd.css", "static/css/main.2fa7c03f.css",
"static/js/main.bfdd2931.js" "static/js/main.68f1bd69.js"
] ]
} }

2
app/vlselect/vmui/index.html
View file

@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.bfdd2931.js"></script><link href="./static/css/main.7e9644cd.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html> <!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.68f1bd69.js"></script><link href="./static/css/main.2fa7c03f.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

1
app/vlselect/vmui/static/css/main.2fa7c03f.css Normal file
View file

File diff suppressed because one or more lines are too long

1
app/vlselect/vmui/static/css/main.7e9644cd.css
View file

File diff suppressed because one or more lines are too long

2
app/vlselect/vmui/static/js/main.68f1bd69.js Normal file
View file

File diff suppressed because one or more lines are too long

0
app/vlselect/vmui/static/js/main.bfdd2931.js.LICENSE.txt → app/vlselect/vmui/static/js/main.68f1bd69.js.LICENSE.txt
View file

2
app/vlselect/vmui/static/js/main.bfdd2931.js
View file

File diff suppressed because one or more lines are too long

2
deployment/docker/docker-compose-victorialogs.yml
View file

@ -42,7 +42,7 @@ services:
# storing logs and serving read queries. # storing logs and serving read queries.
victorialogs: victorialogs:
container_name: victorialogs container_name: victorialogs
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
command: command:
- "--storageDataPath=/vlogs" - "--storageDataPath=/vlogs"
- "--httpListenAddr=:9428" - "--httpListenAddr=:9428"

2
deployment/docker/victorialogs/filebeat-docker/docker-compose.yml
View file

@ -22,7 +22,7 @@ services:
- -beat.uri=http://filebeat-victorialogs:5066 - -beat.uri=http://filebeat-victorialogs:5066
victorialogs: victorialogs:
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
volumes: volumes:
- victorialogs-filebeat-docker-vl:/vlogs - victorialogs-filebeat-docker-vl:/vlogs
ports: ports:

2
deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml
View file

@ -13,7 +13,7 @@ services:
- "5140:5140" - "5140:5140"
victorialogs: victorialogs:
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
volumes: volumes:
- victorialogs-filebeat-syslog-vl:/vlogs - victorialogs-filebeat-syslog-vl:/vlogs
ports: ports:

2
deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml
View file

@ -11,7 +11,7 @@ services:
- "5140:5140" - "5140:5140"
victorialogs: victorialogs:
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
volumes: volumes:
- victorialogs-fluentbit-vl:/vlogs - victorialogs-fluentbit-vl:/vlogs
ports: ports:

2
deployment/docker/victorialogs/logstash/docker-compose.yml
View file

@ -14,7 +14,7 @@ services:
- "5140:5140" - "5140:5140"
victorialogs: victorialogs:
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
volumes: volumes:
- victorialogs-logstash-vl:/vlogs - victorialogs-logstash-vl:/vlogs
ports: ports:

2
deployment/docker/victorialogs/promtail/docker-compose.yml
View file

@ -12,7 +12,7 @@ services:
- "5140:5140" - "5140:5140"
vlogs: vlogs:
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
volumes: volumes:
- victorialogs-promtail-docker:/vlogs - victorialogs-promtail-docker:/vlogs
ports: ports:

2
deployment/docker/victorialogs/vector-docker/docker-compose.yml
View file

@ -22,7 +22,7 @@ services:
condition: service_healthy condition: service_healthy
victorialogs: victorialogs:
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
volumes: volumes:
- victorialogs-vector-docker-vl:/vlogs - victorialogs-vector-docker-vl:/vlogs
ports: ports:

2
deployment/logs-benchmark/docker-compose.yml
View file

@ -3,7 +3,7 @@ version: '3'
services: services:
# Run `make package-victoria-logs` to build victoria-logs image # Run `make package-victoria-logs` to build victoria-logs image
vlogs: vlogs:
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
volumes: volumes:
- vlogs:/vlogs - vlogs:/vlogs
ports: ports:

7
docs/VictoriaLogs/CHANGELOG.md
View file

@ -19,11 +19,18 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
## tip ## tip
## [v0.19.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.19.0-victorialogs)
Released at 2024-06-11
* FEATURE: do not allow starting the [filter](https://docs.victoriametrics.com/victorialogs/logsql/#filters) with [pipe names](https://docs.victoriametrics.com/victorialogs/logsql/#pipes) and [stats function names](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe-functions). This prevents from unexpected results returned by incorrect queries, which miss mandatory [filter](https://docs.victoriametrics.com/victorialogs/logsql/#query-syntax).
* FEATURE: treat unexpected syslog message as [RFC3164](https://datatracker.ietf.org/doc/html/rfc3164) containing only the `message` field when using [`unpack_syslog` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe). * FEATURE: treat unexpected syslog message as [RFC3164](https://datatracker.ietf.org/doc/html/rfc3164) containing only the `message` field when using [`unpack_syslog` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe).
* FEATURE: allow using `where` prefix instead of `filter` prefix in [`filter` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#filter-pipe). * FEATURE: allow using `where` prefix instead of `filter` prefix in [`filter` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#filter-pipe).
* FEATURE: disallow unescaped `!` char in [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) queries, since it permits writing incorrect query, which may look like correct one. For example, `foo!:bar` instead of `foo:!bar`. * FEATURE: disallow unescaped `!` char in [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) queries, since it permits writing incorrect query, which may look like correct one. For example, `foo!:bar` instead of `foo:!bar`.
* FEATURE: [web UI](https://docs.victoriametrics.com/VictoriaLogs/querying/#web-ui): add markdown support to the `Group` view. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6292). * FEATURE: [web UI](https://docs.victoriametrics.com/VictoriaLogs/querying/#web-ui): add markdown support to the `Group` view. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6292).
* BUGFIX: return back the improved performance for queries with `*` filters (aka `SELECT *`). This has been broken in [v0.16.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.16.0-victorialogs).
## [v0.18.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.18.0-victorialogs) ## [v0.18.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.18.0-victorialogs)
Released at 2024-06-06 Released at 2024-06-06

6
docs/VictoriaLogs/QuickStart.md
View file

@ -36,8 +36,8 @@ Just download archive for the needed Operating system and architecture, unpack i
For example, the following commands download VictoriaLogs archive for Linux/amd64, unpack and run it: For example, the following commands download VictoriaLogs archive for Linux/amd64, unpack and run it:
```sh ```sh
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.18.0-victorialogs/victoria-logs-linux-amd64-v0.18.0-victorialogs.tar.gz curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.19.0-victorialogs/victoria-logs-linux-amd64-v0.19.0-victorialogs.tar.gz
tar xzf victoria-logs-linux-amd64-v0.18.0-victorialogs.tar.gz tar xzf victoria-logs-linux-amd64-v0.19.0-victorialogs.tar.gz
./victoria-logs-prod ./victoria-logs-prod
``` ```
@ -61,7 +61,7 @@ Here is the command to run VictoriaLogs in a Docker container:
```sh ```sh
docker run --rm -it -p 9428:9428 -v ./victoria-logs-data:/victoria-logs-data \ docker run --rm -it -p 9428:9428 -v ./victoria-logs-data:/victoria-logs-data \
docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
``` ```
See also: See also:

18
lib/bytesutil/internstring.go
View file

@ -26,6 +26,7 @@ type internStringMap struct {
readonly atomic.Pointer[map[string]internStringMapEntry] readonly atomic.Pointer[map[string]internStringMapEntry]
cleanupInterval uint64
nextCleanupTime atomic.Uint64 nextCleanupTime atomic.Uint64
} }
@ -35,13 +36,14 @@ type internStringMapEntry struct {
} }
func newInternStringMap() *internStringMap { func newInternStringMap() *internStringMap {
ism := &internStringMap{ m := &internStringMap{
mutable: make(map[string]string), mutable: make(map[string]string),
} }
readonly := make(map[string]internStringMapEntry) readonly := make(map[string]internStringMapEntry)
ism.readonly.Store(&readonly) m.readonly.Store(&readonly)
ism.nextCleanupTime.Store(fasttime.UnixTimestamp() + 61) m.cleanupInterval = uint64(cacheExpireDuration.Seconds() / 3)
return ism m.nextCleanupTime.Store(fasttime.UnixTimestamp() + m.cleanupInterval)
return m
} }
func (m *internStringMap) getReadonly() map[string]internStringMapEntry { func (m *internStringMap) getReadonly() map[string]internStringMapEntry {
@ -49,12 +51,12 @@ func (m *internStringMap) getReadonly() map[string]internStringMapEntry {
} }
func (m *internStringMap) intern(s string) string { func (m *internStringMap) intern(s string) string {
if *disableCache || len(s) > *internStringMaxLen { if isSkipCache(s) {
return strings.Clone(s) return strings.Clone(s)
} }
currentTime := fasttime.UnixTimestamp() currentTime := fasttime.UnixTimestamp()
if currentTime >= m.nextCleanupTime.Load() { if currentTime >= m.nextCleanupTime.Load() {
m.nextCleanupTime.Store(currentTime + 61) m.nextCleanupTime.Store(currentTime + m.cleanupInterval)
m.cleanup() m.cleanup()
} }
@ -65,11 +67,11 @@ func (m *internStringMap) intern(s string) string {
return e.s return e.s
} }
// Slower path - search for the string in mutable map // Slower path - search for the string in mutable map under the lock.
m.mu.Lock() m.mu.Lock()
sInterned, ok := m.mutable[s] sInterned, ok := m.mutable[s]
if !ok { if !ok {
// Verify whether the s has been already registered by concurrent goroutines in m.readonly // Verify whether s has been already registered by concurrent goroutines in m.readonly
readonly = m.getReadonly() readonly = m.getReadonly()
e, ok = readonly[s] e, ok = readonly[s]
if !ok { if !ok {

17
lib/logstorage/block_result.go
View file

@ -486,6 +486,18 @@ func (br *blockResult) addTimeColumn() {
} }
func (br *blockResult) addStreamColumn(bs *blockSearch) bool { func (br *blockResult) addStreamColumn(bs *blockSearch) bool {
if !bs.prevStreamID.equal(&bs.bsw.bh.streamID) {
return br.addStreamColumnSlow(bs)
}
if len(bs.prevStream) == 0 {
return false
}
br.addConstColumn("_stream", bytesutil.ToUnsafeString(bs.prevStream))
return true
}
func (br *blockResult) addStreamColumnSlow(bs *blockSearch) bool {
bb := bbPool.Get() bb := bbPool.Get()
defer bbPool.Put(bb) defer bbPool.Put(bb)
@ -496,6 +508,8 @@ func (br *blockResult) addStreamColumn(bs *blockSearch) bool {
// was recently registered and its tags aren't visible to search yet. // was recently registered and its tags aren't visible to search yet.
// The stream tags must become visible in a few seconds. // The stream tags must become visible in a few seconds.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6042 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6042
bs.prevStreamID = *streamID
bs.prevStream = bs.prevStream[:0]
return false return false
} }
@ -506,6 +520,9 @@ func (br *blockResult) addStreamColumn(bs *blockSearch) bool {
s := bytesutil.ToUnsafeString(bb.B) s := bytesutil.ToUnsafeString(bb.B)
br.addConstColumn("_stream", s) br.addConstColumn("_stream", s)
bs.prevStreamID = *streamID
bs.prevStream = append(bs.prevStream[:0], s...)
return true return true
} }

5
lib/logstorage/block_search.go
View file

@ -114,6 +114,11 @@ type blockSearch struct {
// a is used for storing unmarshaled data in csh // a is used for storing unmarshaled data in csh
a arena a arena
// prevStreamID and prevStream are used for speeding up fetching _stream columns
// across sequential blocks belonging to the same stream.
prevStreamID streamID
prevStream []byte
} }
func (bs *blockSearch) reset() { func (bs *blockSearch) reset() {

11
lib/logstorage/parser.go
View file

@ -514,6 +514,14 @@ func (q *Query) getNeededColumns() ([]string, []string) {
// ParseQuery parses s. // ParseQuery parses s.
func ParseQuery(s string) (*Query, error) { func ParseQuery(s string) (*Query, error) {
lex := newLexer(s) lex := newLexer(s)
// Verify the first token doesn't match pipe names.
firstToken := strings.ToLower(lex.rawToken)
if _, ok := pipeNames[firstToken]; ok {
return nil, fmt.Errorf("the query [%s] cannot start with pipe - it must start with madatory filter; see https://docs.victoriametrics.com/victorialogs/logsql/#query-syntax; "+
"if the filter isn't missing, then please put the first word of the filter into quotes: %q", s, firstToken)
}
q, err := parseQuery(lex) q, err := parseQuery(lex)
if err != nil { if err != nil {
return nil, err return nil, err
@ -1842,6 +1850,9 @@ func needQuoteToken(s string) bool {
if _, ok := reservedKeywords[sLower]; ok { if _, ok := reservedKeywords[sLower]; ok {
return true return true
} }
if _, ok := pipeNames[sLower]; ok {
return true
}
for _, r := range s { for _, r := range s {
if !isTokenRune(r) && r != '.' && r != '-' { if !isTokenRune(r) && r != '.' && r != '-' {
return true return true

7
lib/logstorage/parser_test.go
View file

@ -676,6 +676,8 @@ func TestParseQuerySuccess(t *testing.T) {
f(`foo or bar baz or xyz`, `foo or bar baz or xyz`) f(`foo or bar baz or xyz`, `foo or bar baz or xyz`)
f(`(foo or bar) (baz or xyz)`, `(foo or bar) (baz or xyz)`) f(`(foo or bar) (baz or xyz)`, `(foo or bar) (baz or xyz)`)
f(`(foo OR bar) AND baz`, `(foo or bar) baz`) f(`(foo OR bar) AND baz`, `(foo or bar) baz`)
f(`'stats' foo`, `"stats" foo`)
f(`"filter" bar copy fields avg baz`, `"filter" bar "copy" "fields" "avg" baz`)
// parens // parens
f(`foo:(bar baz or not :xxx)`, `foo:bar foo:baz or !foo:xxx`) f(`foo:(bar baz or not :xxx)`, `foo:bar foo:baz or !foo:xxx`)
@ -1214,6 +1216,11 @@ func TestParseQueryFailure(t *testing.T) {
f("not (abc") f("not (abc")
f("!") f("!")
// pipe names without quoutes
f(`filter foo:bar`)
f(`stats count()`)
f(`count()`)
// invalid parens // invalid parens
f("(") f("(")
f("foo (bar ") f("foo (bar ")

41
lib/logstorage/pipe.go
View file

@ -264,3 +264,44 @@ func parsePipe(lex *lexer) (pipe, error) {
return nil, fmt.Errorf("unexpected pipe %q", lex.token) return nil, fmt.Errorf("unexpected pipe %q", lex.token)
} }
} }
var pipeNames = func() map[string]struct{} {
a := []string{
"copy", "cp",
"delete", "del", "rm", "drop",
"drop_empty_fields",
"extract",
"extract_regexp",
"field_names",
"field_values",
"fields", "keep",
"filter", "where",
"format",
"limit", "head",
"math", "eval",
"offset", "skip",
"pack_json",
"pack_logmft",
"rename", "mv",
"replace",
"replace_regexp",
"sort",
"stats",
"uniq",
"unpack_json",
"unpack_logfmt",
"unpack_syslog",
"unroll",
}
m := make(map[string]struct{}, len(a))
for _, s := range a {
m[s] = struct{}{}
}
// add stats names here, since they can be used without the initial `stats` keyword
for _, s := range statsNames {
m[s] = struct{}{}
}
return m
}()

18
lib/logstorage/pipe_stats.go
View file

@ -702,6 +702,24 @@ func parseStatsFunc(lex *lexer) (statsFunc, error) {
} }
} }
var statsNames = []string{
"avg",
"count",
"count_empty",
"count_uniq",
"max",
"median",
"min",
"quantile",
"row_any",
"row_max",
"row_min",
"sum",
"sum_len",
"uniq_values",
"values",
}
var zeroByStatsField = &byStatsField{} var zeroByStatsField = &byStatsField{}
// byStatsField represents 'by (...)' part of the pipeStats. // byStatsField represents 'by (...)' part of the pipeStats.