mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
lib/logstorage: work-in-progress
This commit is contained in:
Aliaksandr Valialkin
parent
65a97317e4
commit
8f5dc966f6
25 changed files with 136 additions and 28 deletions
|
|
@ -326,8 +326,8 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
|
|||
}
|
||||
|
||||
q.AddPipeLimit(uint64(limit))
|
||||
q.Optimize()
|
||||
}
|
||||
q.Optimize()
|
||||
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
if len(columns) == 0 || len(columns[0].Values) == 0 {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
{
|
||||
"files": {
|
||||
"main.css": "./static/css/main.7e9644cd.css",
|
||||
"main.js": "./static/js/main.bfdd2931.js",
|
||||
"main.css": "./static/css/main.2fa7c03f.css",
|
||||
"main.js": "./static/js/main.68f1bd69.js",
|
||||
"static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
|
||||
"static/media/MetricsQL.md": "./static/media/MetricsQL.cb83d071da309a358bc0.md",
|
||||
"index.html": "./index.html"
|
||||
},
|
||||
"entrypoints": [
|
||||
"static/css/main.7e9644cd.css",
|
||||
"static/js/main.bfdd2931.js"
|
||||
"static/css/main.2fa7c03f.css",
|
||||
"static/js/main.68f1bd69.js"
|
||||
]
|
||||
}
|
||||
|
|
@ -1 +1 @@
|
|||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.bfdd2931.js"></script><link href="./static/css/main.7e9644cd.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.68f1bd69.js"></script><link href="./static/css/main.2fa7c03f.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
1
app/vlselect/vmui/static/css/main.2fa7c03f.css
Normal file
1
app/vlselect/vmui/static/css/main.2fa7c03f.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
2
app/vlselect/vmui/static/js/main.68f1bd69.js
Normal file
2
app/vlselect/vmui/static/js/main.68f1bd69.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -42,7 +42,7 @@ services:
|
|||
# storing logs and serving read queries.
|
||||
victorialogs:
|
||||
container_name: victorialogs
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
|
||||
command:
|
||||
- "--storageDataPath=/vlogs"
|
||||
- "--httpListenAddr=:9428"
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ services:
|
|||
- -beat.uri=http://filebeat-victorialogs:5066
|
||||
|
||||
victorialogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
|
||||
volumes:
|
||||
- victorialogs-filebeat-docker-vl:/vlogs
|
||||
ports:
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ services:
|
|||
- "5140:5140"
|
||||
|
||||
victorialogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
|
||||
volumes:
|
||||
- victorialogs-filebeat-syslog-vl:/vlogs
|
||||
ports:
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ services:
|
|||
- "5140:5140"
|
||||
|
||||
victorialogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
|
||||
volumes:
|
||||
- victorialogs-fluentbit-vl:/vlogs
|
||||
ports:
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ services:
|
|||
- "5140:5140"
|
||||
|
||||
victorialogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
|
||||
volumes:
|
||||
- victorialogs-logstash-vl:/vlogs
|
||||
ports:
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ services:
|
|||
- "5140:5140"
|
||||
|
||||
vlogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
|
||||
volumes:
|
||||
- victorialogs-promtail-docker:/vlogs
|
||||
ports:
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ services:
|
|||
condition: service_healthy
|
||||
|
||||
victorialogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
|
||||
volumes:
|
||||
- victorialogs-vector-docker-vl:/vlogs
|
||||
ports:
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ version: '3'
|
|||
services:
|
||||
# Run `make package-victoria-logs` to build victoria-logs image
|
||||
vlogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs
|
||||
image: docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
|
||||
volumes:
|
||||
- vlogs:/vlogs
|
||||
ports:
|
||||
|
|
|
|||
|
|
@ -19,11 +19,18 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
|
|||
|
||||
## tip
|
||||
|
||||
## [v0.19.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.19.0-victorialogs)
|
||||
|
||||
Released at 2024-06-11
|
||||
|
||||
* FEATURE: do not allow starting the [filter](https://docs.victoriametrics.com/victorialogs/logsql/#filters) with [pipe names](https://docs.victoriametrics.com/victorialogs/logsql/#pipes) and [stats function names](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe-functions). This prevents from unexpected results returned by incorrect queries, which miss mandatory [filter](https://docs.victoriametrics.com/victorialogs/logsql/#query-syntax).
|
||||
* FEATURE: treat unexpected syslog message as [RFC3164](https://datatracker.ietf.org/doc/html/rfc3164) containing only the `message` field when using [`unpack_syslog` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe).
|
||||
* FEATURE: allow using `where` prefix instead of `filter` prefix in [`filter` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#filter-pipe).
|
||||
* FEATURE: disallow unescaped `!` char in [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) queries, since it permits writing incorrect query, which may look like correct one. For example, `foo!:bar` instead of `foo:!bar`.
|
||||
* FEATURE: [web UI](https://docs.victoriametrics.com/VictoriaLogs/querying/#web-ui): add markdown support to the `Group` view. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6292).
|
||||
|
||||
* BUGFIX: return back the improved performance for queries with `*` filters (aka `SELECT *`). This has been broken in [v0.16.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.16.0-victorialogs).
|
||||
|
||||
## [v0.18.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.18.0-victorialogs)
|
||||
|
||||
Released at 2024-06-06
|
||||
|
|
|
|||
|
|
@ -36,8 +36,8 @@ Just download archive for the needed Operating system and architecture, unpack i
|
|||
For example, the following commands download VictoriaLogs archive for Linux/amd64, unpack and run it:
|
||||
|
||||
```sh
|
||||
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.18.0-victorialogs/victoria-logs-linux-amd64-v0.18.0-victorialogs.tar.gz
|
||||
tar xzf victoria-logs-linux-amd64-v0.18.0-victorialogs.tar.gz
|
||||
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.19.0-victorialogs/victoria-logs-linux-amd64-v0.19.0-victorialogs.tar.gz
|
||||
tar xzf victoria-logs-linux-amd64-v0.19.0-victorialogs.tar.gz
|
||||
./victoria-logs-prod
|
||||
```
|
||||
|
||||
|
|
@ -61,7 +61,7 @@ Here is the command to run VictoriaLogs in a Docker container:
|
|||
|
||||
```sh
|
||||
docker run --rm -it -p 9428:9428 -v ./victoria-logs-data:/victoria-logs-data \
|
||||
docker.io/victoriametrics/victoria-logs:v0.18.0-victorialogs
|
||||
docker.io/victoriametrics/victoria-logs:v0.19.0-victorialogs
|
||||
```
|
||||
|
||||
See also:
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ type internStringMap struct {
|
|||
|
||||
readonly atomic.Pointer[map[string]internStringMapEntry]
|
||||
|
||||
cleanupInterval uint64
|
||||
nextCleanupTime atomic.Uint64
|
||||
}
|
||||
|
||||
|
|
@ -35,13 +36,14 @@ type internStringMapEntry struct {
|
|||
}
|
||||
|
||||
func newInternStringMap() *internStringMap {
|
||||
ism := &internStringMap{
|
||||
m := &internStringMap{
|
||||
mutable: make(map[string]string),
|
||||
}
|
||||
readonly := make(map[string]internStringMapEntry)
|
||||
ism.readonly.Store(&readonly)
|
||||
ism.nextCleanupTime.Store(fasttime.UnixTimestamp() + 61)
|
||||
return ism
|
||||
m.readonly.Store(&readonly)
|
||||
m.cleanupInterval = uint64(cacheExpireDuration.Seconds() / 3)
|
||||
m.nextCleanupTime.Store(fasttime.UnixTimestamp() + m.cleanupInterval)
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *internStringMap) getReadonly() map[string]internStringMapEntry {
|
||||
|
|
@ -49,12 +51,12 @@ func (m *internStringMap) getReadonly() map[string]internStringMapEntry {
|
|||
}
|
||||
|
||||
func (m *internStringMap) intern(s string) string {
|
||||
if *disableCache || len(s) > *internStringMaxLen {
|
||||
if isSkipCache(s) {
|
||||
return strings.Clone(s)
|
||||
}
|
||||
currentTime := fasttime.UnixTimestamp()
|
||||
if currentTime >= m.nextCleanupTime.Load() {
|
||||
m.nextCleanupTime.Store(currentTime + 61)
|
||||
m.nextCleanupTime.Store(currentTime + m.cleanupInterval)
|
||||
m.cleanup()
|
||||
}
|
||||
|
||||
|
|
@ -65,11 +67,11 @@ func (m *internStringMap) intern(s string) string {
|
|||
return e.s
|
||||
}
|
||||
|
||||
// Slower path - search for the string in mutable map
|
||||
// Slower path - search for the string in mutable map under the lock.
|
||||
m.mu.Lock()
|
||||
sInterned, ok := m.mutable[s]
|
||||
if !ok {
|
||||
// Verify whether the s has been already registered by concurrent goroutines in m.readonly
|
||||
// Verify whether s has been already registered by concurrent goroutines in m.readonly
|
||||
readonly = m.getReadonly()
|
||||
e, ok = readonly[s]
|
||||
if !ok {
|
||||
|
|
|
|||
|
|
@ -486,6 +486,18 @@ func (br *blockResult) addTimeColumn() {
|
|||
}
|
||||
|
||||
func (br *blockResult) addStreamColumn(bs *blockSearch) bool {
|
||||
if !bs.prevStreamID.equal(&bs.bsw.bh.streamID) {
|
||||
return br.addStreamColumnSlow(bs)
|
||||
}
|
||||
|
||||
if len(bs.prevStream) == 0 {
|
||||
return false
|
||||
}
|
||||
br.addConstColumn("_stream", bytesutil.ToUnsafeString(bs.prevStream))
|
||||
return true
|
||||
}
|
||||
|
||||
func (br *blockResult) addStreamColumnSlow(bs *blockSearch) bool {
|
||||
bb := bbPool.Get()
|
||||
defer bbPool.Put(bb)
|
||||
|
||||
|
|
@ -496,6 +508,8 @@ func (br *blockResult) addStreamColumn(bs *blockSearch) bool {
|
|||
// was recently registered and its tags aren't visible to search yet.
|
||||
// The stream tags must become visible in a few seconds.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6042
|
||||
bs.prevStreamID = *streamID
|
||||
bs.prevStream = bs.prevStream[:0]
|
||||
return false
|
||||
}
|
||||
|
||||
|
|
@ -506,6 +520,9 @@ func (br *blockResult) addStreamColumn(bs *blockSearch) bool {
|
|||
|
||||
s := bytesutil.ToUnsafeString(bb.B)
|
||||
br.addConstColumn("_stream", s)
|
||||
|
||||
bs.prevStreamID = *streamID
|
||||
bs.prevStream = append(bs.prevStream[:0], s...)
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -114,6 +114,11 @@ type blockSearch struct {
|
|||
|
||||
// a is used for storing unmarshaled data in csh
|
||||
a arena
|
||||
|
||||
// prevStreamID and prevStream are used for speeding up fetching _stream columns
|
||||
// across sequential blocks belonging to the same stream.
|
||||
prevStreamID streamID
|
||||
prevStream []byte
|
||||
}
|
||||
|
||||
func (bs *blockSearch) reset() {
|
||||
|
|
|
|||
|
|
@ -514,6 +514,14 @@ func (q *Query) getNeededColumns() ([]string, []string) {
|
|||
// ParseQuery parses s.
|
||||
func ParseQuery(s string) (*Query, error) {
|
||||
lex := newLexer(s)
|
||||
|
||||
// Verify the first token doesn't match pipe names.
|
||||
firstToken := strings.ToLower(lex.rawToken)
|
||||
if _, ok := pipeNames[firstToken]; ok {
|
||||
return nil, fmt.Errorf("the query [%s] cannot start with pipe - it must start with madatory filter; see https://docs.victoriametrics.com/victorialogs/logsql/#query-syntax; "+
|
||||
"if the filter isn't missing, then please put the first word of the filter into quotes: %q", s, firstToken)
|
||||
}
|
||||
|
||||
q, err := parseQuery(lex)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -1842,6 +1850,9 @@ func needQuoteToken(s string) bool {
|
|||
if _, ok := reservedKeywords[sLower]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := pipeNames[sLower]; ok {
|
||||
return true
|
||||
}
|
||||
for _, r := range s {
|
||||
if !isTokenRune(r) && r != '.' && r != '-' {
|
||||
return true
|
||||
|
|
|
|||
|
|
@ -676,6 +676,8 @@ func TestParseQuerySuccess(t *testing.T) {
|
|||
f(`foo or bar baz or xyz`, `foo or bar baz or xyz`)
|
||||
f(`(foo or bar) (baz or xyz)`, `(foo or bar) (baz or xyz)`)
|
||||
f(`(foo OR bar) AND baz`, `(foo or bar) baz`)
|
||||
f(`'stats' foo`, `"stats" foo`)
|
||||
f(`"filter" bar copy fields avg baz`, `"filter" bar "copy" "fields" "avg" baz`)
|
||||
|
||||
// parens
|
||||
f(`foo:(bar baz or not :xxx)`, `foo:bar foo:baz or !foo:xxx`)
|
||||
|
|
@ -1214,6 +1216,11 @@ func TestParseQueryFailure(t *testing.T) {
|
|||
f("not (abc")
|
||||
f("!")
|
||||
|
||||
// pipe names without quoutes
|
||||
f(`filter foo:bar`)
|
||||
f(`stats count()`)
|
||||
f(`count()`)
|
||||
|
||||
// invalid parens
|
||||
f("(")
|
||||
f("foo (bar ")
|
||||
|
|
|
|||
|
|
@ -264,3 +264,44 @@ func parsePipe(lex *lexer) (pipe, error) {
|
|||
return nil, fmt.Errorf("unexpected pipe %q", lex.token)
|
||||
}
|
||||
}
|
||||
|
||||
var pipeNames = func() map[string]struct{} {
|
||||
a := []string{
|
||||
"copy", "cp",
|
||||
"delete", "del", "rm", "drop",
|
||||
"drop_empty_fields",
|
||||
"extract",
|
||||
"extract_regexp",
|
||||
"field_names",
|
||||
"field_values",
|
||||
"fields", "keep",
|
||||
"filter", "where",
|
||||
"format",
|
||||
"limit", "head",
|
||||
"math", "eval",
|
||||
"offset", "skip",
|
||||
"pack_json",
|
||||
"pack_logmft",
|
||||
"rename", "mv",
|
||||
"replace",
|
||||
"replace_regexp",
|
||||
"sort",
|
||||
"stats",
|
||||
"uniq",
|
||||
"unpack_json",
|
||||
"unpack_logfmt",
|
||||
"unpack_syslog",
|
||||
"unroll",
|
||||
}
|
||||
|
||||
m := make(map[string]struct{}, len(a))
|
||||
for _, s := range a {
|
||||
m[s] = struct{}{}
|
||||
}
|
||||
|
||||
// add stats names here, since they can be used without the initial `stats` keyword
|
||||
for _, s := range statsNames {
|
||||
m[s] = struct{}{}
|
||||
}
|
||||
return m
|
||||
}()
|
||||
|
|
|
|||
|
|
@ -702,6 +702,24 @@ func parseStatsFunc(lex *lexer) (statsFunc, error) {
|
|||
}
|
||||
}
|
||||
|
||||
var statsNames = []string{
|
||||
"avg",
|
||||
"count",
|
||||
"count_empty",
|
||||
"count_uniq",
|
||||
"max",
|
||||
"median",
|
||||
"min",
|
||||
"quantile",
|
||||
"row_any",
|
||||
"row_max",
|
||||
"row_min",
|
||||
"sum",
|
||||
"sum_len",
|
||||
"uniq_values",
|
||||
"values",
|
||||
}
|
||||
|
||||
var zeroByStatsField = &byStatsField{}
|
||||
|
||||
// byStatsField represents 'by (...)' part of the pipeStats.
|
||||
|
|
|
|||
Loading…
Reference in a new issue