From 7da541360efc656b0c5669a3eeb63311accac6a7 Mon Sep 17 00:00:00 2001 From: Alexander Marshalov <_@marshalov.org> Date: Wed, 22 May 2024 21:46:50 +0200 Subject: [PATCH] [vmlogs] fixed time parsing with millisecond precision time (#6293) (#6295) fix for #6293 Co-authored-by: Aliaksandr Valialkin --- app/victoria-logs/Makefile | 7 +++++++ docs/VictoriaLogs/CHANGELOG.md | 1 + lib/logstorage/parser.go | 8 ++++++-- lib/logstorage/parser_test.go | 15 +++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/app/victoria-logs/Makefile b/app/victoria-logs/Makefile index 6b2170613..7c8054418 100644 --- a/app/victoria-logs/Makefile +++ b/app/victoria-logs/Makefile @@ -101,3 +101,10 @@ victoria-logs-windows-amd64: victoria-logs-pure: APP_NAME=victoria-logs $(MAKE) app-local-pure + +run-victoria-logs: + mkdir -p victoria-logs-data + DOCKER_OPTS='-v $(shell pwd)/victoria-logs-data:/victoria-logs-data' \ + APP_NAME=victoria-logs \ + ARGS='' \ + $(MAKE) run-via-docker diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index 2fcab1a4d..fb7c3d6a9 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -34,6 +34,7 @@ according to [these docs](https://docs.victoriametrics.com/VictoriaLogs/QuickSta * BUGFIX: fix `invalid memory address or nil pointer dereference` panic when using [`extract`](https://docs.victoriametrics.com/victorialogs/logsql/#extract-pipe), [`unpack_json`](https://docs.victoriametrics.com/victorialogs/logsql/#unpack_json-pipe) or [`unpack_logfmt`](https://docs.victoriametrics.com/victorialogs/logsql/#unpack_logfmt-pipe) pipes. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6306). * BUGFIX: [web UI](https://docs.victoriametrics.com/VictoriaLogs/querying/#web-ui): fix an issue where logs with long `_msg` values might not display. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6281). +* BUGFIX: properly handle time range boundaries with millisecond precision. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6293). ## [v0.8.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.8.0-victorialogs) diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index 536b0016e..49b677b68 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -1307,7 +1307,8 @@ func parseFilterTime(lex *lexer) (*filterTime, error) { if err != nil { return nil, fmt.Errorf("cannot parse _time filter: %w", err) } - startTime := int64(t * 1e9) + // Round to milliseconds + startTime := int64(math.Round(t*1e3)) * 1e6 endTime := getMatchingEndTime(startTime, s) ft := &filterTime{ minTimestamp: startTime, @@ -1416,6 +1417,8 @@ func getMatchingEndTime(startTime int64, stringRepr string) int64 { tEnd = tStart.Add(time.Minute) case len(timeStr) == len("YYYY-MM-DDThh:mm:ss") && timeStr[len("YYYY")] == '-': tEnd = tStart.Add(time.Second) + case len(timeStr) == len("YYYY-MM-DDThh:mm:ss.SSS") && timeStr[len("YYYY")] == '-': + tEnd = tStart.Add(time.Millisecond) default: tEnd = tStart.Add(time.Nanosecond) } @@ -1459,7 +1462,8 @@ func parseTime(lex *lexer) (int64, string, error) { if err != nil { return 0, "", err } - return int64(t * 1e9), s, nil + // round to milliseconds + return int64(math.Round(t*1e3)) * 1e6, s, nil } func quoteTokenIfNeeded(s string) string { diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index 6e8c458a6..824133796 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -191,6 +191,21 @@ func TestParseTimeRange(t *testing.T) { f("2023-02-28T23:59:59", minTimestamp, maxTimestamp) f("2023-02-28T23:59:59Z", minTimestamp, maxTimestamp) + // _time:[YYYY-MM-DDTHH:MM:SS.sss, YYYY-MM-DDTHH:MM:SS.sss) + minTimestamp = time.Date(2024, time.May, 12, 0, 0, 0, 333000000, time.UTC).UnixNano() + maxTimestamp = time.Date(2024, time.May, 12, 0, 0, 0, 555000000, time.UTC).UnixNano() - 1 + f("[2024-05-12T00:00:00.333+00:00,2024-05-12T00:00:00.555+00:00)", minTimestamp, maxTimestamp) + + // _time:[YYYY-MM-DDTHH:MM:SS.sss, YYYY-MM-DDTHH:MM:SS.sss] + minTimestamp = time.Date(2024, time.May, 12, 0, 0, 0, 333000000, time.UTC).UnixNano() + maxTimestamp = time.Date(2024, time.May, 12, 0, 0, 0, 556000000, time.UTC).UnixNano() - 1 + f("[2024-05-12T00:00:00.333+00:00,2024-05-12T00:00:00.555+00:00]", minTimestamp, maxTimestamp) + + // _time:YYYY-MM-DDTHH:MM:SS.sss + minTimestamp = time.Date(2024, time.May, 14, 13, 54, 59, 134000000, time.UTC).UnixNano() + maxTimestamp = time.Date(2024, time.May, 14, 13, 54, 59, 135000000, time.UTC).UnixNano() - 1 + f("2024-05-14T13:54:59.134Z", minTimestamp, maxTimestamp) + // _time:YYYY-MM-DDTHH:MM:SS-hh:mm minTimestamp = time.Date(2023, time.February, 28, 23, 59, 59, 0, time.UTC).UnixNano() maxTimestamp = time.Date(2023, time.March, 1, 0, 0, 0, 0, time.UTC).UnixNano() - 1