github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vlinsert: properly parse timestamps with nanosecond precision at /insert/jsonline HTTP endpoint

Browse source 
This has been broken in 2b6a634ec0
This commit is contained in:
Aliaksandr Valialkin 2024-06-18 00:23:17 +02:00
parent 6f6fd9cb19
commit 3eda4617c0
No known key found for this signature in database
GPG key ID: 52C003EE2BCDB9EB
13 changed files with 22 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
app/vlinsert/insertutils/timestamp.go
View file

@ -7,19 +7,19 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
)
// ExtractTimestampISO8601FromFields extracts ISO8601 timestamp in nanoseconds from the field with the name timeField at fields.
// ExtractTimestampRFC3339NanoFromFields extracts RFC3339 timestamp in nanoseconds from the field with the name timeField at fields.
//
// The value for the timeField is set to empty string after returning from the function,
// so it could be ignored during data ingestion.
//
// The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
func ExtractTimestampISO8601FromFields(timeField string, fields []logstorage.Field) (int64, error) {
func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error) {
for i := range fields {
f := &fields[i]
if f.Name != timeField {
continue
}
nsecs, ok := logstorage.TryParseTimestampISO8601(f.Value)
nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(f.Value)
if !ok {
if f.Value == "0" || f.Value == "" {
return time.Now().UnixNano(), nil

2
app/vlinsert/jsonline/jsonline.go
View file

@ -112,7 +112,7 @@ func readLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.Log
if err := p.ParseLogMessage(line); err != nil {
return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
}
ts, err := insertutils.ExtractTimestampISO8601FromFields(timeField, p.Fields)
ts, err := insertutils.ExtractTimestampRFC3339NanoFromFields(timeField, p.Fields)
if err != nil {
return false, fmt.Errorf("cannot get timestamp: %w", err)
}

2
app/vlinsert/syslog/syslog.go
View file

@ -485,7 +485,7 @@ func processLine(line []byte, currentYear int, timezone *time.Location, lmp inse
p := logstorage.GetSyslogParser(currentYear, timezone)
lineStr := bytesutil.ToUnsafeString(line)
p.Parse(lineStr)
ts, err := insertutils.ExtractTimestampISO8601FromFields("timestamp", p.Fields)
ts, err := insertutils.ExtractTimestampRFC3339NanoFromFields("timestamp", p.Fields)
if err != nil {
return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
}

2
deployment/docker/docker-compose-victorialogs.yml
View file

@ -42,7 +42,7 @@ services:
# storing logs and serving read queries.
victorialogs:
container_name: victorialogs
image: docker.io/victoriametrics/victoria-logs:v0.20.1-victorialogs
image: docker.io/victoriametrics/victoria-logs:v0.20.2-victorialogs
command:
- "--storageDataPath=/vlogs"
- "--httpListenAddr=:9428"

2
deployment/docker/victorialogs/filebeat-docker/docker-compose.yml
View file

@ -22,7 +22,7 @@ services:
- -beat.uri=http://filebeat-victorialogs:5066
victorialogs:
image: docker.io/victoriametrics/victoria-logs:v0.20.1-victorialogs
image: docker.io/victoriametrics/victoria-logs:v0.20.2-victorialogs
volumes:
- victorialogs-filebeat-docker-vl:/vlogs
ports:

2
deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml
View file

@ -13,7 +13,7 @@ services:
- "5140:5140"
victorialogs:
image: docker.io/victoriametrics/victoria-logs:v0.20.1-victorialogs
image: docker.io/victoriametrics/victoria-logs:v0.20.2-victorialogs
volumes:
- victorialogs-filebeat-syslog-vl:/vlogs
ports:

2
deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml
View file

@ -11,7 +11,7 @@ services:
- "5140:5140"
victorialogs:
image: docker.io/victoriametrics/victoria-logs:v0.20.1-victorialogs
image: docker.io/victoriametrics/victoria-logs:v0.20.2-victorialogs
volumes:
- victorialogs-fluentbit-vl:/vlogs
ports:

2
deployment/docker/victorialogs/logstash/docker-compose.yml
View file

@ -14,7 +14,7 @@ services:
- "5140:5140"
victorialogs:
image: docker.io/victoriametrics/victoria-logs:v0.20.1-victorialogs
image: docker.io/victoriametrics/victoria-logs:v0.20.2-victorialogs
volumes:
- victorialogs-logstash-vl:/vlogs
ports:

2
deployment/docker/victorialogs/promtail/docker-compose.yml
View file

@ -12,7 +12,7 @@ services:
- "5140:5140"
vlogs:
image: docker.io/victoriametrics/victoria-logs:v0.20.1-victorialogs
image: docker.io/victoriametrics/victoria-logs:v0.20.2-victorialogs
volumes:
- victorialogs-promtail-docker:/vlogs
ports:

2
deployment/docker/victorialogs/vector-docker/docker-compose.yml
View file

@ -22,7 +22,7 @@ services:
condition: service_healthy
victorialogs:
image: docker.io/victoriametrics/victoria-logs:v0.20.1-victorialogs
image: docker.io/victoriametrics/victoria-logs:v0.20.2-victorialogs
volumes:
- victorialogs-vector-docker-vl:/vlogs
ports:

2
deployment/logs-benchmark/docker-compose.yml
View file

@ -3,7 +3,7 @@ version: '3'
services:
# Run `make package-victoria-logs` to build victoria-logs image
vlogs:
image: docker.io/victoriametrics/victoria-logs:v0.20.1-victorialogs
image: docker.io/victoriametrics/victoria-logs:v0.20.2-victorialogs
volumes:
- vlogs:/vlogs
ports:

6
docs/VictoriaLogs/CHANGELOG.md
View file

@ -19,6 +19,12 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
## tip
## [v0.20.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.20.2-victorialogs)
Released at 2024-06-19
* BUGFIX: properly parse timestamps with nanosecond precision for logs ingested via [jsonline format](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api). The bug has been introduced in [v0.20.0 release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.20.0-victorialogs).
## [v0.20.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.20.1-victorialogs)
Released at 2024-06-18

6
docs/VictoriaLogs/QuickStart.md
View file

@ -36,8 +36,8 @@ Just download archive for the needed Operating system and architecture, unpack i
For example, the following commands download VictoriaLogs archive for Linux/amd64, unpack and run it:
```sh
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.20.1-victorialogs/victoria-logs-linux-amd64-v0.20.1-victorialogs.tar.gz
tar xzf victoria-logs-linux-amd64-v0.20.1-victorialogs.tar.gz
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.20.2-victorialogs/victoria-logs-linux-amd64-v0.20.2-victorialogs.tar.gz
tar xzf victoria-logs-linux-amd64-v0.20.2-victorialogs.tar.gz
./victoria-logs-prod
```
@ -61,7 +61,7 @@ Here is the command to run VictoriaLogs in a Docker container:
```sh
docker run --rm -it -p 9428:9428 -v ./victoria-logs-data:/victoria-logs-data \
docker.io/victoriametrics/victoria-logs:v0.20.1-victorialogs
docker.io/victoriametrics/victoria-logs:v0.20.2-victorialogs
```
See also: