diff --git a/app/vmselect/main.go b/app/vmselect/main.go index aede06d22..1fb1bac21 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -117,6 +117,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { return true } return true + case "/api/v1/labels/count": + labelsCountRequests.Inc() + httpserver.EnableCORS(w, r) + if err := prometheus.LabelsCountHandler(w, r); err != nil { + labelsCountErrors.Inc() + sendPrometheusError(w, r, err) + return true + } + return true case "/api/v1/export": exportRequests.Inc() if err := prometheus.ExportHandler(w, r); err != nil { @@ -180,6 +189,9 @@ var ( labelsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/labels"}`) labelsErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/labels"}`) + labelsCountRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/labels/count"}`) + labelsCountErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/labels/count"}`) + deleteRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/admin/tsdb/delete_series"}`) deleteErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/admin/tsdb/delete_series"}`) diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go index f0a3cf728..28b1420ce 100644 --- a/app/vmselect/netstorage/netstorage.go +++ b/app/vmselect/netstorage/netstorage.go @@ -400,6 +400,36 @@ func GetLabelValues(labelName string, deadline Deadline) ([]string, error) { return labelValues, nil } +// GetLabelEntries returns all the label entries until the given deadline. +func GetLabelEntries(deadline Deadline) ([]storage.TagEntry, error) { + labelEntries, err := vmstorage.SearchTagEntries(*maxTagKeysPerSearch, *maxTagValuesPerSearch) + if err != nil { + return nil, fmt.Errorf("error during label entries request: %s", err) + } + + // Substitute "" with "__name__" + for i := range labelEntries { + e := &labelEntries[i] + if e.Key == "" { + e.Key = "__name__" + } + } + + // Sort labelEntries by the number of label values in each entry. + sort.Slice(labelEntries, func(i, j int) bool { + a, b := labelEntries[i].Values, labelEntries[j].Values + if len(a) < len(b) { + return true + } + if len(a) > len(b) { + return false + } + return labelEntries[i].Key < labelEntries[j].Key + }) + + return labelEntries, nil +} + // GetSeriesCount returns the number of unique series. func GetSeriesCount(deadline Deadline) (uint64, error) { n, err := vmstorage.GetSeriesCount() diff --git a/app/vmselect/prometheus/labels_count_response.qtpl b/app/vmselect/prometheus/labels_count_response.qtpl new file mode 100644 index 000000000..b96bc843d --- /dev/null +++ b/app/vmselect/prometheus/labels_count_response.qtpl @@ -0,0 +1,17 @@ +{% import "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" %} + +{% stripspace %} +LabelsCountResponse generates response for /api/v1/labels/count . +{% func LabelsCountResponse(labelEntries []storage.TagEntry) %} +{ + "status":"success", + "data":{ + {% for i, e := range labelEntries %} + {%q= e.Key %}:{%d= len(e.Values) %} + {% if i+1 < len(labelEntries) %},{% endif %} + {% endfor %} + } +} +{% endfunc %} + +{% endstripspace %} diff --git a/app/vmselect/prometheus/labels_count_response.qtpl.go b/app/vmselect/prometheus/labels_count_response.qtpl.go new file mode 100644 index 000000000..177a91df1 --- /dev/null +++ b/app/vmselect/prometheus/labels_count_response.qtpl.go @@ -0,0 +1,74 @@ +// Code generated by qtc from "labels_count_response.qtpl". DO NOT EDIT. +// See https://github.com/valyala/quicktemplate for details. + +//line app/vmselect/prometheus/labels_count_response.qtpl:1 +package prometheus + +//line app/vmselect/prometheus/labels_count_response.qtpl:1 +import "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" + +// LabelsCountResponse generates response for /api/v1/labels/count . + +//line app/vmselect/prometheus/labels_count_response.qtpl:5 +import ( + qtio422016 "io" + + qt422016 "github.com/valyala/quicktemplate" +) + +//line app/vmselect/prometheus/labels_count_response.qtpl:5 +var ( + _ = qtio422016.Copy + _ = qt422016.AcquireByteBuffer +) + +//line app/vmselect/prometheus/labels_count_response.qtpl:5 +func StreamLabelsCountResponse(qw422016 *qt422016.Writer, labelEntries []storage.TagEntry) { +//line app/vmselect/prometheus/labels_count_response.qtpl:5 + qw422016.N().S(`{"status":"success","data":{`) +//line app/vmselect/prometheus/labels_count_response.qtpl:9 + for i, e := range labelEntries { +//line app/vmselect/prometheus/labels_count_response.qtpl:10 + qw422016.N().Q(e.Key) +//line app/vmselect/prometheus/labels_count_response.qtpl:10 + qw422016.N().S(`:`) +//line app/vmselect/prometheus/labels_count_response.qtpl:10 + qw422016.N().D(len(e.Values)) +//line app/vmselect/prometheus/labels_count_response.qtpl:11 + if i+1 < len(labelEntries) { +//line app/vmselect/prometheus/labels_count_response.qtpl:11 + qw422016.N().S(`,`) +//line app/vmselect/prometheus/labels_count_response.qtpl:11 + } +//line app/vmselect/prometheus/labels_count_response.qtpl:12 + } +//line app/vmselect/prometheus/labels_count_response.qtpl:12 + qw422016.N().S(`}}`) +//line app/vmselect/prometheus/labels_count_response.qtpl:15 +} + +//line app/vmselect/prometheus/labels_count_response.qtpl:15 +func WriteLabelsCountResponse(qq422016 qtio422016.Writer, labelEntries []storage.TagEntry) { +//line app/vmselect/prometheus/labels_count_response.qtpl:15 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmselect/prometheus/labels_count_response.qtpl:15 + StreamLabelsCountResponse(qw422016, labelEntries) +//line app/vmselect/prometheus/labels_count_response.qtpl:15 + qt422016.ReleaseWriter(qw422016) +//line app/vmselect/prometheus/labels_count_response.qtpl:15 +} + +//line app/vmselect/prometheus/labels_count_response.qtpl:15 +func LabelsCountResponse(labelEntries []storage.TagEntry) string { +//line app/vmselect/prometheus/labels_count_response.qtpl:15 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmselect/prometheus/labels_count_response.qtpl:15 + WriteLabelsCountResponse(qb422016, labelEntries) +//line app/vmselect/prometheus/labels_count_response.qtpl:15 + qs422016 := string(qb422016.B) +//line app/vmselect/prometheus/labels_count_response.qtpl:15 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmselect/prometheus/labels_count_response.qtpl:15 + return qs422016 +//line app/vmselect/prometheus/labels_count_response.qtpl:15 +} diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index c67d4c6b2..4fcaaae68 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -234,6 +234,23 @@ func LabelValuesHandler(labelName string, w http.ResponseWriter, r *http.Request var labelValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/label/{}/values"}`) +// LabelsCountHandler processes /api/v1/labels/count request. +func LabelsCountHandler(w http.ResponseWriter, r *http.Request) error { + startTime := time.Now() + deadline := getDeadline(r) + labelEntries, err := netstorage.GetLabelEntries(deadline) + if err != nil { + return fmt.Errorf(`cannot obtain label entries: %s`, err) + } + + w.Header().Set("Content-Type", "application/json") + WriteLabelsCountResponse(w, labelEntries) + labelsCountDuration.UpdateDuration(startTime) + return nil +} + +var labelsCountDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/labels/count"}`) + // LabelsHandler processes /api/v1/labels request. // // See https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go index 09ab08eea..535af1380 100644 --- a/app/vmstorage/main.go +++ b/app/vmstorage/main.go @@ -96,6 +96,14 @@ func SearchTagValues(tagKey []byte, maxTagValues int) ([]string, error) { return values, err } +// SearchTagEntries searches for tag entries. +func SearchTagEntries(maxTagKeys, maxTagValues int) ([]storage.TagEntry, error) { + WG.Add(1) + tagEntries, err := Storage.SearchTagEntries(maxTagKeys, maxTagValues) + WG.Done() + return tagEntries, err +} + // GetSeriesCount returns the number of time series in the storage. func GetSeriesCount() (uint64, error) { WG.Add(1) diff --git a/lib/storage/storage.go b/lib/storage/storage.go index 31f5f6d4f..4b8e77e8f 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -516,6 +516,39 @@ func (s *Storage) SearchTagValues(tagKey []byte, maxTagValues int) ([]string, er return s.idb().SearchTagValues(tagKey, maxTagValues) } +// SearchTagEntries returns a list of (tagName -> tagValues) for (accountID, projectID). +func (s *Storage) SearchTagEntries(maxTagKeys, maxTagValues int) ([]TagEntry, error) { + idb := s.idb() + keys, err := idb.SearchTagKeys(maxTagKeys) + if err != nil { + return nil, fmt.Errorf("cannot search tag keys: %s", err) + } + + // Sort keys for faster seeks below + sort.Strings(keys) + + tes := make([]TagEntry, len(keys)) + for i, key := range keys { + values, err := idb.SearchTagValues([]byte(key), maxTagValues) + if err != nil { + return nil, fmt.Errorf("cannot search values for tag %q: %s", key, err) + } + te := &tes[i] + te.Key = key + te.Values = values + } + return tes, nil +} + +// TagEntry contains (tagName -> tagValues) mapping +type TagEntry struct { + // Key is tagName + Key string + + // Values contains all the values for Key. + Values []string +} + // GetSeriesCount returns the approximate number of unique time series. // // It includes the deleted series too and may count the same series