diff --git a/README.md b/README.md
index 045c3d000b..cb81548c10 100644
--- a/README.md
+++ b/README.md
@@ -1332,7 +1332,8 @@ An alternative solution is to query `/internal/resetRollupResultCache` url after
the query cache, which could contain incomplete data cached during the backfilling.
Yet another solution is to increase `-search.cacheTimestampOffset` flag value in order to disable caching
-for data with timestamps close to the current time.
+for data with timestamps close to the current time. Single-node VictoriaMetrics automatically resets response
+cache when samples with timestamps older than `now - search.cacheTimestampOffset` are ingested to it.
## Data updates
diff --git a/app/victoria-metrics/main.go b/app/victoria-metrics/main.go
index c74d747682..517c10aea7 100644
--- a/app/victoria-metrics/main.go
+++ b/app/victoria-metrics/main.go
@@ -3,12 +3,15 @@ package main
import (
"flag"
"fmt"
+ "io"
"net/http"
"os"
+ "path"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
+ "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
@@ -50,7 +53,7 @@ func main() {
logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
startTime := time.Now()
storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
- vmstorage.Init()
+ vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
vmselect.Init()
vminsert.Init()
startSelfScraper()
@@ -80,8 +83,16 @@ func main() {
}
func requestHandler(w http.ResponseWriter, r *http.Request) bool {
- if r.RequestURI == "/" {
- fmt.Fprintf(w, "Single-node VictoriaMetrics. See docs at https://victoriametrics.github.io/")
+ if r.URL.Path == "/" {
+ fmt.Fprintf(w, "<h2>Single-node VictoriaMetrics.</h2></br>")
+ fmt.Fprintf(w, "See docs at <a href='https://victoriametrics.github.io/'>https://victoriametrics.github.io/</a></br>")
+ fmt.Fprintf(w, "Useful endpoints: </br>")
+ writeAPIHelp(w, [][]string{
+ {"/targets", "discovered targets list"},
+ {"/api/v1/targets", "advanced information about discovered targets in JSON format"},
+ {"/metrics", "available service metrics"},
+ {"/api/v1/status/tsdb", "tsdb status page"},
+ })
return true
}
if vminsert.RequestHandler(w, r) {
@@ -95,3 +106,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
}
return false
}
+
+func writeAPIHelp(w io.Writer, pathList [][]string) {
+ pathPrefix := httpserver.GetPathPrefix()
+ for _, p := range pathList {
+ p, doc := p[0], p[1]
+ p = path.Join(pathPrefix, p)
+ fmt.Fprintf(w, "<a href='%s'>%q</a> - %s<br/>", p, p, doc)
+ }
+}
diff --git a/app/victoria-metrics/main_test.go b/app/victoria-metrics/main_test.go
index 6eb2c13c9c..9261576239 100644
--- a/app/victoria-metrics/main_test.go
+++ b/app/victoria-metrics/main_test.go
@@ -20,6 +20,7 @@ import (
testutil "github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics/test"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
+ "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -129,7 +130,7 @@ func setUp() {
storagePath = filepath.Join(os.TempDir(), testStorageSuffix)
processFlags()
logger.Init()
- vmstorage.InitWithoutMetrics()
+ vmstorage.InitWithoutMetrics(promql.ResetRollupResultCacheIfNeeded)
vmselect.Init()
vminsert.Init()
go httpserver.Serve(*httpListenAddr, requestHandler)
@@ -192,7 +193,7 @@ func TestWriteRead(t *testing.T) {
time.Sleep(1 * time.Second)
vmstorage.Stop()
// open storage after stop in write
- vmstorage.InitWithoutMetrics()
+ vmstorage.InitWithoutMetrics(promql.ResetRollupResultCacheIfNeeded)
t.Run("read", testRead)
}
diff --git a/app/vmagent/main.go b/app/vmagent/main.go
index a66bbce222..865f034598 100644
--- a/app/vmagent/main.go
+++ b/app/vmagent/main.go
@@ -5,7 +5,6 @@ import (
"fmt"
"net/http"
"os"
- "strconv"
"strings"
"sync/atomic"
"time"
@@ -144,7 +143,7 @@ func main() {
}
func requestHandler(w http.ResponseWriter, r *http.Request) bool {
- if r.RequestURI == "/" {
+ if r.URL.Path == "/" {
fmt.Fprintf(w, "vmagent - see docs at https://victoriametrics.github.io/vmagent.html")
return true
}
@@ -212,9 +211,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
return true
case "/targets":
promscrapeTargetsRequests.Inc()
- w.Header().Set("Content-Type", "text/plain; charset=utf-8")
- showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
- promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels)
+ promscrape.WriteHumanReadableTargetsStatus(w, r)
return true
case "/api/v1/targets":
promscrapeAPIV1TargetsRequests.Inc()
diff --git a/app/vminsert/main.go b/app/vminsert/main.go
index 3bfd7f2c33..d58c0f86fa 100644
--- a/app/vminsert/main.go
+++ b/app/vminsert/main.go
@@ -4,7 +4,6 @@ import (
"flag"
"fmt"
"net/http"
- "strconv"
"strings"
"sync/atomic"
@@ -155,9 +154,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
return true
case "/targets":
promscrapeTargetsRequests.Inc()
- w.Header().Set("Content-Type", "text/plain; charset=utf-8")
- showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
- promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels)
+ promscrape.WriteHumanReadableTargetsStatus(w, r)
return true
case "/api/v1/targets":
promscrapeAPIV1TargetsRequests.Inc()
diff --git a/app/vmselect/promql/rollup_result_cache.go b/app/vmselect/promql/rollup_result_cache.go
index db6f289315..8ef54398e2 100644
--- a/app/vmselect/promql/rollup_result_cache.go
+++ b/app/vmselect/promql/rollup_result_cache.go
@@ -13,6 +13,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache"
"github.com/VictoriaMetrics/fastcache"
"github.com/VictoriaMetrics/metrics"
@@ -25,6 +26,39 @@ var (
"due to time synchronization issues between VictoriaMetrics and data sources")
)
+// ResetRollupResultCacheIfNeeded resets rollup result cache if mrs contains timestamps outside `now - search.cacheTimestampOffset`.
+func ResetRollupResultCacheIfNeeded(mrs []storage.MetricRow) {
+ checkRollupResultCacheResetOnce.Do(func() {
+ go checkRollupResultCacheReset()
+ })
+ minTimestamp := int64(fasttime.UnixTimestamp()*1000) - cacheTimestampOffset.Milliseconds() + checkRollupResultCacheResetInterval.Milliseconds()
+ needCacheReset := false
+ for i := range mrs {
+ if mrs[i].Timestamp < minTimestamp {
+ needCacheReset = true
+ break
+ }
+ }
+ if needCacheReset {
+ // Do not call ResetRollupResultCache() here, since it may be heavy when frequently called.
+ atomic.StoreUint32(&needRollupResultCacheReset, 1)
+ }
+}
+
+func checkRollupResultCacheReset() {
+ for {
+ time.Sleep(checkRollupResultCacheResetInterval)
+ if atomic.SwapUint32(&needRollupResultCacheReset, 0) > 0 {
+ ResetRollupResultCache()
+ }
+ }
+}
+
+const checkRollupResultCacheResetInterval = 5 * time.Second
+
+var needRollupResultCacheReset uint32
+var checkRollupResultCacheResetOnce sync.Once
+
var rollupResultCacheV = &rollupResultCache{
c: workingsetcache.New(1024*1024, time.Hour), // This is a cache for testing.
}
diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go
index ca9378663e..211d482464 100644
--- a/app/vmstorage/main.go
+++ b/app/vmstorage/main.go
@@ -58,19 +58,20 @@ func CheckTimeRange(tr storage.TimeRange) error {
}
// Init initializes vmstorage.
-func Init() {
- InitWithoutMetrics()
+func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
+ InitWithoutMetrics(resetCacheIfNeeded)
registerStorageMetrics()
}
// InitWithoutMetrics must be called instead of Init inside tests.
//
// This allows multiple Init / Stop cycles.
-func InitWithoutMetrics() {
+func InitWithoutMetrics(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
if err := encoding.CheckPrecisionBits(uint8(*precisionBits)); err != nil {
logger.Fatalf("invalid `-precisionBits`: %s", err)
}
+ resetResponseCacheIfNeeded = resetCacheIfNeeded
storage.SetFinalMergeDelay(*finalMergeDelay)
storage.SetBigMergeWorkersCount(*bigMergeConcurrency)
storage.SetSmallMergeWorkersCount(*smallMergeConcurrency)
@@ -108,8 +109,12 @@ var Storage *storage.Storage
// Use syncwg instead of sync, since Add is called from concurrent goroutines.
var WG syncwg.WaitGroup
+// resetResponseCacheIfNeeded is a callback for automatic resetting of response cache if needed.
+var resetResponseCacheIfNeeded func(mrs []storage.MetricRow)
+
// AddRows adds mrs to the storage.
func AddRows(mrs []storage.MetricRow) error {
+ resetResponseCacheIfNeeded(mrs)
WG.Add(1)
err := Storage.AddRows(mrs, uint8(*precisionBits))
WG.Done()
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3c2afbff71..b39011b339 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,8 @@
# tip
+* FEATURE: automatically reset response cache when samples with timestamps older than `now - search.cacheTimestampOffset` are ingested to VictoriaMetrics. This makes unnecessary disabling response cache during data backfilling or resetting it after backfilling is complete as described [in these docs](https://victoriametrics.github.io/#backfilling). This feature applies only to single-node VictoriaMetrics. It doesn't apply to cluster version of VictoriaMetrics because `vminsert` nodes don't know about `vmselect` nodes where the response cache must be reset.
+* FEATURE: vmagent: return user-friendly HTML page when requesting `/targets` page from web browser. The page is returned in the old plaintext format when requesting via curl or similar tool.
* FEATURE: allow multiple whitespace chars between measurements, fields and timestamp when parsing InfluxDB line protocol.
Though [InfluxDB line protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/) denies multiple whitespace chars between these entities,
some apps improperly put multiple whitespace chars. This workaround allows accepting data from such apps.
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index 045c3d000b..cb81548c10 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -1332,7 +1332,8 @@ An alternative solution is to query `/internal/resetRollupResultCache` url after
the query cache, which could contain incomplete data cached during the backfilling.
Yet another solution is to increase `-search.cacheTimestampOffset` flag value in order to disable caching
-for data with timestamps close to the current time.
+for data with timestamps close to the current time. Single-node VictoriaMetrics automatically resets response
+cache when samples with timestamps older than `now - search.cacheTimestampOffset` are ingested to it.
## Data updates
diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go
index daa8c8e59f..6a760af5bd 100644
--- a/lib/httpserver/httpserver.go
+++ b/lib/httpserver/httpserver.go
@@ -274,9 +274,12 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
}
func getCanonicalPath(path string) (string, error) {
- if len(*pathPrefix) == 0 {
+ if len(*pathPrefix) == 0 || path == "/" {
return path, nil
}
+ if *pathPrefix == path {
+ return "/", nil
+ }
prefix := *pathPrefix
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
@@ -573,3 +576,8 @@ func isTrivialNetworkError(err error) bool {
func IsTLS() bool {
return *tlsEnable
}
+
+// GetPathPrefix - returns http server path prefix.
+func GetPathPrefix() string {
+ return *pathPrefix
+}
diff --git a/lib/promscrape/targets_response.qtpl b/lib/promscrape/targets_response.qtpl
new file mode 100644
index 0000000000..f1fa91f9b5
--- /dev/null
+++ b/lib/promscrape/targets_response.qtpl
@@ -0,0 +1,104 @@
+{% import "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+%}
+
+{% stripspace %}
+
+{% func TargetsResponsePlain (jts []jobTargetsStatuses, showOriginLabels bool) -%}
+
+{% for _, js := range jts %}
+job={%q= js.job %}{% space %} ({%d js.upCount %}/{%d js.targetsTotal %} {% space %} up)
+{% newline %}
+{% for _, ts := range js.targetsStatus %}
+ {% code
+ labels := promLabelsString(ts.labels)
+ ol := promLabelsString(ts.originalLabels)
+ %}
+{%s= "\t" %}state={% if ts.up %}up{% else %}down{% endif %},
+ {% space %} endpoint={%s= ts.endpoint %},
+ {% space %} labels={%s= labels %}
+ {% if showOriginLabels %},{% space %} originalLabels={%s= ol %}{% endif %},
+ {% space %} last_scrape={%f.3 ts.lastScrapeTime.Seconds() %}s {% space %} ago,
+ {% space %} scrape_duration={%f.3 float64(ts.scrapeDuration.Seconds()) %}s,
+ {% space %} error={%q= ts.error %}
+ {% newline %}
+{% endfor %}
+{% endfor %}
+{% newline %}
+
+{% endfunc %}
+
+{% func TargetsResponseHTML(jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) %}
+<!DOCTYPE html>
+<style>
+ .border{
+ border-collapse: collapse;
+ border: 1px solid black;
+ }
+ .table-row:hover{
+ background-color: #f5f5f5;
+ }
+</style>
+<html lang="en">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+ <title>VictoriaMetrics Database</title>
+ </head>
+ <body>
+ <h1>Targets</h1>
+ <div id="showTargets" class="btn-group btn-group-toggle" data-toggle="buttons">
+ <label class="btn">
+ <input type="radio" name="targets" id="all-targets" autocomplete="off" onclick="location.href='{%s= redirectPath %}';" {% if !onlyUnhealthy %}checked {% endif %}> All
+ </label>
+ <label class="btn">
+ <input type="radio" name="targets" id="unhealthy-targets" autocomplete="off" onclick="location.href='{%s= redirectPath %}?show_only_unhealthy=true';" {% if onlyUnhealthy %}checked {% endif %}> Unhealthy
+ </label>
+ <br />
+ </div>
+ {% for _,js :=range jts %}
+ <div class="table-container">
+ <h2 class="job_header danger">
+ <a id="job-{%q= js.job %}" >{%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up)</a>
+ </h2>
+ <table class="table-bordered table-hover border">
+ <thead class="job_details border">
+ <tr class="table-row border">
+ <th class="border">Endpoint</th>
+ <th class="border">State</th>
+ <th class="border">Labels</th>
+ <th class="border">Last Scrape</th>
+ <th class="border">Scrape Duration</th>
+ <th class="border">Error</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for _, ts := range js.targetsStatus %}
+ {% if onlyUnhealthy && ts.up %} {% continue %} {% endif %}
+ <tr class="table-row border">
+ <td class="endpoint border">
+ <a href="{%s= ts.endpoint %}">{%s= ts.endpoint %}</a><br>
+ </td>
+ <td class="state border">
+ <span class="state_indicator">{% if ts.up %}UP{% else %}DOWN{% endif %}</span>
+ </td>
+ <td class="labels border", title="Original {% space %} labels: {% space %} {%= formatLabel(ts.originalLabels) %}">
+ {%= formatLabel(ts.labels) %}
+ </td>
+ <td class="last-scrape border">{%s ts.lastScrapeTime.String() %} {% space %} ago</td>
+ <td class="scrape-duration border">{%s ts.scrapeDuration.String() %}</td>
+ <td class="errors border"><span class="alert alert-danger state_indicator">{%s= ts.error %}</span></td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ </div>
+ {% endfor %}
+ </body>
+</html>
+{% endfunc %}
+
+{% func formatLabel(labels []prompbmarshal.Label) %}
+{% for _, label := range labels %}
+ {% space %} {%s label.Name %}={%q label.Value %} {% space %}
+{% endfor %}
+{% endfunc %}
+{% endstripspace %}
diff --git a/lib/promscrape/targets_response.qtpl.go b/lib/promscrape/targets_response.qtpl.go
new file mode 100644
index 0000000000..b0472bddbf
--- /dev/null
+++ b/lib/promscrape/targets_response.qtpl.go
@@ -0,0 +1,349 @@
+// Code generated by qtc from "targets_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line lib/promscrape/targets_response.qtpl:1
+package promscrape
+
+//line lib/promscrape/targets_response.qtpl:1
+import "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+
+//line lib/promscrape/targets_response.qtpl:6
+import (
+ qtio422016 "io"
+
+ qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line lib/promscrape/targets_response.qtpl:6
+var (
+ _ = qtio422016.Copy
+ _ = qt422016.AcquireByteBuffer
+)
+
+//line lib/promscrape/targets_response.qtpl:6
+func StreamTargetsResponsePlain(qw422016 *qt422016.Writer, jts []jobTargetsStatuses, showOriginLabels bool) {
+//line lib/promscrape/targets_response.qtpl:8
+ for _, js := range jts {
+//line lib/promscrape/targets_response.qtpl:8
+ qw422016.N().S(`job=`)
+//line lib/promscrape/targets_response.qtpl:9
+ qw422016.N().Q(js.job)
+//line lib/promscrape/targets_response.qtpl:9
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:9
+ qw422016.N().S(`(`)
+//line lib/promscrape/targets_response.qtpl:9
+ qw422016.N().D(js.upCount)
+//line lib/promscrape/targets_response.qtpl:9
+ qw422016.N().S(`/`)
+//line lib/promscrape/targets_response.qtpl:9
+ qw422016.N().D(js.targetsTotal)
+//line lib/promscrape/targets_response.qtpl:9
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:9
+ qw422016.N().S(`up)`)
+//line lib/promscrape/targets_response.qtpl:10
+ qw422016.N().S(`
+`)
+//line lib/promscrape/targets_response.qtpl:11
+ for _, ts := range js.targetsStatus {
+//line lib/promscrape/targets_response.qtpl:13
+ labels := promLabelsString(ts.labels)
+ ol := promLabelsString(ts.originalLabels)
+
+//line lib/promscrape/targets_response.qtpl:16
+ qw422016.N().S("\t")
+//line lib/promscrape/targets_response.qtpl:16
+ qw422016.N().S(`state=`)
+//line lib/promscrape/targets_response.qtpl:16
+ if ts.up {
+//line lib/promscrape/targets_response.qtpl:16
+ qw422016.N().S(`up`)
+//line lib/promscrape/targets_response.qtpl:16
+ } else {
+//line lib/promscrape/targets_response.qtpl:16
+ qw422016.N().S(`down`)
+//line lib/promscrape/targets_response.qtpl:16
+ }
+//line lib/promscrape/targets_response.qtpl:16
+ qw422016.N().S(`,`)
+//line lib/promscrape/targets_response.qtpl:17
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:17
+ qw422016.N().S(`endpoint=`)
+//line lib/promscrape/targets_response.qtpl:17
+ qw422016.N().S(ts.endpoint)
+//line lib/promscrape/targets_response.qtpl:17
+ qw422016.N().S(`,`)
+//line lib/promscrape/targets_response.qtpl:18
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:18
+ qw422016.N().S(`labels=`)
+//line lib/promscrape/targets_response.qtpl:18
+ qw422016.N().S(labels)
+//line lib/promscrape/targets_response.qtpl:19
+ if showOriginLabels {
+//line lib/promscrape/targets_response.qtpl:19
+ qw422016.N().S(`,`)
+//line lib/promscrape/targets_response.qtpl:19
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:19
+ qw422016.N().S(`originalLabels=`)
+//line lib/promscrape/targets_response.qtpl:19
+ qw422016.N().S(ol)
+//line lib/promscrape/targets_response.qtpl:19
+ }
+//line lib/promscrape/targets_response.qtpl:19
+ qw422016.N().S(`,`)
+//line lib/promscrape/targets_response.qtpl:20
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:20
+ qw422016.N().S(`last_scrape=`)
+//line lib/promscrape/targets_response.qtpl:20
+ qw422016.N().FPrec(ts.lastScrapeTime.Seconds(), 3)
+//line lib/promscrape/targets_response.qtpl:20
+ qw422016.N().S(`s`)
+//line lib/promscrape/targets_response.qtpl:20
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:20
+ qw422016.N().S(`ago,`)
+//line lib/promscrape/targets_response.qtpl:21
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:21
+ qw422016.N().S(`scrape_duration=`)
+//line lib/promscrape/targets_response.qtpl:21
+ qw422016.N().FPrec(float64(ts.scrapeDuration.Seconds()), 3)
+//line lib/promscrape/targets_response.qtpl:21
+ qw422016.N().S(`s,`)
+//line lib/promscrape/targets_response.qtpl:22
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:22
+ qw422016.N().S(`error=`)
+//line lib/promscrape/targets_response.qtpl:22
+ qw422016.N().Q(ts.error)
+//line lib/promscrape/targets_response.qtpl:23
+ qw422016.N().S(`
+`)
+//line lib/promscrape/targets_response.qtpl:24
+ }
+//line lib/promscrape/targets_response.qtpl:25
+ }
+//line lib/promscrape/targets_response.qtpl:26
+ qw422016.N().S(`
+`)
+//line lib/promscrape/targets_response.qtpl:28
+}
+
+//line lib/promscrape/targets_response.qtpl:28
+func WriteTargetsResponsePlain(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, showOriginLabels bool) {
+//line lib/promscrape/targets_response.qtpl:28
+ qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promscrape/targets_response.qtpl:28
+ StreamTargetsResponsePlain(qw422016, jts, showOriginLabels)
+//line lib/promscrape/targets_response.qtpl:28
+ qt422016.ReleaseWriter(qw422016)
+//line lib/promscrape/targets_response.qtpl:28
+}
+
+//line lib/promscrape/targets_response.qtpl:28
+func TargetsResponsePlain(jts []jobTargetsStatuses, showOriginLabels bool) string {
+//line lib/promscrape/targets_response.qtpl:28
+ qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promscrape/targets_response.qtpl:28
+ WriteTargetsResponsePlain(qb422016, jts, showOriginLabels)
+//line lib/promscrape/targets_response.qtpl:28
+ qs422016 := string(qb422016.B)
+//line lib/promscrape/targets_response.qtpl:28
+ qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promscrape/targets_response.qtpl:28
+ return qs422016
+//line lib/promscrape/targets_response.qtpl:28
+}
+
+//line lib/promscrape/targets_response.qtpl:30
+func StreamTargetsResponseHTML(qw422016 *qt422016.Writer, jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) {
+//line lib/promscrape/targets_response.qtpl:30
+ qw422016.N().S(`<!DOCTYPE html><style>.border{border-collapse: collapse;border: 1px solid black;}.table-row:hover{background-color: #f5f5f5;}</style><html lang="en"><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"><title>VictoriaMetrics Database</title></head><body><h1>Targets</h1><div id="showTargets" class="btn-group btn-group-toggle" data-toggle="buttons"><label class="btn"><input type="radio" name="targets" id="all-targets" autocomplete="off" onclick="location.href='`)
+//line lib/promscrape/targets_response.qtpl:50
+ qw422016.N().S(redirectPath)
+//line lib/promscrape/targets_response.qtpl:50
+ qw422016.N().S(`';"`)
+//line lib/promscrape/targets_response.qtpl:50
+ if !onlyUnhealthy {
+//line lib/promscrape/targets_response.qtpl:50
+ qw422016.N().S(`checked`)
+//line lib/promscrape/targets_response.qtpl:50
+ }
+//line lib/promscrape/targets_response.qtpl:50
+ qw422016.N().S(`> All</label><label class="btn"><input type="radio" name="targets" id="unhealthy-targets" autocomplete="off" onclick="location.href='`)
+//line lib/promscrape/targets_response.qtpl:53
+ qw422016.N().S(redirectPath)
+//line lib/promscrape/targets_response.qtpl:53
+ qw422016.N().S(`?show_only_unhealthy=true';"`)
+//line lib/promscrape/targets_response.qtpl:53
+ if onlyUnhealthy {
+//line lib/promscrape/targets_response.qtpl:53
+ qw422016.N().S(`checked`)
+//line lib/promscrape/targets_response.qtpl:53
+ }
+//line lib/promscrape/targets_response.qtpl:53
+ qw422016.N().S(`> Unhealthy</label><br /></div>`)
+//line lib/promscrape/targets_response.qtpl:57
+ for _, js := range jts {
+//line lib/promscrape/targets_response.qtpl:57
+ qw422016.N().S(`<div class="table-container"><h2 class="job_header danger"><a id="job-`)
+//line lib/promscrape/targets_response.qtpl:60
+ qw422016.N().Q(js.job)
+//line lib/promscrape/targets_response.qtpl:60
+ qw422016.N().S(`" >`)
+//line lib/promscrape/targets_response.qtpl:60
+ qw422016.N().Q(js.job)
+//line lib/promscrape/targets_response.qtpl:60
+ qw422016.N().S(`(`)
+//line lib/promscrape/targets_response.qtpl:60
+ qw422016.N().D(js.upCount)
+//line lib/promscrape/targets_response.qtpl:60
+ qw422016.N().S(`/`)
+//line lib/promscrape/targets_response.qtpl:60
+ qw422016.N().D(js.targetsTotal)
+//line lib/promscrape/targets_response.qtpl:60
+ qw422016.N().S(`up)</a></h2><table class="table-bordered table-hover border"><thead class="job_details border"><tr class="table-row border"><th class="border">Endpoint</th><th class="border">State</th><th class="border">Labels</th><th class="border">Last Scrape</th><th class="border">Scrape Duration</th><th class="border">Error</th></tr></thead><tbody>`)
+//line lib/promscrape/targets_response.qtpl:74
+ for _, ts := range js.targetsStatus {
+//line lib/promscrape/targets_response.qtpl:75
+ if onlyUnhealthy && ts.up {
+//line lib/promscrape/targets_response.qtpl:75
+ continue
+//line lib/promscrape/targets_response.qtpl:75
+ }
+//line lib/promscrape/targets_response.qtpl:75
+ qw422016.N().S(`<tr class="table-row border"><td class="endpoint border"><a href="`)
+//line lib/promscrape/targets_response.qtpl:78
+ qw422016.N().S(ts.endpoint)
+//line lib/promscrape/targets_response.qtpl:78
+ qw422016.N().S(`">`)
+//line lib/promscrape/targets_response.qtpl:78
+ qw422016.N().S(ts.endpoint)
+//line lib/promscrape/targets_response.qtpl:78
+ qw422016.N().S(`</a><br></td><td class="state border"><span class="state_indicator">`)
+//line lib/promscrape/targets_response.qtpl:81
+ if ts.up {
+//line lib/promscrape/targets_response.qtpl:81
+ qw422016.N().S(`UP`)
+//line lib/promscrape/targets_response.qtpl:81
+ } else {
+//line lib/promscrape/targets_response.qtpl:81
+ qw422016.N().S(`DOWN`)
+//line lib/promscrape/targets_response.qtpl:81
+ }
+//line lib/promscrape/targets_response.qtpl:81
+ qw422016.N().S(`</span></td><td class="labels border", title="Original`)
+//line lib/promscrape/targets_response.qtpl:83
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:83
+ qw422016.N().S(`labels:`)
+//line lib/promscrape/targets_response.qtpl:83
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:83
+ streamformatLabel(qw422016, ts.originalLabels)
+//line lib/promscrape/targets_response.qtpl:83
+ qw422016.N().S(`">`)
+//line lib/promscrape/targets_response.qtpl:84
+ streamformatLabel(qw422016, ts.labels)
+//line lib/promscrape/targets_response.qtpl:84
+ qw422016.N().S(`</td><td class="last-scrape border">`)
+//line lib/promscrape/targets_response.qtpl:86
+ qw422016.E().S(ts.lastScrapeTime.String())
+//line lib/promscrape/targets_response.qtpl:86
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:86
+ qw422016.N().S(`ago</td><td class="scrape-duration border">`)
+//line lib/promscrape/targets_response.qtpl:87
+ qw422016.E().S(ts.scrapeDuration.String())
+//line lib/promscrape/targets_response.qtpl:87
+ qw422016.N().S(`</td><td class="errors border"><span class="alert alert-danger state_indicator">`)
+//line lib/promscrape/targets_response.qtpl:88
+ qw422016.N().S(ts.error)
+//line lib/promscrape/targets_response.qtpl:88
+ qw422016.N().S(`</span></td></tr>`)
+//line lib/promscrape/targets_response.qtpl:90
+ }
+//line lib/promscrape/targets_response.qtpl:90
+ qw422016.N().S(`</tbody></table></div>`)
+//line lib/promscrape/targets_response.qtpl:94
+ }
+//line lib/promscrape/targets_response.qtpl:94
+ qw422016.N().S(`</body></html>`)
+//line lib/promscrape/targets_response.qtpl:97
+}
+
+//line lib/promscrape/targets_response.qtpl:97
+func WriteTargetsResponseHTML(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) {
+//line lib/promscrape/targets_response.qtpl:97
+ qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promscrape/targets_response.qtpl:97
+ StreamTargetsResponseHTML(qw422016, jts, redirectPath, onlyUnhealthy)
+//line lib/promscrape/targets_response.qtpl:97
+ qt422016.ReleaseWriter(qw422016)
+//line lib/promscrape/targets_response.qtpl:97
+}
+
+//line lib/promscrape/targets_response.qtpl:97
+func TargetsResponseHTML(jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) string {
+//line lib/promscrape/targets_response.qtpl:97
+ qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promscrape/targets_response.qtpl:97
+ WriteTargetsResponseHTML(qb422016, jts, redirectPath, onlyUnhealthy)
+//line lib/promscrape/targets_response.qtpl:97
+ qs422016 := string(qb422016.B)
+//line lib/promscrape/targets_response.qtpl:97
+ qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promscrape/targets_response.qtpl:97
+ return qs422016
+//line lib/promscrape/targets_response.qtpl:97
+}
+
+//line lib/promscrape/targets_response.qtpl:99
+func streamformatLabel(qw422016 *qt422016.Writer, labels []prompbmarshal.Label) {
+//line lib/promscrape/targets_response.qtpl:100
+ for _, label := range labels {
+//line lib/promscrape/targets_response.qtpl:101
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:101
+ qw422016.E().S(label.Name)
+//line lib/promscrape/targets_response.qtpl:101
+ qw422016.N().S(`=`)
+//line lib/promscrape/targets_response.qtpl:101
+ qw422016.E().Q(label.Value)
+//line lib/promscrape/targets_response.qtpl:101
+ qw422016.N().S(` `)
+//line lib/promscrape/targets_response.qtpl:102
+ }
+//line lib/promscrape/targets_response.qtpl:103
+}
+
+//line lib/promscrape/targets_response.qtpl:103
+func writeformatLabel(qq422016 qtio422016.Writer, labels []prompbmarshal.Label) {
+//line lib/promscrape/targets_response.qtpl:103
+ qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promscrape/targets_response.qtpl:103
+ streamformatLabel(qw422016, labels)
+//line lib/promscrape/targets_response.qtpl:103
+ qt422016.ReleaseWriter(qw422016)
+//line lib/promscrape/targets_response.qtpl:103
+}
+
+//line lib/promscrape/targets_response.qtpl:103
+func formatLabel(labels []prompbmarshal.Label) string {
+//line lib/promscrape/targets_response.qtpl:103
+ qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promscrape/targets_response.qtpl:103
+ writeformatLabel(qb422016, labels)
+//line lib/promscrape/targets_response.qtpl:103
+ qs422016 := string(qb422016.B)
+//line lib/promscrape/targets_response.qtpl:103
+ qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promscrape/targets_response.qtpl:103
+ return qs422016
+//line lib/promscrape/targets_response.qtpl:103
+}
diff --git a/lib/promscrape/targetstatus.go b/lib/promscrape/targetstatus.go
index 0a75f00e41..93ef68bcc7 100644
--- a/lib/promscrape/targetstatus.go
+++ b/lib/promscrape/targetstatus.go
@@ -4,11 +4,16 @@ import (
"flag"
"fmt"
"io"
+ "net/http"
+ "path"
"sort"
+ "strconv"
+ "strings"
"sync"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+ "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
)
@@ -19,9 +24,17 @@ var maxDroppedTargets = flag.Int("promscrape.maxDroppedTargets", 1000, "The maxi
var tsmGlobal = newTargetStatusMap()
-// WriteHumanReadableTargetsStatus writes human-readable status for all the scrape targets to w.
-func WriteHumanReadableTargetsStatus(w io.Writer, showOriginalLabels bool) {
- tsmGlobal.WriteHumanReadable(w, showOriginalLabels)
+// WriteHumanReadableTargetsStatus writes human-readable status for all the scrape targets to w according to r.
+func WriteHumanReadableTargetsStatus(w http.ResponseWriter, r *http.Request) {
+ showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels"))
+ showOnlyUnhealthy, _ := strconv.ParseBool(r.FormValue("show_only_unhealthy"))
+ if accept := r.Header.Get("Accept"); strings.Contains(accept, "text/html") {
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ tsmGlobal.WriteTargetsHTML(w, showOnlyUnhealthy)
+ } else {
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+ tsmGlobal.WriteTargetsPlain(w, showOriginalLabels)
+ }
}
// WriteAPIV1Targets writes /api/v1/targets to w according to https://prometheus.io/docs/prometheus/latest/querying/api/#targets
@@ -162,64 +175,6 @@ func writeLabelsJSON(w io.Writer, labels []prompbmarshal.Label) {
fmt.Fprintf(w, `}`)
}
-func (tsm *targetStatusMap) WriteHumanReadable(w io.Writer, showOriginalLabels bool) {
- byJob := make(map[string][]targetStatus)
- tsm.mu.Lock()
- for _, st := range tsm.m {
- job := st.sw.Job()
- byJob[job] = append(byJob[job], *st)
- }
- tsm.mu.Unlock()
-
- var jss []jobStatus
- for job, statuses := range byJob {
- jss = append(jss, jobStatus{
- job: job,
- statuses: statuses,
- })
- }
- sort.Slice(jss, func(i, j int) bool {
- return jss[i].job < jss[j].job
- })
-
- for _, js := range jss {
- sts := js.statuses
- sort.Slice(sts, func(i, j int) bool {
- return sts[i].sw.ScrapeURL < sts[j].sw.ScrapeURL
- })
- ups := 0
- for _, st := range sts {
- if st.up {
- ups++
- }
- }
- fmt.Fprintf(w, "job=%q (%d/%d up)\n", js.job, ups, len(sts))
- for _, st := range sts {
- state := "up"
- if !st.up {
- state = "down"
- }
- labelsStr := st.sw.LabelsString()
- if showOriginalLabels {
- labelsStr += ", originalLabels=" + promLabelsString(st.sw.OriginalLabels)
- }
- lastScrape := st.getDurationFromLastScrape()
- errMsg := ""
- if st.err != nil {
- errMsg = st.err.Error()
- }
- fmt.Fprintf(w, "\tstate=%s, endpoint=%s, labels=%s, last_scrape=%.3fs ago, scrape_duration=%.3fs, error=%q\n",
- state, st.sw.ScrapeURL, labelsStr, lastScrape.Seconds(), float64(st.scrapeDuration)/1000, errMsg)
- }
- }
- fmt.Fprintf(w, "\n")
-}
-
-type jobStatus struct {
- job string
- statuses []targetStatus
-}
-
type targetStatus struct {
sw ScrapeWork
up bool
@@ -303,3 +258,84 @@ func (dt *droppedTargets) WriteDroppedTargetsJSON(w io.Writer) {
var droppedTargetsMap = &droppedTargets{
m: make(map[string]droppedTarget),
}
+
+type jobTargetStatus struct {
+ up bool
+ endpoint string
+ labels []prompbmarshal.Label
+ originalLabels []prompbmarshal.Label
+ lastScrapeTime time.Duration
+ scrapeDuration time.Duration
+ error string
+}
+
+type jobTargetsStatuses struct {
+ job string
+ upCount int
+ targetsTotal int
+ targetsStatus []jobTargetStatus
+}
+
+func (tsm *targetStatusMap) getTargetsStatusByJob() []jobTargetsStatuses {
+ byJob := make(map[string][]targetStatus)
+ tsm.mu.Lock()
+ for _, st := range tsm.m {
+ job := st.sw.Job()
+ byJob[job] = append(byJob[job], *st)
+ }
+ tsm.mu.Unlock()
+
+ var jts []jobTargetsStatuses
+ for job, statuses := range byJob {
+ sort.Slice(statuses, func(i, j int) bool {
+ return statuses[i].sw.ScrapeURL < statuses[j].sw.ScrapeURL
+ })
+ ups := 0
+ var targetsStatuses []jobTargetStatus
+ for _, ts := range statuses {
+ if ts.up {
+ ups++
+ }
+ }
+ for _, st := range statuses {
+ errMsg := ""
+ if st.err != nil {
+ errMsg = st.err.Error()
+ }
+ targetsStatuses = append(targetsStatuses, jobTargetStatus{
+ up: st.up,
+ endpoint: st.sw.ScrapeURL,
+ labels: promrelabel.FinalizeLabels(nil, st.sw.Labels),
+ originalLabels: st.sw.OriginalLabels,
+ lastScrapeTime: st.getDurationFromLastScrape(),
+ scrapeDuration: time.Duration(st.scrapeDuration),
+ error: errMsg,
+ })
+ }
+ jts = append(jts, jobTargetsStatuses{
+ job: job,
+ upCount: ups,
+ targetsTotal: len(statuses),
+ targetsStatus: targetsStatuses,
+ })
+ }
+ sort.Slice(jts, func(i, j int) bool {
+ return jts[i].job < jts[j].job
+ })
+ return jts
+}
+
+// WriteTargetsHTML writes targets status grouped by job into writer w in html table,
+// accepts filter to show only unhealthy targets.
+func (tsm *targetStatusMap) WriteTargetsHTML(w io.Writer, showOnlyUnhealthy bool) {
+ jss := tsm.getTargetsStatusByJob()
+ targetsPath := path.Join(httpserver.GetPathPrefix(), "/targets")
+ WriteTargetsResponseHTML(w, jss, targetsPath, showOnlyUnhealthy)
+}
+
+// WriteTargetsPlain writes targets grouped by job into writer w in plain text,
+// accept filter to show original labels.
+func (tsm *targetStatusMap) WriteTargetsPlain(w io.Writer, showOriginalLabels bool) {
+ jss := tsm.getTargetsStatusByJob()
+ WriteTargetsResponsePlain(w, jss, showOriginalLabels)
+}