From 3ad80e281f5b73ed7a6a0ee741a8945388c722d7 Mon Sep 17 00:00:00 2001 From: Yury Molodov Date: Thu, 20 Jul 2023 00:47:21 +0200 Subject: [PATCH] vmui: add Active Queries page (#4653) * feat: add page to display a list of active queries (#4598) * app/vmagent: code formatting * fix: remove console --------- Co-authored-by: dmitryk-dk --- app/vmselect/main.go | 12 ++- app/vmselect/promql/active_queries.go | 37 ++++++-- app/vmui/packages/vmui/src/App.tsx | 5 ++ .../packages/vmui/src/api/active-queries.ts | 2 + .../packages/vmui/src/constants/navigation.ts | 4 + .../hooks/useFetchActiveQueries.ts | 51 +++++++++++ .../vmui/src/pages/ActiveQueries/index.tsx | 89 +++++++++++++++++++ .../vmui/src/pages/ActiveQueries/style.scss | 25 ++++++ app/vmui/packages/vmui/src/router/index.ts | 5 ++ app/vmui/packages/vmui/src/types/index.ts | 13 +++ docs/CHANGELOG.md | 1 + lib/logstorage/parser_test.go | 4 +- 12 files changed, 237 insertions(+), 11 deletions(-) create mode 100644 app/vmui/packages/vmui/src/api/active-queries.ts create mode 100644 app/vmui/packages/vmui/src/pages/ActiveQueries/hooks/useFetchActiveQueries.ts create mode 100644 app/vmui/packages/vmui/src/pages/ActiveQueries/index.tsx create mode 100644 app/vmui/packages/vmui/src/pages/ActiveQueries/style.scss diff --git a/app/vmselect/main.go b/app/vmselect/main.go index e4f74227e..61469fa21 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -266,6 +266,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool { } return true } + if path == "/api/v1/status/active_queries" { + globalStatusActiveQueriesRequests.Inc() + httpserver.EnableCORS(w, r) + promql.WriteActiveQueries(w, r) + return true + } if path == "/admin/tenants" { tenantsRequests.Inc() httpserver.EnableCORS(w, r) @@ -500,7 +506,8 @@ func selectHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW return true case "prometheus/api/v1/status/active_queries": statusActiveQueriesRequests.Inc() - promql.WriteActiveQueries(w) + httpserver.EnableCORS(w, r) + promql.WriteActiveQueriesForTenant(at, w, r) return true case "prometheus/api/v1/status/top_queries": topQueriesRequests.Inc() @@ -767,7 +774,8 @@ var ( statusTSDBRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/status/tsdb"}`) statusTSDBErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/status/tsdb"}`) - statusActiveQueriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}prometheus/api/v1/status/active_queries"}`) + globalStatusActiveQueriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/status/active_queries"}`) + statusActiveQueriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}prometheus/api/v1/status/active_queries"}`) topQueriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/status/top_queries"}`) topQueriesErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/status/top_queries"}`) diff --git a/app/vmselect/promql/active_queries.go b/app/vmselect/promql/active_queries.go index e808c8565..b7320bcb0 100644 --- a/app/vmselect/promql/active_queries.go +++ b/app/vmselect/promql/active_queries.go @@ -2,27 +2,50 @@ package promql import ( "fmt" - "io" + "net/http" "sort" "sync" "sync/atomic" "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" ) -// WriteActiveQueries writes active queries to w. -// -// The written active queries are sorted in descending order of their exeuction duration. -func WriteActiveQueries(w io.Writer) { +// WriteActiveQueriesForTenant writes active queries for the given (accountID, projectID) to w. +func WriteActiveQueriesForTenant(at *auth.Token, w http.ResponseWriter, r *http.Request) { aqes := activeQueriesV.GetAll() + var dst []activeQueryEntry + for _, aqe := range aqes { + if aqe.accountID == at.AccountID && aqe.projectID == at.ProjectID { + dst = append(dst, aqe) + } + } + writeActiveQueries(w, dst) +} + +// WriteActiveQueries writes active queries to w. +func WriteActiveQueries(w http.ResponseWriter, r *http.Request) { + aqes := activeQueriesV.GetAll() + writeActiveQueries(w, aqes) +} + +func writeActiveQueries(w http.ResponseWriter, aqes []activeQueryEntry) { + w.Header().Set("Content-Type", "application/json") + sort.Slice(aqes, func(i, j int) bool { return aqes[i].startTime.Sub(aqes[j].startTime) < 0 }) now := time.Now() - for _, aqe := range aqes { + fmt.Fprintf(w, `{"status":"ok","data":[`) + for i, aqe := range aqes { d := now.Sub(aqe.startTime) - fmt.Fprintf(w, "\tduration: %.3fs, id=%016X, remote_addr=%s, accountID=%d, projectID=%d, query=%q, start=%d, end=%d, step=%d\n", + fmt.Fprintf(w, `{"duration":"%.3fs","id":"%016X","remote_addr":%s,"account_id":"%d","project_id":"%d","query":%q,"start":%d,"end":%d,"step":%d}`, d.Seconds(), aqe.qid, aqe.quotedRemoteAddr, aqe.accountID, aqe.projectID, aqe.q, aqe.start, aqe.end, aqe.step) + if i+1 < len(aqes) { + fmt.Fprintf(w, `,`) + } } + fmt.Fprintf(w, `]}`) } var activeQueriesV = newActiveQueries() diff --git a/app/vmui/packages/vmui/src/App.tsx b/app/vmui/packages/vmui/src/App.tsx index aa018e7d3..f5b736381 100644 --- a/app/vmui/packages/vmui/src/App.tsx +++ b/app/vmui/packages/vmui/src/App.tsx @@ -14,6 +14,7 @@ import PreviewIcons from "./components/Main/Icons/PreviewIcons"; import WithTemplate from "./pages/WithTemplate"; import Relabel from "./pages/Relabel"; import ExploreLogs from "./pages/ExploreLogs/ExploreLogs"; +import ActiveQueries from "./pages/ActiveQueries"; const App: FC = () => { const { REACT_APP_LOGS } = process.env; @@ -65,6 +66,10 @@ const App: FC = () => { path={router.relabel} element={} /> + } + /> } diff --git a/app/vmui/packages/vmui/src/api/active-queries.ts b/app/vmui/packages/vmui/src/api/active-queries.ts new file mode 100644 index 000000000..f3e269b75 --- /dev/null +++ b/app/vmui/packages/vmui/src/api/active-queries.ts @@ -0,0 +1,2 @@ +export const getActiveQueries = (server: string): string => + `${server}/api/v1/status/active_queries`; diff --git a/app/vmui/packages/vmui/src/constants/navigation.ts b/app/vmui/packages/vmui/src/constants/navigation.ts index 9bfe6b557..8639c5e69 100644 --- a/app/vmui/packages/vmui/src/constants/navigation.ts +++ b/app/vmui/packages/vmui/src/constants/navigation.ts @@ -34,6 +34,10 @@ export const defaultNavigation: NavigationItem[] = [ label: routerOptions[router.topQueries].title, value: router.topQueries, }, + { + label: routerOptions[router.activeQueries].title, + value: router.activeQueries, + }, { label: routerOptions[router.logs].title, value: router.logs, diff --git a/app/vmui/packages/vmui/src/pages/ActiveQueries/hooks/useFetchActiveQueries.ts b/app/vmui/packages/vmui/src/pages/ActiveQueries/hooks/useFetchActiveQueries.ts new file mode 100644 index 000000000..10e5fe97f --- /dev/null +++ b/app/vmui/packages/vmui/src/pages/ActiveQueries/hooks/useFetchActiveQueries.ts @@ -0,0 +1,51 @@ +import { useEffect, useMemo, useState } from "preact/compat"; +import { getActiveQueries } from "../../../api/active-queries"; +import { useAppState } from "../../../state/common/StateContext"; +import { ActiveQueriesType, ErrorTypes } from "../../../types"; +import dayjs from "dayjs"; +import { DATE_FULL_TIMEZONE_FORMAT } from "../../../constants/date"; + +interface FetchActiveQueries { + data: ActiveQueriesType[]; + isLoading: boolean; + lastUpdated: string; + error?: ErrorTypes | string; + fetchData: () => Promise; +} + +export const useFetchActiveQueries = (): FetchActiveQueries => { + const { serverUrl } = useAppState(); + + const [activeQueries, setActiveQueries] = useState([]); + const [lastUpdated, setLastUpdated] = useState(dayjs().format(DATE_FULL_TIMEZONE_FORMAT)); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(); + + const fetchUrl = useMemo(() => getActiveQueries(serverUrl), [serverUrl]); + + const fetchData = async () => { + setIsLoading(true); + try { + const response = await fetch(fetchUrl); + const resp = await response.json(); + setActiveQueries(resp.data); + setLastUpdated(dayjs().format("HH:mm:ss:SSS")); + if (response.ok) { + setError(undefined); + } else { + setError(`${resp.errorType}\r\n${resp?.error}`); + } + } catch (e) { + if (e instanceof Error) { + setError(`${e.name}: ${e.message}`); + } + } + setIsLoading(false); + }; + + useEffect(() => { + fetchData().catch(console.error); + }, [fetchUrl]); + + return { data: activeQueries, lastUpdated, isLoading, error, fetchData }; +}; diff --git a/app/vmui/packages/vmui/src/pages/ActiveQueries/index.tsx b/app/vmui/packages/vmui/src/pages/ActiveQueries/index.tsx new file mode 100644 index 000000000..fc6ebed81 --- /dev/null +++ b/app/vmui/packages/vmui/src/pages/ActiveQueries/index.tsx @@ -0,0 +1,89 @@ +import React, { FC, useMemo } from "preact/compat"; +import { useFetchActiveQueries } from "./hooks/useFetchActiveQueries"; +import Alert from "../../components/Main/Alert/Alert"; +import Spinner from "../../components/Main/Spinner/Spinner"; +import Table from "../../components/Table/Table"; +import { ActiveQueriesType } from "../../types"; +import dayjs from "dayjs"; +import { useTimeState } from "../../state/time/TimeStateContext"; +import useDeviceDetect from "../../hooks/useDeviceDetect"; +import classNames from "classnames"; +import Button from "../../components/Main/Button/Button"; +import { RefreshIcon } from "../../components/Main/Icons"; +import "./style.scss"; + +const ActiveQueries: FC = () => { + const { isMobile } = useDeviceDetect(); + const { timezone } = useTimeState(); + + const { data, lastUpdated, isLoading, error, fetchData } = useFetchActiveQueries(); + + const activeQueries = useMemo(() => data.map(({ duration, ...item }: ActiveQueriesType) => ({ + duration, + data: JSON.stringify(item, null, 2), + from: dayjs(item.start).tz().format("MMM DD, YYYY \nHH:mm:ss.SSS"), + to: dayjs(item.end).tz().format("MMM DD, YYYY \nHH:mm:ss.SSS"), + ...item, + })), [data, timezone]); + + const columns = useMemo(() => { + if (!activeQueries?.length) return []; + const hideColumns = ["end", "start", "data"]; + const keys = new Set(); + for (const item of activeQueries) { + for (const key in item) { + keys.add(key); + } + } + return Array.from(keys) + .filter((col) => !hideColumns.includes(col)) + .map((key) => ({ + key: key as keyof ActiveQueriesType, + title: key, + })); + }, [activeQueries]); + + const handleRefresh = async () => { + fetchData().catch(console.error); + }; + + return ( +
+ {isLoading && } +
+ {!activeQueries.length && !error && There are currently no active queries running} + {error && {error}} +
+ +
+ Last updated: {lastUpdated} +
+
+
+ {!!activeQueries.length && ( +
+ + + )} + + ); +}; + +export default ActiveQueries; diff --git a/app/vmui/packages/vmui/src/pages/ActiveQueries/style.scss b/app/vmui/packages/vmui/src/pages/ActiveQueries/style.scss new file mode 100644 index 000000000..6693128dc --- /dev/null +++ b/app/vmui/packages/vmui/src/pages/ActiveQueries/style.scss @@ -0,0 +1,25 @@ +@use "src/styles/variables" as *; + +.vm-active-queries { + + &-header { + display: grid; + grid-template-columns: 1fr auto; + align-items: center; + justify-content: space-between; + gap: $padding-global; + margin-bottom: $padding-global; + + &-controls { + grid-column: 2; + display: grid; + gap: $padding-small; + } + + &__update-msg { + white-space: nowrap; + color: $color-text-secondary; + font-size: $font-size-small; + } + } +} diff --git a/app/vmui/packages/vmui/src/router/index.ts b/app/vmui/packages/vmui/src/router/index.ts index 4e6446123..4af8accd7 100644 --- a/app/vmui/packages/vmui/src/router/index.ts +++ b/app/vmui/packages/vmui/src/router/index.ts @@ -8,6 +8,7 @@ const router = { withTemplate: "/expand-with-exprs", relabel: "/relabeling", logs: "/logs", + activeQueries: "/active-queries", icons: "/icons" }; @@ -82,6 +83,10 @@ export const routerOptions: {[key: string]: RouterOptions} = { title: "Logs Explorer", header: {} }, + [router.activeQueries]: { + title: "Active Queries", + header: {} + }, [router.icons]: { title: "Icons", header: {} diff --git a/app/vmui/packages/vmui/src/types/index.ts b/app/vmui/packages/vmui/src/types/index.ts index 8df6df748..c199808a9 100644 --- a/app/vmui/packages/vmui/src/types/index.ts +++ b/app/vmui/packages/vmui/src/types/index.ts @@ -138,3 +138,16 @@ export interface RelabelData { resultingLabels?: string; steps: RelabelStep[]; } + +export interface ActiveQueriesType { + duration: string; + end: number; + start: number; + id: string; + query: string; + remote_addr: string; + step: number; + from?: string; + to?: string; + data?: string; +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c6622d0aa..30b1e496c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -54,6 +54,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components `-search.maxGraphiteTagValues` for limiting the number of tag values returned from [Graphite API for tag values](https://docs.victoriametrics.com/#graphite-tags-api-usage) `-search.maxGraphiteSeries` for limiting the number of series (aka paths) returned from [Graphite API for series](https://docs.victoriametrics.com/#graphite-tags-api-usage) See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4339). +* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): added a new page to display a list of currently running queries. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4598). * BUGFIX: properly return series from [/api/v1/series](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#prometheus-querying-api-usage) if it finds more than the `limit` series (`limit` is an optional query arg passed to this API). Previously the `limit exceeded error` error was returned in this case. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2841#issuecomment-1560055631). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix application routing issues and problems with manual URL changes. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4408). diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index d1a9e4a9d..ca510a785 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -647,9 +647,9 @@ func TestParseQuerySuccess(t *testing.T) { f(`_time:(2023-01-05, 2023-01-06] OFFset 5m`, `_time:(2023-01-05,2023-01-06] offset 5m`) f(`_time:(2023-01-05, 2023-01-06) OFFset 5m`, `_time:(2023-01-05,2023-01-06) offset 5m`) f(`_time:1h offset 5m`, `_time:1h offset 5m`) - f(`_time:1h "offSet"`, `_time:1h "offSet"`) // "offset" is a search word, since it is quoted + f(`_time:1h "offSet"`, `_time:1h "offSet"`) // "offset" is a search word, since it is quoted f(`_time:1h (Offset)`, `_time:1h "Offset"`) // "offset" is a search word, since it is in parens - f(`_time:1h "and"`, `_time:1h "and"`) // "and" is a search word, since it is quoted + f(`_time:1h "and"`, `_time:1h "and"`) // "and" is a search word, since it is quoted // reserved keywords f("and", `"and"`)