app/vmui: define custom path for dashboards json file (#3545)

* app/vmui: define custom path for dashboards json file

* app/vmui: remove unneeded code

* app/vmui: move handler to own file, fix show dashboards,

* app/vmui: move flag to handler, add flag description

* app/vmauth: fix part of the comments

* feat: add store for dashboards

* fix: prevent fetch dashboards for app mode

* app/vmauth: use simple cache for predefined dashboards

* app/vmauth: update dashboards doc

* app/vmauth: fix ci

* app/vmui: decrease timeout

* app/vmselect: removed cache, fix comments

* app/vmselect: remove unused const

* app/vmselect: fix error log, use slice byte instead of struct

Co-authored-by: Yury Moladau <yurymolodov@gmail.com>

README.md

@@ -2503,4 +2503,6 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Show VictoriaMetrics version
   -vmalert.proxyURL string
      Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules
+  -vmui.customDashboardsPath string
+     Optional path to vmui predefined dashboards.   
 ```

app/vmselect/main.go

@@ -161,8 +161,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case strings.HasPrefix(path, "/graphite/"):
 		path = path[len("/graphite"):]
 	}
+
 	// vmui access.
-	if path == "/vmui" || path == "/graph" {
+	switch {
+	case path == "/vmui" || path == "/graph":
 		// VMUI access via incomplete url without `/` in the end. Redirect to complete url.
 		// Use relative redirect, since, since the hostname and path prefix may be incorrect if VictoriaMetrics
 		// is hidden behind vmauth or similar proxy.
@@ -171,18 +173,21 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		newURL := path + "/?" + r.Form.Encode()
 		httpserver.Redirect(w, newURL)
 		return true
-	}
-	if strings.HasPrefix(path, "/vmui/") {
+	case strings.HasPrefix(path, "/vmui/"):
+		if path == "/vmui/custom-dashboards" {
+			handleVMUICustomDashboards(w)
+			return true
+		}
 		r.URL.Path = path
 		vmuiFileServer.ServeHTTP(w, r)
 		return true
-	}
-	if strings.HasPrefix(path, "/graph/") {
+	case strings.HasPrefix(path, "/graph/"):
 		// This is needed for serving /graph URLs from Prometheus datasource in Grafana.
 		r.URL.Path = strings.Replace(path, "/graph/", "/vmui/", 1)
 		vmuiFileServer.ServeHTTP(w, r)
 		return true
 	}
+
 	if strings.HasPrefix(path, "/api/v1/label/") {
 		s := path[len("/api/v1/label/"):]
 		if strings.HasSuffix(s, "/values") {

app/vmselect/vmui.go

@@ -0,0 +1,128 @@
+package vmselect
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// more information how to use this flag please check this link
+// https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui/packages/vmui/public/dashboards
+var (
+	vmuiCustomDashboardsPath = flag.String("vmui.customDashboardsPath", "", "Optional path to vmui predefined dashboards."+
+		"How to create dashboards https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui/packages/vmui/public/dashboards")
+)
+
+// dashboardSetting represents dashboard settings file struct
+// fields of the dashboardSetting you can find by following next link
+// https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui/packages/vmui/public/dashboards
+type dashboardSetting struct {
+	Title    string         `json:"title,omitempty"`
+	Filename string         `json:"filename,omitempty"`
+	Rows     []dashboardRow `json:"rows"`
+}
+
+// panelSettings represents fields which used to show graph
+// fields of the panelSettings you can find by following next link
+// https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui/packages/vmui/public/dashboards
+type panelSettings struct {
+	Title       string   `json:"title,omitempty"`
+	Description string   `json:"description,omitempty"`
+	Unit        string   `json:"unit,omitempty"`
+	Expr        []string `json:"expr"`
+	Alias       []string `json:"alias,omitempty"`
+	ShowLegend  bool     `json:"showLegend,omitempty"`
+	Width       int      `json:"width,omitempty"`
+}
+
+// dashboardRow represents panels on dashboard
+// fields of the dashboardRow you can find by following next link
+// https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui/packages/vmui/public/dashboards
+type dashboardRow struct {
+	Title  string          `json:"title,omitempty"`
+	Panels []panelSettings `json:"panels"`
+}
+
+// dashboardsData represents all dashboards settings
+type dashboardsData struct {
+	DashboardsSettings []dashboardSetting `json:"dashboardsSettings"`
+}
+
+func handleVMUICustomDashboards(w http.ResponseWriter) {
+	path := *vmuiCustomDashboardsPath
+	if path == "" {
+		writeSuccessResponse(w, []byte(`{"dashboardsSettings": []}`))
+		return
+	}
+
+	settings, err := collectDashboardsSettings(path)
+	if err != nil {
+		writeErrorResponse(w, fmt.Errorf("cannot collect dashboards settings by -vmui.customDashboardsPath=%q", path))
+		return
+	}
+
+	writeSuccessResponse(w, settings)
+}
+
+func writeErrorResponse(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusBadRequest)
+	w.Header().Set("Content-Type", "application/json")
+	fmt.Fprintf(w, `{"status":"error","error":"%s"}`, err.Error())
+}
+
+func writeSuccessResponse(w http.ResponseWriter, data []byte) {
+	w.WriteHeader(http.StatusOK)
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(data)
+}
+
+func collectDashboardsSettings(path string) ([]byte, error) {
+
+	if !fs.IsPathExist(path) {
+		return nil, fmt.Errorf("cannot find folder pointed by -vmui.customDashboardsPath=%q", path)
+	}
+
+	files, err := os.ReadDir(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read folder pointed by -vmui.customDashboardsPath=%q", path)
+	}
+
+	var settings []dashboardSetting
+	for _, file := range files {
+		info, err := file.Info()
+		if err != nil {
+			logger.Errorf("skipping %q at -vmui.customDashboardsPath=%q, since the info for this file cannot be obtained: %s", file.Name(), path, err)
+			continue
+		}
+		if fs.IsDirOrSymlink(info) {
+			logger.Infof("skip directory or symlinks: %q in the -vmui.customDashboardsPath=%q", info.Name(), path)
+			continue
+		}
+		filename := file.Name()
+		if filepath.Ext(filename) == ".json" {
+			filePath := filepath.Join(path, filename)
+			f, err := os.ReadFile(filePath)
+			if err != nil {
+				return nil, fmt.Errorf("cannot open file at -vmui.customDashboardsPath=%q: %w", filePath, err)
+			}
+			var dSettings dashboardSetting
+			err = json.Unmarshal(f, &dSettings)
+			if err != nil {
+				return nil, fmt.Errorf("cannot parse file %s: %w", filename, err)
+			}
+			if len(dSettings.Rows) == 0 {
+				continue
+			}
+			settings = append(settings, dSettings)
+		}
+	}
+
+	dd := dashboardsData{DashboardsSettings: settings}
+	return json.Marshal(dd)
+}

app/vmui/packages/vmui/public/dashboards/README.md

@@ -3,6 +3,24 @@
 2. Import your config file into the `dashboards/index.js`
 3. Add filename into the array `window.__VMUI_PREDEFINED_DASHBOARDS__`
 
+It is possible to define path to the predefined dashboards by setting `--vmui.customDashboardsPath`.
+
+1. Single Version
+If you use single version of the VictoriaMetrics this flag should be provided for you execution file.
+```
+./victoria-metrics  --vmui.customDashboardsPath=/path/to/your/dashboards
+```
+
+2. Cluster Version
+If you use cluster version this flag should be defined for each `vmselect` component.
+```
+./vmselect -storageNode=:8418 --vmui.customDashboardsPath=/path/to/your/dashboards
+```
+At that moment all predefined dashboards files show be near each  `vmselect`. For example 
+if you have 3 `vmselect` instances you should create 3 copy of your predefined dashboards.    
+
+
+
 ### Configuration options
 
 <br/>

app/vmui/packages/vmui/src/components/Layout/Header/Header.tsx

@@ -14,10 +14,12 @@ import { getCssVariable } from "../../../utils/theme";
 import Tabs from "../../Main/Tabs/Tabs";
 import "./style.scss";
 import classNames from "classnames";
+import { useDashboardsState } from "../../../state/dashboards/DashboardsStateContext";
 
 const Header: FC = () => {
   const primaryColor = getCssVariable("color-primary");
   const appModeEnable = getAppModeEnable();
+  const { dashboardsSettings } = useDashboardsState();
 
   const { headerStyles: {
     background = appModeEnable ? "#FFF" : primaryColor,
@@ -50,9 +52,9 @@ const Header: FC = () => {
     {
       label: routerOptions[router.dashboards].title,
       value: router.dashboards,
-      hide: appModeEnable
+      hide: appModeEnable || !dashboardsSettings.length
     }
-  ]), [appModeEnable]);
+  ]), [appModeEnable, dashboardsSettings]);
 
   const [activeMenu, setActiveMenu] = useState(pathname);
 

app/vmui/packages/vmui/src/components/Layout/Layout.tsx

@@ -6,9 +6,11 @@ import { getAppModeEnable } from "../../utils/app-mode";
 import classNames from "classnames";
 import Footer from "./Footer/Footer";
 import { routerOptions } from "../../router";
+import { useFetchDashboards } from "../../pages/PredefinedPanels/hooks/useFetchDashboards";
 
 const Layout: FC = () => {
   const appModeEnable = getAppModeEnable();
+  useFetchDashboards();
 
   const { pathname } = useLocation();
   useEffect(() => {

app/vmui/packages/vmui/src/contexts/AppContextProvider.tsx

@@ -8,6 +8,7 @@ import { TopQueriesStateProvider } from "../state/topQueries/TopQueriesStateCont
 import { SnackbarProvider } from "./Snackbar";
 
 import { combineComponents } from "../utils/combine-components";
+import { DashboardsStateProvider } from "../state/dashboards/DashboardsStateContext";
 
 const providers = [
   AppStateProvider,
@@ -17,7 +18,8 @@ const providers = [
   GraphStateProvider,
   CardinalityStateProvider,
   TopQueriesStateProvider,
-  SnackbarProvider
+  SnackbarProvider,
+  DashboardsStateProvider
 ];
 
 export default combineComponents(...providers);

app/vmui/packages/vmui/src/pages/PredefinedPanels/getDashboardSettings.ts

@@ -1,12 +0,0 @@
-import { DashboardSettings } from "../../types";
-
-const importModule = async (filename: string) => {
-  const data = await fetch(`./dashboards/${filename}`);
-  const json = await data.json();
-  return json as DashboardSettings;
-};
-
-export default async () => {
-  const filenames = window.__VMUI_PREDEFINED_DASHBOARDS__;
-  return await Promise.all(filenames.map(async f => importModule(f)));
-};

app/vmui/packages/vmui/src/pages/PredefinedPanels/hooks/useFetchDashboards.ts

@@ -0,0 +1,78 @@
+import { useEffect, useState } from "preact/compat";
+import { DashboardSettings, ErrorTypes } from "../../../types";
+import { useAppState } from "../../../state/common/StateContext";
+import { useDashboardsDispatch } from "../../../state/dashboards/DashboardsStateContext";
+import { getAppModeEnable } from "../../../utils/app-mode";
+
+const importModule = async (filename: string) => {
+  const data = await fetch(`./dashboards/${filename}`);
+  const json = await data.json();
+  return json as DashboardSettings;
+};
+
+export const useFetchDashboards = (): {
+  isLoading: boolean,
+  error?: ErrorTypes | string,
+  dashboardsSettings: DashboardSettings[],
+} => {
+
+  const appModeEnable = getAppModeEnable();
+  const { serverUrl } = useAppState();
+  const dispatch = useDashboardsDispatch();
+  const [isLoading, setIsLoading] = useState(false);
+  const [error, setError] = useState<ErrorTypes | string>("");
+  const [dashboardsSettings, setDashboards] = useState<DashboardSettings[]>([]);
+
+  const fetchLocalDashboards = async () => {
+    const filenames = window.__VMUI_PREDEFINED_DASHBOARDS__;
+    if (!filenames?.length) return [];
+    return await Promise.all(filenames.map(async f => importModule(f)));
+  };
+
+  const fetchRemoteDashboards = async () => {
+    if (!serverUrl) return;
+    setError("");
+    setIsLoading(true);
+
+    try {
+      const response = await fetch(`${serverUrl}/vmui/custom-dashboards`);
+      const resp = await response.json();
+
+      if (response.ok) {
+        const { dashboardsSettings } = resp;
+        if (dashboardsSettings && dashboardsSettings.length > 0) {
+          setDashboards((prevDash) => [...prevDash, ...dashboardsSettings]);
+        }
+        setIsLoading(false);
+      } else {
+        setError(resp.error);
+        setIsLoading(false);
+      }
+    } catch (e) {
+      setIsLoading(false);
+      if (e instanceof Error) setError(`${e.name}: ${e.message}`);
+    }
+  };
+
+  useEffect(() => {
+    if (appModeEnable) return;
+    setDashboards([]);
+    fetchLocalDashboards().then(d => d.length && setDashboards((prevDash) => [...d, ...prevDash]));
+    fetchRemoteDashboards();
+  }, [serverUrl]);
+
+  useEffect(() => {
+    dispatch({ type: "SET_DASHBOARDS_SETTINGS", payload: dashboardsSettings });
+  }, [dashboardsSettings]);
+
+  useEffect(() => {
+    dispatch({ type: "SET_DASHBOARDS_LOADING", payload: isLoading });
+  }, [isLoading]);
+
+  useEffect(() => {
+    dispatch({ type: "SET_DASHBOARDS_ERROR", payload: error });
+  }, [error]);
+
+  return { dashboardsSettings, isLoading, error };
+};
+

app/vmui/packages/vmui/src/pages/PredefinedPanels/index.tsx

@@ -1,60 +1,67 @@
-import React, { FC, useEffect, useMemo, useState } from "preact/compat";
-import getDashboardSettings from "./getDashboardSettings";
-import { DashboardSettings } from "../../types";
+import React, { FC, useMemo, useState } from "preact/compat";
 import PredefinedDashboard from "./PredefinedDashboard/PredefinedDashboard";
 import { useSetQueryParams } from "./hooks/useSetQueryParams";
-import Tabs from "../../components/Main/Tabs/Tabs";
 import Alert from "../../components/Main/Alert/Alert";
+import classNames from "classnames";
 import "./style.scss";
+import { useDashboardsState } from "../../state/dashboards/DashboardsStateContext";
+import Spinner from "../../components/Main/Spinner/Spinner";
 
-const Index: FC = () => {
+const DashboardsLayout: FC = () => {
   useSetQueryParams();
+  const { dashboardsSettings, dashboardsLoading, dashboardsError } = useDashboardsState();
+  const [dashboard, setDashboard] = useState(0);
 
-  const [dashboards, setDashboards] = useState<DashboardSettings[]>([]);
-  const [tab, setTab] = useState("0");
-
-  const tabs = useMemo(() => dashboards.map((d, i) => ({
+  const dashboards = useMemo(() => dashboardsSettings.map((d, i) => ({
     label: d.title || "",
-    value: `${i}`,
-    className: "vm-predefined-panels-tabs__tab"
-  })), [dashboards]);
+    value: i,
+  })), [dashboardsSettings]);
 
-  const activeDashboard = useMemo(() => dashboards[+tab] || {}, [dashboards, tab]);
+  const activeDashboard = useMemo(() => dashboardsSettings[dashboard] || {}, [dashboardsSettings, dashboard]);
   const rows = useMemo(() => activeDashboard?.rows, [activeDashboard]);
   const filename = useMemo(() => activeDashboard.title || activeDashboard.filename || "", [activeDashboard]);
   const validDashboardRows = useMemo(() => Array.isArray(rows) && !!rows.length, [rows]);
 
-  const handleChangeTab = (value: string) => {
-    setTab(value);
+  const handleChangeDashboard = (value: number) => {
+    setDashboard(value);
   };
 
-  useEffect(() => {
-    getDashboardSettings().then(d => d.length && setDashboards(d));
-  }, []);
+  const createHandlerSelectDashboard = (value: number) => () => {
+    handleChangeDashboard(value);
+  };
 
   return <div className="vm-predefined-panels">
-    {!dashboards.length && <Alert variant="info">Dashboards not found</Alert>}
-    {tabs.length > 1 && (
-      <div className="vm-predefined-panels-tabs vm-block vm-block_empty-padding">
-        <Tabs
-          activeItem={tab}
-          items={tabs}
-          onChange={handleChangeTab}
-        />
+    {dashboardsLoading && <Spinner />}
+    {dashboardsError && <Alert variant="error">{dashboardsError}</Alert>}
+    {!dashboardsSettings.length && <Alert variant="info">Dashboards not found</Alert>}
+    {dashboards.length > 1 && (
+      <div className="vm-predefined-panels-tabs vm-block">
+        {dashboards.map(tab => (
+          <div
+            key={tab.value}
+            className={classNames({
+              "vm-predefined-panels-tabs__tab": true,
+              "vm-predefined-panels-tabs__tab_active": tab.value == dashboard
+            })}
+            onClick={createHandlerSelectDashboard(tab.value)}
+          >
+            {tab.label}
+          </div>
+        ))}
       </div>
     )}
     <div className="vm-predefined-panels__dashboards">
       {validDashboardRows && (
         rows.map((r,i) =>
           <PredefinedDashboard
-            key={`${tab}_${i}`}
+            key={`${dashboard}_${i}`}
             index={i}
             filename={filename}
             title={r.title}
             panels={r.panels}
           />)
       )}
-      {!!dashboards.length && !validDashboardRows && (
+      {!!dashboardsSettings.length && !validDashboardRows && (
         <Alert variant="error">
           <code>&quot;rows&quot;</code> not found. Check the configuration file <b>{filename}</b>.
         </Alert>
@@ -63,4 +70,4 @@ const Index: FC = () => {
   </div>;
 };
 
-export default Index;
+export default DashboardsLayout;

app/vmui/packages/vmui/src/pages/PredefinedPanels/style.scss

@@ -5,22 +5,37 @@
   gap: $padding-global;
   align-items: flex-start;
 
+  &-tabs.vm-block {
+    padding: $padding-global;
+  }
+
   &-tabs {
     display: flex;
+    flex-wrap: wrap;
     align-items: center;
     justify-content: flex-start;
     font-size: $font-size-small;
+    gap: $padding-small;
+    white-space: nowrap;
     overflow: hidden;
 
     &__tab {
-      padding: $padding-global;
+      padding: $padding-small $padding-global;
+      border-radius: $border-radius-medium;
       cursor: pointer;
-      transition: opacity 200ms ease-in-out, color 150ms ease-in;
-      border-right: $border-divider;
+      transition: background 200ms ease-in-out, color 150ms ease-in;
+      background: $color-white;
       text-transform: uppercase;
+      color: rgba($color-black, 0.2);
+      border: 1px solid rgba($color-black, 0.2);
 
       &:hover {
-        opacity: 1;
+        color: $color-primary;
+      }
+
+      &_active {
+        border-color: $color-primary;
+        color: $color-primary;
       }
     }
   }

app/vmui/packages/vmui/src/state/dashboards/DashboardsStateContext.tsx

@@ -0,0 +1,24 @@
+import React, { createContext, FC, useContext, useMemo, useReducer } from "preact/compat";
+import { DashboardsAction, DashboardsState, initialDashboardsState, reducer } from "./reducer";
+
+import { Dispatch } from "react";
+
+type DashboardsStateContextType = { state: DashboardsState, dispatch: Dispatch<DashboardsAction> };
+
+export const DashboardsStateContext = createContext<DashboardsStateContextType>({} as DashboardsStateContextType);
+
+export const useDashboardsState = (): DashboardsState => useContext(DashboardsStateContext).state;
+export const useDashboardsDispatch = (): Dispatch<DashboardsAction> => useContext(DashboardsStateContext).dispatch;
+export const DashboardsStateProvider: FC = ({ children }) => {
+  const [state, dispatch] = useReducer(reducer, initialDashboardsState);
+
+  const contextValue = useMemo(() => {
+    return { state, dispatch };
+  }, [state, dispatch]);
+
+  return <DashboardsStateContext.Provider value={contextValue}>
+    {children}
+  </DashboardsStateContext.Provider>;
+};
+
+

app/vmui/packages/vmui/src/state/dashboards/reducer.ts

@@ -0,0 +1,41 @@
+import { DashboardSettings } from "../../types";
+
+export interface DashboardsState {
+  dashboardsSettings: DashboardSettings[];
+  dashboardsLoading: boolean,
+  dashboardsError: string
+}
+
+export type DashboardsAction =
+  | { type: "SET_DASHBOARDS_SETTINGS", payload: DashboardSettings[] }
+  | { type: "SET_DASHBOARDS_LOADING", payload: boolean }
+  | { type: "SET_DASHBOARDS_ERROR", payload: string }
+
+
+export const initialDashboardsState: DashboardsState = {
+  dashboardsSettings: [],
+  dashboardsLoading: false,
+  dashboardsError: "",
+};
+
+export function reducer(state: DashboardsState, action: DashboardsAction): DashboardsState {
+  switch (action.type) {
+    case "SET_DASHBOARDS_SETTINGS":
+      return {
+        ...state,
+        dashboardsSettings: action.payload
+      };
+    case "SET_DASHBOARDS_LOADING":
+      return {
+        ...state,
+        dashboardsLoading: action.payload
+      };
+    case "SET_DASHBOARDS_ERROR":
+      return {
+        ...state,
+        dashboardsError: action.payload
+      };
+    default:
+      throw new Error();
+  }
+}

docs/CHANGELOG.md

@@ -54,6 +54,7 @@ Released at 2023-01-10
   - `vm_vmselect_concurrent_requests_current` - the current number of concurrently executed requests
   - `vm_vmselect_concurrent_requests_limit_reached_total` - the total number of requests, which were put in the wait queue when `-search.maxConcurrentRequests` concurrent requests are being executed
   - `vm_vmselect_concurrent_requests_limit_timeout_total` - the total number of canceled requests because they were sitting in the wait queue for more than `-search.maxQueueDuration`
+* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add ability to define path to custom dashboards via `vmui.customDashboardsPath` flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3322).
 
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly update the `step` value in url after the `step` input field has been manually changed. This allows preserving the proper `step` when copy-n-pasting the url to another instance of web browser. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3513).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly update tooltip when quickly hovering multiple lines on the graph. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3530).