vmui: add tab for displaying raw data

This commit is contained in:
Yury Molodov 2024-11-04 14:16:29 +01:00
parent 4e50d6eed3
commit 449e2ead43
No known key found for this signature in database
GPG key ID: 79AD43149C47BDE7
18 changed files with 392 additions and 18 deletions

View file

@ -17,6 +17,7 @@ import ActiveQueries from "./pages/ActiveQueries";
import QueryAnalyzer from "./pages/QueryAnalyzer";
import DownsamplingFilters from "./pages/DownsamplingFilters";
import RetentionFilters from "./pages/RetentionFilters";
import RawQueryPage from "./pages/RawQueryPage";
const App: FC = () => {
const [loadedTheme, setLoadedTheme] = useState(false);
@ -36,6 +37,10 @@ const App: FC = () => {
path={router.home}
element={<CustomPanel/>}
/>
<Route
path={router.rawQuery}
element={<RawQueryPage/>}
/>
<Route
path={router.metrics}
element={<ExploreMetrics/>}

View file

@ -5,3 +5,13 @@ export const getQueryRangeUrl = (server: string, query: string, period: TimePara
export const getQueryUrl = (server: string, query: string, period: TimeParams, nocache: boolean, queryTracing: boolean): string =>
`${server}/api/v1/query?query=${encodeURIComponent(query)}&time=${period.end}&step=${period.step}${nocache ? "&nocache=1" : ""}${queryTracing ? "&trace=1" : ""}`;
export const getExportDataUrl = (server: string, query: string, period: TimeParams, reduceMemUsage: boolean): string => {
const params = new URLSearchParams({
"match[]": query,
start: period.start.toString(),
end: period.end.toString(),
});
if (reduceMemUsage) params.set("reduce_mem_usage", "1");
return `${server}/api/v1/export?${params}`;
};

View file

@ -15,6 +15,11 @@ export interface InstantMetricResult extends MetricBase {
values?: [number, string][]
}
export interface RawMetricResult extends MetricBase {
values: number[];
timestamps: number[];
}
export interface TracingData {
message: string;
duration_msec: number;

View file

@ -20,13 +20,17 @@ const AdditionalSettingsControls: FC<Props & {isMobile?: boolean}> = ({ isMobile
const { autocomplete } = useQueryState();
const queryDispatch = useQueryDispatch();
const { nocache, isTracingEnabled } = useCustomPanelState();
const { nocache, isTracingEnabled, reduceMemUsage } = useCustomPanelState();
const customPanelDispatch = useCustomPanelDispatch();
const onChangeCache = () => {
customPanelDispatch({ type: "TOGGLE_NO_CACHE" });
};
const onChangeReduceMemUsage = () => {
customPanelDispatch({ type: "TOGGLE_REDUCE_MEM_USAGE" });
};
const onChangeQueryTracing = () => {
customPanelDispatch({ type: "TOGGLE_QUERY_TRACING" });
};
@ -67,12 +71,22 @@ const AdditionalSettingsControls: FC<Props & {isMobile?: boolean}> = ({ isMobile
/>
</Tooltip>
)}
<Switch
label={"Disable cache"}
value={nocache}
onChange={onChangeCache}
fullWidth={isMobile}
/>
{!hideButtons?.disableCache && (
<Switch
label={"Disable cache"}
value={nocache}
onChange={onChangeCache}
fullWidth={isMobile}
/>
)}
{!hideButtons?.reduceMemUsage && (
<Switch
label={"Disable deduplication feature"}
value={reduceMemUsage}
onChange={onChangeReduceMemUsage}
fullWidth={isMobile}
/>
)}
{!hideButtons?.traceQuery && (
<Switch
label={"Trace query"}

View file

@ -23,6 +23,7 @@ export interface QueryEditorProps {
stats?: QueryStats;
label: string;
disabled?: boolean
includeFunctions?: boolean;
}
const QueryEditor: FC<QueryEditorProps> = ({
@ -35,7 +36,8 @@ const QueryEditor: FC<QueryEditorProps> = ({
error,
stats,
label,
disabled = false
disabled = false,
includeFunctions = true
}) => {
const { autocompleteQuick } = useQueryState();
const { isMobile } = useDeviceDetect();
@ -143,6 +145,7 @@ const QueryEditor: FC<QueryEditorProps> = ({
anchorEl={autocompleteAnchorEl}
caretPosition={caretPosition}
hasHelperText={Boolean(warning || error)}
includeFunctions={includeFunctions}
onSelect={handleSelect}
onFoundOptions={handleChangeFoundOptions}
/>

View file

@ -11,6 +11,7 @@ interface QueryEditorAutocompleteProps {
anchorEl: React.RefObject<HTMLElement>;
caretPosition: [number, number]; // [start, end]
hasHelperText: boolean;
includeFunctions: boolean;
onSelect: (val: string, caretPosition: number) => void;
onFoundOptions: (val: AutocompleteOptions[]) => void;
}
@ -20,11 +21,12 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
anchorEl,
caretPosition,
hasHelperText,
includeFunctions,
onSelect,
onFoundOptions
}) => {
const [offsetPos, setOffsetPos] = useState({ top: 0, left: 0 });
const metricsqlFunctions = useGetMetricsQL();
const metricsqlFunctions = useGetMetricsQL(includeFunctions);
const values = useMemo(() => {
if (caretPosition[0] !== caretPosition[1]) return { beforeCursor: value, afterCursor: "" };

View file

@ -48,7 +48,7 @@ const processGroups = (groups: NodeListOf<Element>): AutocompleteOptions[] => {
}).filter(Boolean) as AutocompleteOptions[];
};
const useGetMetricsQL = () => {
const useGetMetricsQL = (includeFunctions: boolean) => {
const { metricsQLFunctions } = useQueryState();
const queryDispatch = useQueryDispatch();
@ -60,6 +60,7 @@ const useGetMetricsQL = () => {
};
useEffect(() => {
if (!includeFunctions || metricsQLFunctions.length) return;
const fetchMarkdown = async () => {
try {
const resp = await fetch(MetricsQL);
@ -70,12 +71,10 @@ const useGetMetricsQL = () => {
console.error("Error fetching or processing the MetricsQL.md file:", e);
}
};
if (metricsQLFunctions.length) return;
fetchMarkdown();
}, []);
return metricsQLFunctions;
return includeFunctions ? metricsQLFunctions : [];
};
export default useGetMetricsQL;

View file

@ -17,7 +17,11 @@ export const displayTypeTabs: DisplayTab[] = [
{ value: DisplayType.table, icon: <TableIcon/>, label: "Table", prometheusCode: 1 }
];
export const DisplayTypeSwitch: FC = () => {
interface Props {
tabFilter?: (tab: DisplayTab) => boolean
}
export const DisplayTypeSwitch: FC<Props> = ({ tabFilter }) => {
const { displayType } = useCustomPanelState();
const dispatch = useCustomPanelDispatch();
@ -26,10 +30,12 @@ export const DisplayTypeSwitch: FC = () => {
dispatch({ type: "SET_DISPLAY_TYPE", payload: newValue as DisplayType ?? displayType });
};
const items = displayTypeTabs.filter(tabFilter ?? (() => true));
return (
<Tabs
activeItem={displayType}
items={displayTypeTabs}
items={items}
onChange={handleChange}
/>
);

View file

@ -31,7 +31,9 @@ export interface QueryConfiguratorProps {
setQueryErrors: Dispatch<SetStateAction<string[]>>;
setHideError: Dispatch<SetStateAction<boolean>>;
stats: QueryStats[];
label?: string;
isLoading?: boolean;
includeFunctions?: boolean;
onHideQuery?: (queries: number[]) => void
onRunQuery: () => void;
abortFetch?: () => void;
@ -41,6 +43,8 @@ export interface QueryConfiguratorProps {
autocomplete?: boolean;
traceQuery?: boolean;
anomalyConfig?: boolean;
disableCache?: boolean;
reduceMemUsage?: boolean;
}
}
@ -49,7 +53,9 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({
setQueryErrors,
setHideError,
stats,
label,
isLoading,
includeFunctions = true,
onHideQuery,
onRunQuery,
abortFetch,
@ -216,8 +222,9 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({
onArrowDown={createHandlerArrow(1, i)}
onEnter={handleRunQuery}
onChange={createHandlerChangeQuery(i)}
label={`Query ${stateQuery.length > 1 ? i + 1 : ""}`}
label={`${label || "Query"} ${stateQuery.length > 1 ? i + 1 : ""}`}
disabled={hideQuery.includes(i)}
includeFunctions={includeFunctions}
/>
{onHideQuery && (
<Tooltip title={hideQuery.includes(i) ? "Enable query" : "Disable query"}>

View file

@ -72,6 +72,16 @@ export const useSetQueryParams = () => {
newSearchParams.set(`${group}.tenantID`, tenantId);
}
});
// Remove extra parameters that exceed the request size
const maxIndex = query.length - 1;
Array.from(newSearchParams.keys()).forEach(key => {
const match = key.match(/^g(\d+)\./);
if (match && parseInt(match[1], 10) > maxIndex) {
newSearchParams.delete(key);
}
});
if (isEqualURLSearchParams(newSearchParams, searchParams) || !newSearchParams.size) return;
setSearchParams(newSearchParams);
}, [tenantId, displayType, query, duration, relativeTime, date, step, customStep]);

View file

@ -85,6 +85,7 @@ const CustomPanel: FC = () => {
onHideQuery={handleHideQuery}
onRunQuery={handleRunQuery}
abortFetch={abortFetch}
hideButtons={{ reduceMemUsage: true }}
/>
<CustomPanelTraces
traces={traces}

View file

@ -87,7 +87,14 @@ const ExploreAnomaly: FC = () => {
setHideError={setHideError}
stats={queryStats}
onRunQuery={handleRunQuery}
hideButtons={{ addQuery: true, prettify: false, autocomplete: false, traceQuery: true, anomalyConfig: true }}
hideButtons={{
addQuery: true,
prettify: false,
autocomplete: false,
traceQuery: true,
anomalyConfig: true,
reduceMemUsage: true,
}}
/>
{isLoading && <Spinner/>}
{(!hideError && error) && <Alert variant="error">{error}</Alert>}

View file

@ -0,0 +1,151 @@
import { Dispatch, SetStateAction, useCallback, useEffect, useMemo, useRef, useState } from "preact/compat";
import { MetricBase, MetricResult, RawMetricResult } from "../../../api/types";
import { ErrorTypes, SeriesLimits } from "../../../types";
import { useQueryState } from "../../../state/query/QueryStateContext";
import { useTimeState } from "../../../state/time/TimeStateContext";
import { useAppState } from "../../../state/common/StateContext";
import { useCustomPanelState } from "../../../state/customPanel/CustomPanelStateContext";
import { isValidHttpUrl } from "../../../utils/url";
import { getExportDataUrl } from "../../../api/query-range";
interface FetchQueryParams {
hideQuery?: number[];
showAllSeries?: boolean;
}
interface FetchQueryReturn {
fetchUrl?: string[],
isLoading: boolean,
data?: MetricResult[],
error?: ErrorTypes | string,
queryErrors: (ErrorTypes | string)[],
setQueryErrors: Dispatch<SetStateAction<string[]>>,
warning?: string,
abortFetch: () => void
}
const parseLineToJSON = (line: string): RawMetricResult | null => {
try {
return JSON.parse(line);
} catch (e) {
return null;
}
};
export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams): FetchQueryReturn => {
const { query } = useQueryState();
const { period } = useTimeState();
const { displayType, reduceMemUsage, seriesLimits: stateSeriesLimits } = useCustomPanelState();
const { serverUrl } = useAppState();
const [isLoading, setIsLoading] = useState(false);
const [data, setData] = useState<MetricResult[]>();
const [error, setError] = useState<ErrorTypes | string>();
const [queryErrors, setQueryErrors] = useState<string[]>([]);
const [warning, setWarning] = useState<string>();
const abortControllerRef = useRef(new AbortController());
const fetchUrl = useMemo(() => {
setError("");
setQueryErrors([]);
if (!period) return;
if (!serverUrl) {
setError(ErrorTypes.emptyServer);
} else if (query.every(q => !q.trim())) {
setQueryErrors(query.map(() => ErrorTypes.validQuery));
} else if (isValidHttpUrl(serverUrl)) {
const updatedPeriod = { ...period };
return query.map(q => getExportDataUrl(serverUrl, q, updatedPeriod, reduceMemUsage));
} else {
setError(ErrorTypes.validServer);
}
}, [serverUrl, period, hideQuery, reduceMemUsage]);
const fetchData = useCallback(async ( { fetchUrl, stateSeriesLimits, showAllSeries }: {
fetchUrl: string[];
stateSeriesLimits: SeriesLimits;
showAllSeries?: boolean;
}) => {
abortControllerRef.current.abort();
abortControllerRef.current = new AbortController();
const { signal } = abortControllerRef.current;
setIsLoading(true);
try {
const tempData: MetricBase[] = [];
const seriesLimit = showAllSeries ? Infinity : +stateSeriesLimits[displayType] || Infinity;
let counter = 1;
let totalLength = 0;
for await (const url of fetchUrl) {
const isHideQuery = hideQuery?.includes(counter - 1);
if (isHideQuery) {
setQueryErrors(prev => [...prev, ""]);
counter++;
continue;
}
const response = await fetch(url, { signal });
const text = await response.text();
if (!response.ok || !response.body) {
tempData.push({ metric: {}, values: [], group: counter } as MetricBase);
setError(text);
setQueryErrors(prev => [...prev, `${text}`]);
} else {
setQueryErrors(prev => [...prev, ""]);
const freeTempSize = seriesLimit - tempData.length;
const lines = text.split("\n").filter(line => line);
const linesLimited = lines.slice(0, freeTempSize);
const responseData = linesLimited.map(parseLineToJSON).filter(line => line) as RawMetricResult[];
const metricResult = responseData.map((d: RawMetricResult) => ({
group: counter,
metric: d.metric,
values: d.values.map((value, index) => [(d.timestamps[index]/1000), value]),
}));
tempData.push(...metricResult);
totalLength += lines.length;
}
counter++;
}
const limitText = `Showing ${tempData.length} series out of ${totalLength} series due to performance reasons. Please narrow down the query, so it returns less series`;
setWarning(totalLength > seriesLimit ? limitText : "");
setData(tempData as MetricResult[]);
setIsLoading(false);
} catch (e) {
setIsLoading(false);
if (e instanceof Error && e.name !== "AbortError") {
setError(error);
console.error(e);
}
}
}, [displayType, hideQuery]);
const abortFetch = useCallback(() => {
abortControllerRef.current.abort();
setData([]);
}, [abortControllerRef]);
useEffect(() => {
if (!fetchUrl?.length) return;
fetchData({
fetchUrl,
stateSeriesLimits,
showAllSeries,
});
return () => abortControllerRef.current?.abort();
}, [fetchUrl, stateSeriesLimits, showAllSeries]);
return {
fetchUrl,
isLoading,
data,
error,
queryErrors,
setQueryErrors,
warning,
abortFetch,
};
};

View file

@ -0,0 +1,138 @@
import React, { FC, useState } from "preact/compat";
import LineLoader from "../../components/Main/LineLoader/LineLoader";
import { useCustomPanelState } from "../../state/customPanel/CustomPanelStateContext";
import { useQueryState } from "../../state/query/QueryStateContext";
import "../CustomPanel/style.scss";
import Alert from "../../components/Main/Alert/Alert";
import classNames from "classnames";
import useDeviceDetect from "../../hooks/useDeviceDetect";
import { useRef } from "react";
import CustomPanelTabs from "../CustomPanel/CustomPanelTabs";
import { DisplayTypeSwitch } from "../CustomPanel/DisplayTypeSwitch";
import QueryConfigurator from "../CustomPanel/QueryConfigurator/QueryConfigurator";
import WarningLimitSeries from "../CustomPanel/WarningLimitSeries/WarningLimitSeries";
import { useFetchExport } from "./hooks/useFetchExport";
import { useSetQueryParams } from "../CustomPanel/hooks/useSetQueryParams";
import { DisplayType } from "../../types";
import Hyperlink from "../../components/Main/Hyperlink/Hyperlink";
import { CloseIcon } from "../../components/Main/Icons";
import Button from "../../components/Main/Button/Button";
const RawQueryPage: FC = () => {
useSetQueryParams();
const { isMobile } = useDeviceDetect();
const { displayType } = useCustomPanelState();
const { query } = useQueryState();
const [hideQuery, setHideQuery] = useState<number[]>([]);
const [hideError, setHideError] = useState(!query[0]);
const [showAllSeries, setShowAllSeries] = useState(false);
const [showPageDescription, setShowPageDescription] = useState(true);
const {
data,
error,
isLoading,
warning,
queryErrors,
setQueryErrors,
abortFetch
} = useFetchExport({ hideQuery, showAllSeries });
const controlsRef = useRef<HTMLDivElement>(null);
const showError = !hideError && error;
const handleHideQuery = (queries: number[]) => {
setHideQuery(queries);
};
const handleRunQuery = () => {
setHideError(false);
};
const handleHidePageDescription = () => {
setShowPageDescription(false);
};
return (
<div
className={classNames({
"vm-custom-panel": true,
"vm-custom-panel_mobile": isMobile,
})}
>
<QueryConfigurator
label={"Time series selector"}
queryErrors={!hideError ? queryErrors : []}
setQueryErrors={setQueryErrors}
setHideError={setHideError}
stats={[]}
isLoading={isLoading}
onHideQuery={handleHideQuery}
onRunQuery={handleRunQuery}
abortFetch={abortFetch}
hideButtons={{ traceQuery: true, disableCache: true }}
includeFunctions={false}
/>
{showPageDescription && (
<Alert variant="info">
<div className="vm-explore-metrics-header-description">
<p>
This page provides a dedicated view for querying and displaying raw data directly from VictoriaMetrics.
Users often assume that the <Hyperlink href="https://docs.victoriametrics.com/keyconcepts/#query-data">Query
API</Hyperlink> returns data exactly as stored, but data samples and timestamps may be modified by the API.
For more details, see <Hyperlink
href="https://docs.victoriametrics.com/single-server-victoriametrics/#how-to-export-data-in-json-line-format"
>How
to export data in JSON line format</Hyperlink>
</p>
<Button
variant="text"
size="small"
startIcon={<CloseIcon/>}
onClick={handleHidePageDescription}
ariaLabel="close tips"
/>
</div>
</Alert>
)}
{showError && <Alert variant="error">{error}</Alert>}
{warning && (
<WarningLimitSeries
warning={warning}
query={query}
onChange={setShowAllSeries}
/>
)}
<div
className={classNames({
"vm-custom-panel-body": true,
"vm-custom-panel-body_mobile": isMobile,
"vm-block": true,
"vm-block_mobile": isMobile,
})}
>
{isLoading && <LineLoader/>}
<div
className="vm-custom-panel-body-header"
ref={controlsRef}
>
<div className="vm-custom-panel-body-header__tabs">
<DisplayTypeSwitch tabFilter={(tab) => (tab.value !== DisplayType.table)}/>
</div>
</div>
<CustomPanelTabs
graphData={data}
liveData={data}
isHistogram={false}
displayType={displayType}
controlsRef={controlsRef}
/>
</div>
</div>
);
};
export default RawQueryPage;

View file

@ -15,6 +15,7 @@ const router = {
icons: "/icons",
anomaly: "/anomaly",
query: "/query",
rawQuery: "/raw-query",
downsamplingDebug: "/downsampling-filters-debug",
retentionDebug: "/retention-filters-debug",
};
@ -45,11 +46,15 @@ const routerOptionsDefault = {
}
};
export const routerOptions: {[key: string]: RouterOptions} = {
export const routerOptions: { [key: string]: RouterOptions } = {
[router.home]: {
title: "Query",
...routerOptionsDefault
},
[router.rawQuery]: {
title: "Raw query",
...routerOptionsDefault
},
[router.metrics]: {
title: "Explore Prometheus metrics",
header: {

View file

@ -65,6 +65,7 @@ export const getDefaultNavigation = ({
showAlertLink,
}: NavigationConfig): NavigationItem[] => [
{ value: router.home },
{ value: router.rawQuery },
{ label: "Explore", submenu: getExploreNav() },
{ label: "Tools", submenu: getToolsNav(isEnterpriseLicense) },
{ value: router.dashboards, hide: !showPredefinedDashboards },

View file

@ -10,6 +10,7 @@ export interface CustomPanelState {
isTracingEnabled: boolean;
seriesLimits: SeriesLimits
tableCompact: boolean;
reduceMemUsage: boolean;
}
export type CustomPanelAction =
@ -18,6 +19,7 @@ export type CustomPanelAction =
| { type: "TOGGLE_NO_CACHE"}
| { type: "TOGGLE_QUERY_TRACING" }
| { type: "TOGGLE_TABLE_COMPACT" }
| { type: "TOGGLE_REDUCE_MEM_USAGE"}
export const getInitialDisplayType = () => {
const queryTab = getQueryStringValue("g0.tab", 0) as string;
@ -33,6 +35,7 @@ export const initialCustomPanelState: CustomPanelState = {
isTracingEnabled: false,
seriesLimits: limitsStorage ? JSON.parse(limitsStorage) : DEFAULT_MAX_SERIES,
tableCompact: getFromStorage("TABLE_COMPACT") as boolean || false,
reduceMemUsage: false
};
export function reducer(state: CustomPanelState, action: CustomPanelAction): CustomPanelState {
@ -65,6 +68,12 @@ export function reducer(state: CustomPanelState, action: CustomPanelAction): Cus
...state,
tableCompact: !state.tableCompact
};
case "TOGGLE_REDUCE_MEM_USAGE":
saveToStorage("TABLE_COMPACT", !state.reduceMemUsage);
return {
...state,
reduceMemUsage: !state.reduceMemUsage
};
default:
throw new Error();
}

View file

@ -21,6 +21,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): `-rule` cmd-line flag now supports multi-document YAML files. This could be useful when rules are retrieved via HTTP URL where multiple rule files were merged together in one response. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6753). Thanks to @Irene-123 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6995).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): support scraping from Kubernetes Native Sidecars. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7287).
* FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): add a separate cache type for storing sparse entries when performing large index scans. This significantly reduces memory usage when applying [downsampling filters](https://docs.victoriametrics.com/#downsampling) and [retention filters](https://docs.victoriametrics.com/#retention-filters) during background merge. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7182) for the details.
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add `Raw Query` tab for displaying raw data. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7024).
* BUGFIX: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards) for Single-node VictoriaMetrics, cluster: The free disk space calculation now will subtract the size of the `-storage.minFreeDiskSpaceBytes` flag to correctly display the remaining available space of Single-node VictoriaMetrics/vmstorage rather than the actual available disk space, as well as the full ETA. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7334) for the details.
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): properly set `group_name` and `file` fields for recording rules in `/api/v1/rules`.