vmui: add field for log entries limit (#5799)

* vmui: add field for log entries limit (#5674)

* vmui: refactor useFetchLogs

* vmui: fix log query encoding

---------

Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx

@@ -11,14 +11,19 @@ import "./style.scss";
 import { ErrorTypes } from "../../types";
 import { useState } from "react";
 import { useTimeState } from "../../state/time/TimeStateContext";
+import { getFromStorage, saveToStorage } from "../../utils/storage";
+
+const storageLimit = Number(getFromStorage("LOGS_LIMIT"));
+const defaultLimit = isNaN(storageLimit) ? 1000 : storageLimit;
 
 const ExploreLogs: FC = () => {
   const { serverUrl } = useAppState();
   const { duration, relativeTime, period } = useTimeState();
   const { setSearchParamsFromKeys } = useSearchParamsFromObject();
 
+  const [limit, setLimit] = useStateSearchParams(defaultLimit, "limit");
   const [query, setQuery] = useStateSearchParams("", "query");
-  const { logs, isLoading, error, fetchLogs } = useFetchLogs(serverUrl, query);
+  const { logs, isLoading, error, fetchLogs } = useFetchLogs(serverUrl, query, limit);
   const [queryError, setQueryError] = useState<ErrorTypes | string>("");
   const [loaded, isLoaded] = useState(false);
 
@@ -40,6 +45,12 @@ const ExploreLogs: FC = () => {
     });
   };
 
+  const handleChangeLimit = (limit: number) => {
+    setLimit(limit);
+    setSearchParamsFromKeys({ limit });
+    saveToStorage("LOGS_LIMIT", `${limit}`);
+  };
+
   useEffect(() => {
     if (query) handleRunQuery();
   }, [period]);
@@ -53,7 +64,9 @@ const ExploreLogs: FC = () => {
       <ExploreLogsHeader
         query={query}
         error={queryError}
+        limit={limit}
         onChange={setQuery}
+        onChangeLimit={handleChangeLimit}
         onRun={handleRunQuery}
       />
       {isLoading && <Spinner />}

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/ExploreLogsBody.tsx

@@ -8,10 +8,8 @@ import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import { Logs } from "../../../api/types";
 import dayjs from "dayjs";
 import { useTimeState } from "../../../state/time/TimeStateContext";
-import SelectLimit from "../../../components/Main/Pagination/SelectLimit/SelectLimit";
 import useStateSearchParams from "../../../hooks/useStateSearchParams";
 import useSearchParamsFromObject from "../../../hooks/useSearchParamsFromObject";
-import { getFromStorage, saveToStorage } from "../../../utils/storage";
 import TableSettings from "../../../components/Table/TableSettings/TableSettings";
 import useBoolean from "../../../hooks/useBoolean";
 import TableLogs from "./TableLogs";
@@ -29,16 +27,15 @@ enum DisplayType {
 }
 
 const tabs = [
-  { label: "Group", value: DisplayType.group, icon: <ListIcon /> },
+  { label: "Group", value: DisplayType.group, icon: <ListIcon/> },
-  { label: "Table", value: DisplayType.table, icon: <TableIcon /> },
+  { label: "Table", value: DisplayType.table, icon: <TableIcon/> },
-  { label: "JSON", value: DisplayType.json, icon: <CodeIcon /> },
+  { label: "JSON", value: DisplayType.json, icon: <CodeIcon/> },
 ];
 
 const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, loaded }) => {
   const { isMobile } = useDeviceDetect();
   const { timezone } = useTimeState();
   const { setSearchParamsFromKeys } = useSearchParamsFromObject();
-  const [limitRows, setLimitRows] = useStateSearchParams(getFromStorage("LOGS_LIMIT") || 50, "limit");
 
   const [activeTab, setActiveTab] = useStateSearchParams(DisplayType.group, "view");
   const [displayColumns, setDisplayColumns] = useState<string[]>([]);
@@ -67,12 +64,6 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, loaded }) => {
     setSearchParamsFromKeys({ view });
   };
 
-  const handleChangeLimit = (limit: number) => {
-    setLimitRows(limit);
-    setSearchParamsFromKeys({ limit });
-    saveToStorage("LOGS_LIMIT", `${limit}`);
-  };
-
   return (
     <div
       className={classNames({
@@ -97,10 +88,6 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, loaded }) => {
         </div>
         {activeTab === DisplayType.table && (
           <div className="vm-explore-logs-body-header__settings">
-            <SelectLimit
-              limit={+limitRows}
-              onChange={handleChangeLimit}
-            />
             <TableSettings
               columns={columns}
               defaultColumns={displayColumns}
@@ -128,7 +115,6 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, loaded }) => {
             {activeTab === DisplayType.table && (
               <TableLogs
                 logs={logs}
-                limitRows={+limitRows}
                 displayColumns={displayColumns}
                 tableCompact={tableCompact}
                 columns={columns}
@@ -141,7 +127,7 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, loaded }) => {
               />
             )}
             {activeTab === DisplayType.json && (
-              <JsonView data={data} />
+              <JsonView data={data}/>
             )}
           </>
         )}

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/TableLogs.tsx

@@ -2,22 +2,15 @@ import React, { FC, useMemo } from "preact/compat";
 import "./style.scss";
 import Table from "../../../components/Table/Table";
 import { Logs } from "../../../api/types";
-import useStateSearchParams from "../../../hooks/useStateSearchParams";
-import useSearchParamsFromObject from "../../../hooks/useSearchParamsFromObject";
-import PaginationControl from "../../../components/Main/Pagination/PaginationControl/PaginationControl";
 
 interface TableLogsProps {
   logs: Logs[];
-  limitRows: number;
   displayColumns: string[];
   tableCompact: boolean;
   columns: string[];
 }
 
-const TableLogs: FC<TableLogsProps> = ({ logs, limitRows, displayColumns, tableCompact, columns }) => {
+const TableLogs: FC<TableLogsProps> = ({ logs, displayColumns, tableCompact, columns }) => {
-  const { setSearchParamsFromKeys } = useSearchParamsFromObject();
-  const [page, setPage] = useStateSearchParams(1, "page");
-
   const getColumnClass = (key: string) => {
     switch (key) {
       case "time":
@@ -29,16 +22,6 @@ const TableLogs: FC<TableLogsProps> = ({ logs, limitRows, displayColumns, tableC
     }
   };
 
-  // TODO: Remove when pagination is implemented on the backend.
-  const paginationOffset = useMemo(() => {
-    const startIndex = (page - 1) * Number(limitRows);
-    const endIndex = startIndex + Number(limitRows);
-    return {
-      startIndex,
-      endIndex
-    };
-  }, [page, limitRows]);
-
   const tableColumns = useMemo(() => {
     if (tableCompact) {
       return [{
@@ -60,11 +43,6 @@ const TableLogs: FC<TableLogsProps> = ({ logs, limitRows, displayColumns, tableC
     return tableColumns.filter(c => displayColumns.includes(c.key as string));
   }, [tableColumns, displayColumns, tableCompact]);
 
-  const handleChangePage = (page: number) => {
-    setPage(page);
-    setSearchParamsFromKeys({ page });
-  };
-
   return (
     <>
       <Table
@@ -72,14 +50,7 @@ const TableLogs: FC<TableLogsProps> = ({ logs, limitRows, displayColumns, tableC
         columns={filteredColumns}
         defaultOrderBy={"time"}
         copyToClipboard={"data"}
-        paginationOffset={paginationOffset}
+        paginationOffset={{ startIndex: 0, endIndex: Infinity }}
-      />
-      <PaginationControl
-        page={page}
-        limit={+limitRows}
-        // TODO: Remove .slice() when pagination is implemented on the backend.
-        length={logs.slice(paginationOffset.startIndex, paginationOffset.endIndex).length}
-        onChange={handleChangePage}
       />
     </>
   );

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/ExploreLogsHeader.tsx

@@ -1,21 +1,42 @@
-import React, { FC } from "react";
+import React, { FC, useEffect, useState } from "preact/compat";
 import { InfoIcon, PlayIcon, WikiIcon } from "../../../components/Main/Icons";
 import "./style.scss";
 import classNames from "classnames";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import Button from "../../../components/Main/Button/Button";
 import QueryEditor from "../../../components/Configurators/QueryEditor/QueryEditor";
+import TextField from "../../../components/Main/TextField/TextField";
 
 export interface ExploreLogHeaderProps {
   query: string;
+  limit: number;
   error?: string;
   onChange: (val: string) => void;
+  onChangeLimit: (val: number) => void;
   onRun: () => void;
 }
 
-const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({ query, error, onChange, onRun }) => {
+const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({ query, limit, error, onChange, onChangeLimit, onRun }) => {
   const { isMobile } = useDeviceDetect();
 
+  const [errorLimit, setErrorLimit] = useState("");
+  const [limitInput, setLimitInput] = useState(limit);
+
+  const handleChangeLimit = (val: string) => {
+    const number = +val;
+    setLimitInput(number);
+    if (isNaN(number) || number < 0) {
+      setErrorLimit("Number must be bigger than zero");
+    } else {
+      setErrorLimit("");
+      onChangeLimit(number);
+    }
+  };
+
+  useEffect(() => {
+    setLimitInput(limit);
+  }, [limit]);
+
   return (
     <div
       className={classNames({
@@ -24,7 +45,7 @@ const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({ query, error, onChange,
         "vm-block_mobile": isMobile,
       })}
     >
-      <div className="vm-explore-logs-header__input">
+      <div className="vm-explore-logs-header-top">
         <QueryEditor
           value={query}
           autocomplete={false}
@@ -35,6 +56,14 @@ const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({ query, error, onChange,
           label={"Log query"}
           error={error}
         />
+        <TextField
+          label="Limit entries"
+          type="number"
+          value={limitInput}
+          error={errorLimit}
+          onChange={handleChangeLimit}
+          onEnter={onRun}
+        />
       </div>
       <div className="vm-explore-logs-header-bottom">
         <div className="vm-explore-logs-header-bottom-helpful">

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/style.scss

@@ -5,7 +5,13 @@
   align-items: center;
   gap: $padding-global;
 
-  &__input {}
+  &-top {
+    display: grid;
+    grid-template-columns: 8fr 2fr;
+    align-items: flex-start;
+    justify-content: center;
+    gap: $padding-global;
+  }
 
   &-bottom {
     display: flex;

app/vmui/packages/vmui/src/pages/ExploreLogs/hooks/useFetchLogs.ts

@@ -5,11 +5,8 @@ import { Logs } from "../../../api/types";
 import { useTimeState } from "../../../state/time/TimeStateContext";
 import dayjs from "dayjs";
 
-const MAX_LINES = 1000;
+export const useFetchLogs = (server: string, query: string, limit: number) => {
-
-export const useFetchLogs = (server: string, query: string) => {
   const { period } = useTimeState();
-
   const [logs, setLogs] = useState<Logs[]>([]);
   const [isLoading, setIsLoading] = useState(false);
   const [error, setError] = useState<ErrorTypes | string>();
@@ -22,7 +19,7 @@ export const useFetchLogs = (server: string, query: string) => {
       const start = dayjs(period.start * 1000).tz().toISOString();
       const end = dayjs(period.end * 1000).tz().toISOString();
       const timerange = `_time:[${start}, ${end}]`;
-      return `${timerange} AND (${query})`;
+      return `${timerange} AND ${query}`;
     }
     return query;
   }, [query, period]);
@@ -30,13 +27,24 @@ export const useFetchLogs = (server: string, query: string) => {
   const options = useMemo(() => ({
     method: "POST",
     headers: {
-      "Accept": "application/stream+json; charset=utf-8",
+      "Accept": "application/stream+json",
-      "Content-Type": "application/x-www-form-urlencoded",
     },
-    body: `query=${encodeURIComponent(queryWithTime.trim())}`
+    body: new URLSearchParams({
-  }), [queryWithTime]);
+      query: queryWithTime.trim(),
+      limit: `${limit}`
+    })
+  }), [queryWithTime, limit]);
+
+  const parseLineToJSON = (line: string): Logs | null => {
+    try {
+      return JSON.parse(line);
+    } catch (e) {
+      return null;
+    }
+  };
 
   const fetchLogs = useCallback(async () => {
+    const limit = Number(options.body.get("limit")) + 1;
     setIsLoading(true);
     setError(undefined);
     try {
@@ -65,29 +73,21 @@ export const useFetchLogs = (server: string, query: string) => {
         const lines = decoder.decode(value, { stream: true }).split("\n");
         result.push(...lines);
 
-        // Trim result to MAX_LINES
+        // Trim result to limit
-        if (result.length > MAX_LINES) {
+        // This will lose its meaning with these changes:
-          result.splice(0, result.length - MAX_LINES);
+        // https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5778
+        if (result.length > limit) {
+          result.splice(0, result.length - limit);
         }
 
-        if (result.length >= MAX_LINES) {
+        if (result.length >= limit) {
           // Reached the maximum line limit
           reader.cancel();
           break;
         }
       }
-      const data = result
+      const data = result.map(parseLineToJSON).filter(line => line) as Logs[];
-        .map((line) => {
-          try {
-            return JSON.parse(line);
-          } catch (e) {
-            return "";
-          }
-        })
-        .filter(line => line);
-
       setLogs(data);
-
     } catch (e) {
       console.error(e);
       setLogs([]);