diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 282e528638..9de2f3defd 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -880,6 +880,7 @@ def start_webui( "SqlDbPort": clp_config.database.port, "SqlDbName": clp_config.database.name, "SqlDbQueryJobsTableName": QUERY_JOBS_TABLE_NAME, + "SqlDbCompressionJobsTableName": COMPRESSION_JOBS_TABLE_NAME, "MongoDbHost": clp_config.results_cache.host, "MongoDbPort": clp_config.results_cache.port, "MongoDbName": clp_config.results_cache.db_name, @@ -888,7 +889,13 @@ def start_webui( "ClientDir": str(container_webui_dir / "client"), "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"), "StreamTargetUncompressedSize": container_clp_config.stream_output.target_uncompressed_size, + "ArchiveOutputCompressionLevel": clp_config.archive_output.compression_level, + "ArchiveOutputTargetArchiveSize": clp_config.archive_output.target_archive_size, + "ArchiveOutputTargetDictionariesSize": clp_config.archive_output.target_dictionaries_size, + "ArchiveOutputTargetEncodedFileSize": clp_config.archive_output.target_encoded_file_size, + "ArchiveOutputTargetSegmentSize": clp_config.archive_output.target_segment_size, "ClpQueryEngine": clp_config.package.query_engine, + "ClpStorageEngine": clp_config.package.storage_engine, } container_cmd_extra_opts = [] diff --git a/components/webui/client/src/api/compress/index.ts b/components/webui/client/src/api/compress/index.ts new file mode 100644 index 0000000000..22335a443e --- /dev/null +++ b/components/webui/client/src/api/compress/index.ts @@ -0,0 +1,23 @@ +import { + CompressionJob, + CompressionJobCreation, +} from "@webui/common/schemas/compression"; +import axios from "axios"; + + +/** + * Submits a compression job. + * + * @param payload + * @return + */ +const submitCompressionJob = async (payload: CompressionJobCreation) +: Promise => { + console.log("Submitting compression job:", JSON.stringify(payload)); + const {data} = await axios.post("/api/compress", payload); + + return data; +}; + + +export {submitCompressionJob}; diff --git a/components/webui/client/src/config/index.ts b/components/webui/client/src/config/index.ts index 08a29413ea..b8916659c2 100644 --- a/components/webui/client/src/config/index.ts +++ b/components/webui/client/src/config/index.ts @@ -1,16 +1,11 @@ -import {CLP_QUERY_ENGINES} from "@webui/common/config"; +import { + CLP_QUERY_ENGINES, + CLP_STORAGE_ENGINES, +} from "@webui/common/config"; import {settings} from "../settings"; -/** - * CLP variants. - */ -enum CLP_STORAGE_ENGINES { - CLP = "clp", - CLP_S = "clp-s", -} - const SETTINGS_STORAGE_ENGINE = settings.ClpStorageEngine as CLP_STORAGE_ENGINES; const SETTINGS_QUERY_ENGINE = settings.ClpQueryEngine as CLP_QUERY_ENGINES; @@ -22,8 +17,6 @@ const STREAM_TYPE = CLP_STORAGE_ENGINES.CLP === SETTINGS_STORAGE_ENGINE ? "json"; export { - CLP_QUERY_ENGINES, - CLP_STORAGE_ENGINES, SETTINGS_QUERY_ENGINE, SETTINGS_STORAGE_ENGINE, STREAM_TYPE, diff --git a/components/webui/client/src/pages/IngestPage/Compress/ClpSFormItems.tsx b/components/webui/client/src/pages/IngestPage/Compress/ClpSFormItems.tsx new file mode 100644 index 0000000000..39267af6ac --- /dev/null +++ b/components/webui/client/src/pages/IngestPage/Compress/ClpSFormItems.tsx @@ -0,0 +1,40 @@ +import {CLP_DEFAULT_DATASET_NAME} from "@webui/common/config"; +import { + Form, + Input, +} from "antd"; + + +const DATASET_HELPER_TEXT = `If left empty, dataset "${CLP_DEFAULT_DATASET_NAME}" will be used.`; +const DATASET_PLACEHOLDER_TEXT = "The dataset for new archives"; +const TIMESTAMP_KEY_HELPER_TEXT = "If not provided, events will not have assigned" + + " timestamps and can only be searched from the command line without a timestamp filter."; +const TIMESTAMP_KEY_PLACEHOLDER_TEXT = + "The path (e.g. x.y) for the field containing the log event's timestamp"; + +/** + * Renders additional compression job submission form items for CLP-S storage engine. + * + * @return + */ +const ClpSFormItems = () => ( + <> + + + + + + + +); + + +export default ClpSFormItems; diff --git a/components/webui/client/src/pages/IngestPage/Compress/PathsInputFormItem.tsx b/components/webui/client/src/pages/IngestPage/Compress/PathsInputFormItem.tsx new file mode 100644 index 0000000000..4633aa0dce --- /dev/null +++ b/components/webui/client/src/pages/IngestPage/Compress/PathsInputFormItem.tsx @@ -0,0 +1,25 @@ +import { + Form, + Input, +} from "antd"; + + +/** + * Renders a form item for inputting file paths for compression job submission. + * + * @return + */ +const PathsInputFormItem = () => ( + + + +); + + +export default PathsInputFormItem; diff --git a/components/webui/client/src/pages/IngestPage/Compress/SubmitFormItem.tsx b/components/webui/client/src/pages/IngestPage/Compress/SubmitFormItem.tsx new file mode 100644 index 0000000000..db1529d329 --- /dev/null +++ b/components/webui/client/src/pages/IngestPage/Compress/SubmitFormItem.tsx @@ -0,0 +1,31 @@ +import { + Button, + Form, +} from "antd"; + + +interface SubmitFormItemProps { + isSubmitting: boolean; +} + +/** + * Render a submit button for a form. + * + * @param props + * @param props.isSubmitting + * @return + */ +const SubmitFormItem = ({isSubmitting}: SubmitFormItemProps) => ( + + + +); + + +export default SubmitFormItem; diff --git a/components/webui/client/src/pages/IngestPage/Compress/index.tsx b/components/webui/client/src/pages/IngestPage/Compress/index.tsx new file mode 100644 index 0000000000..775e723865 --- /dev/null +++ b/components/webui/client/src/pages/IngestPage/Compress/index.tsx @@ -0,0 +1,108 @@ +import { + useMutation, + useQueryClient, +} from "@tanstack/react-query"; +import {CLP_STORAGE_ENGINES} from "@webui/common/config"; +import {CompressionJobCreation} from "@webui/common/schemas/compression"; +import { + Form, + Typography, +} from "antd"; + +import {submitCompressionJob} from "../../../api/compress"; +import {DashboardCard} from "../../../components/DashboardCard"; +import {SETTINGS_STORAGE_ENGINE} from "../../../config"; +import ClpSFormItems from "./ClpSFormItems"; +import PathsInputFormItem from "./PathsInputFormItem"; +import SubmitFormItem from "./SubmitFormItem"; + + +type FormValues = { + paths: string; + dataset?: string; + timestampKey?: string; +}; + + +/** + * Renders a compression job submission form. + * + * @return + */ +const Compress = () => { + const [form] = Form.useForm(); + + const queryClient = useQueryClient(); + const { + mutate, + isPending: isSubmitting, + isSuccess, + isError, + data, + error, + } = useMutation({ + mutationFn: submitCompressionJob, + onSettled: async () => { + // Invalidate queries that are affected by a new compression job. + await queryClient.invalidateQueries({queryKey: ["jobs"]}); + }, + onSuccess: () => { + form.resetFields(); + }, + }); + + const handleSubmit = (values: FormValues) => { + // eslint-disable-next-line no-warning-comments + // TODO: replace the UI with a file selector and remove below string manipulation. + // Convert multiline input to array of paths. + const paths = values.paths + .split("\n") + .map((path) => path.trim()) + .filter((path) => 0 < path.length); + + const payload: CompressionJobCreation = {paths}; + + if ("undefined" !== typeof values.dataset) { + payload.dataset = values.dataset; + } + if ("undefined" !== typeof values.timestampKey) { + payload.timestampKey = values.timestampKey; + } + + mutate(payload); + }; + + return ( + +
+ + {CLP_STORAGE_ENGINES.CLP_S === SETTINGS_STORAGE_ENGINE && } + + + {isSuccess && ( + + Compression job submitted successfully with ID: + {" "} + {data.jobId} + + )} + {isError && ( + + Failed to submit compression job: + {" "} + {error instanceof Error ? + error.message : + "Unknown error"} + + )} + +
+ ); +}; + + +export default Compress; diff --git a/components/webui/client/src/pages/IngestPage/Details/index.tsx b/components/webui/client/src/pages/IngestPage/Details/index.tsx index cc94297f26..bce7950c66 100644 --- a/components/webui/client/src/pages/IngestPage/Details/index.tsx +++ b/components/webui/client/src/pages/IngestPage/Details/index.tsx @@ -1,10 +1,8 @@ import {useQuery} from "@tanstack/react-query"; +import {CLP_STORAGE_ENGINES} from "@webui/common/config"; import dayjs from "dayjs"; -import { - CLP_STORAGE_ENGINES, - SETTINGS_STORAGE_ENGINE, -} from "../../../config"; +import {SETTINGS_STORAGE_ENGINE} from "../../../config"; import {fetchDatasetNames} from "../../SearchPage/SearchControls/Dataset/sql"; import Files from "./Files"; import styles from "./index.module.css"; diff --git a/components/webui/client/src/pages/IngestPage/SpaceSavings/index.tsx b/components/webui/client/src/pages/IngestPage/SpaceSavings/index.tsx index 8cb669837d..b174425e93 100644 --- a/components/webui/client/src/pages/IngestPage/SpaceSavings/index.tsx +++ b/components/webui/client/src/pages/IngestPage/SpaceSavings/index.tsx @@ -1,12 +1,10 @@ import {useQuery} from "@tanstack/react-query"; +import {CLP_STORAGE_ENGINES} from "@webui/common/config"; import {theme} from "antd"; import {DashboardCard} from "../../../components/DashboardCard"; import Stat from "../../../components/Stat"; -import { - CLP_STORAGE_ENGINES, - SETTINGS_STORAGE_ENGINE, -} from "../../../config"; +import {SETTINGS_STORAGE_ENGINE} from "../../../config"; import {fetchDatasetNames} from "../../SearchPage/SearchControls/Dataset/sql"; import CompressedSize from "./CompressedSize"; import styles from "./index.module.css"; diff --git a/components/webui/client/src/pages/IngestPage/index.module.css b/components/webui/client/src/pages/IngestPage/index.module.css index a71c2aaa6b..0524d74977 100644 --- a/components/webui/client/src/pages/IngestPage/index.module.css +++ b/components/webui/client/src/pages/IngestPage/index.module.css @@ -7,6 +7,6 @@ gap: 20px; } -.jobsGrid { +.fullRow { grid-column: 1 / -1; } diff --git a/components/webui/client/src/pages/IngestPage/index.tsx b/components/webui/client/src/pages/IngestPage/index.tsx index d82a8fe9e7..cce5765b45 100644 --- a/components/webui/client/src/pages/IngestPage/index.tsx +++ b/components/webui/client/src/pages/IngestPage/index.tsx @@ -1,3 +1,4 @@ +import Compress from "./Compress"; import Details from "./Details"; import styles from "./index.module.css"; import Jobs from "./Jobs"; @@ -14,7 +15,10 @@ const IngestPage = () => {
-
+
+ +
+
diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/NativeControls.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/NativeControls.tsx index ae7b4c7d56..0108dee15c 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/NativeControls.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/NativeControls.tsx @@ -1,7 +1,6 @@ -import { - CLP_STORAGE_ENGINES, - SETTINGS_STORAGE_ENGINE, -} from "../../../config"; +import {CLP_STORAGE_ENGINES} from "@webui/common/config"; + +import {SETTINGS_STORAGE_ENGINE} from "../../../config"; import Dataset from "./Dataset"; import styles from "./index.module.css"; import QueryInput from "./QueryInput"; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton/index.tsx index e321067f61..a7626704e4 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton/index.tsx @@ -1,15 +1,13 @@ import {useCallback} from "react"; import {SearchOutlined} from "@ant-design/icons"; +import {CLP_STORAGE_ENGINES} from "@webui/common/config"; import { Button, Tooltip, } from "antd"; -import { - CLP_STORAGE_ENGINES, - SETTINGS_STORAGE_ENGINE, -} from "../../../../../config"; +import {SETTINGS_STORAGE_ENGINE} from "../../../../../config"; import {computeTimelineConfig} from "../../../SearchResults/SearchResultsTimeline/utils"; import useSearchStore from "../../../SearchState/index"; import {SEARCH_UI_STATE} from "../../../SearchState/typings"; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx index 15910bdae5..01673ee5ab 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx @@ -1,7 +1,6 @@ -import { - CLP_QUERY_ENGINES, - SETTINGS_QUERY_ENGINE, -} from "../../../config"; +import {CLP_QUERY_ENGINES} from "@webui/common/config"; + +import {SETTINGS_QUERY_ENGINE} from "../../../config"; import usePrestoSearchState from "../SearchState/Presto"; import {PRESTO_SQL_INTERFACE} from "../SearchState/Presto/typings"; import NativeControls from "./NativeControls"; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts b/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts index 661fae04f3..a15327c2e5 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts +++ b/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts @@ -1,3 +1,4 @@ +import {CLP_STORAGE_ENGINES} from "@webui/common/config"; import { type QueryJob, type QueryJobCreation, @@ -9,10 +10,7 @@ import { clearQueryResults, submitQuery, } from "../../../api/search"; -import { - CLP_STORAGE_ENGINES, - SETTINGS_STORAGE_ENGINE, -} from "../../../config"; +import {SETTINGS_STORAGE_ENGINE} from "../../../config"; import useSearchStore, {SEARCH_STATE_DEFAULT} from "../SearchState/"; import {SEARCH_UI_STATE} from "../SearchState/typings"; import {unquoteString} from "./utils"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx index 51b6cf6d86..e9d8e0ba21 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx @@ -1,12 +1,10 @@ import SyntaxHighlighter from "react-syntax-highlighter"; import {tomorrow} from "react-syntax-highlighter/dist/esm/styles/hljs"; +import {CLP_STORAGE_ENGINES} from "@webui/common/config"; import {Typography} from "antd"; -import { - CLP_STORAGE_ENGINES, - SETTINGS_STORAGE_ENGINE, -} from "../.././../../../config"; +import {SETTINGS_STORAGE_ENGINE} from "../.././../../../config"; import LogViewerLink from "./LogViewerLink"; import {highlighterCustomStyles} from "./utils"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/typings.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/typings.tsx index 1ee1971a50..bb903f1003 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/typings.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/SearchResultsVirtualTable/typings.tsx @@ -1,10 +1,8 @@ +import {CLP_STORAGE_ENGINES} from "@webui/common/config"; import {TableProps} from "antd"; import dayjs from "dayjs"; -import { - CLP_STORAGE_ENGINES, - SETTINGS_STORAGE_ENGINE, -} from "../../../../../config"; +import {SETTINGS_STORAGE_ENGINE} from "../../../../../config"; import {DATETIME_FORMAT_TEMPLATE} from "../../../../../typings/datetime"; import Message from "../Message"; import {getStreamId} from "../utils"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx index 5cdfcf6841..14a0bc3f9b 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx @@ -4,10 +4,9 @@ import { useState, } from "react"; -import { - CLP_QUERY_ENGINES, - SETTINGS_QUERY_ENGINE, -} from "../../../../config"; +import {CLP_QUERY_ENGINES} from "@webui/common/config"; + +import {SETTINGS_QUERY_ENGINE} from "../../../../config"; import PrestoResultsVirtualTable from "./Presto/PrestoResultsVirtualTable"; import SearchResultsVirtualTable from "./SearchResultsVirtualTable"; import {TABLE_BOTTOM_PADDING} from "./typings"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/utils.ts b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/utils.ts index 3aa3cf6072..39af6ef21c 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/utils.ts +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/utils.ts @@ -1,7 +1,6 @@ -import { - CLP_STORAGE_ENGINES, - SETTINGS_STORAGE_ENGINE, -} from "../../../../config"; +import {CLP_STORAGE_ENGINES} from "@webui/common/config"; + +import {SETTINGS_STORAGE_ENGINE} from "../../../../config"; import type {SearchResult} from "./SearchResultsVirtualTable/typings"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/index.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/index.tsx index 627cc4db06..233ad91dc8 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/index.tsx @@ -3,15 +3,13 @@ import { useEffect, } from "react"; +import {CLP_STORAGE_ENGINES} from "@webui/common/config"; import {Card} from "antd"; import {Dayjs} from "dayjs"; import {TimelineConfig} from "src/components/ResultsTimeline/typings"; import ResultsTimeline from "../../../../components/ResultsTimeline/index"; -import { - CLP_STORAGE_ENGINES, - SETTINGS_STORAGE_ENGINE, -} from "../../../../config"; +import {SETTINGS_STORAGE_ENGINE} from "../../../../config"; import {handleQuerySubmit} from "../../SearchControls/search-requests"; import {TIME_RANGE_OPTION} from "../../SearchControls/TimeRangeInput/utils"; import useSearchStore, {SEARCH_STATE_DEFAULT} from "../../SearchState/index"; diff --git a/components/webui/client/src/pages/SearchPage/index.tsx b/components/webui/client/src/pages/SearchPage/index.tsx index 9345164175..d7a43c26ce 100644 --- a/components/webui/client/src/pages/SearchPage/index.tsx +++ b/components/webui/client/src/pages/SearchPage/index.tsx @@ -1,7 +1,6 @@ -import { - CLP_QUERY_ENGINES, - SETTINGS_QUERY_ENGINE, -} from "../../config"; +import {CLP_QUERY_ENGINES} from "@webui/common/config"; + +import {SETTINGS_QUERY_ENGINE} from "../../config"; import styles from "./index.module.css"; import {ProgressBar} from "./Presto/ProgressBar"; import SearchControls from "./SearchControls"; diff --git a/components/webui/common/src/config.ts b/components/webui/common/src/config.ts index 1b56113857..0132de9e84 100644 --- a/components/webui/common/src/config.ts +++ b/components/webui/common/src/config.ts @@ -7,4 +7,22 @@ enum CLP_QUERY_ENGINES { PRESTO = "presto", } -export {CLP_QUERY_ENGINES}; +/** + * CLP variants. + */ +enum CLP_STORAGE_ENGINES { + CLP = "clp", + CLP_S = "clp-s", +} + +/** + * Matching the `CLP_DEFAULT_DATASET_NAME` in `clp_py_utils.clp_config`. + */ +const CLP_DEFAULT_DATASET_NAME = "default"; + + +export { + CLP_DEFAULT_DATASET_NAME, + CLP_QUERY_ENGINES, + CLP_STORAGE_ENGINES, +}; diff --git a/components/webui/common/src/schemas/compression.ts b/components/webui/common/src/schemas/compression.ts new file mode 100644 index 0000000000..4582fb79ea --- /dev/null +++ b/components/webui/common/src/schemas/compression.ts @@ -0,0 +1,34 @@ +import { + Static, + Type, +} from "@fastify/type-provider-typebox"; + + +/** + * Schema for request to create a new compression job. + */ +const CompressionJobCreationSchema = Type.Object({ + paths: Type.Array(Type.String()), + dataset: Type.Optional(Type.String()), + timestampKey: Type.Optional(Type.String()), +}); + +type CompressionJobCreation = Static; + +/** + * Schema for compression job response. + */ +const CompressionJobSchema = Type.Object({ + jobId: Type.Number(), +}); + +type CompressionJob = Static; + +export { + CompressionJobCreationSchema, + CompressionJobSchema, +}; +export type { + CompressionJob, + CompressionJobCreation, +}; diff --git a/components/webui/server/settings.json b/components/webui/server/settings.json index 9b0386021f..13035219cc 100644 --- a/components/webui/server/settings.json +++ b/components/webui/server/settings.json @@ -3,6 +3,7 @@ "SqlDbPort": 3306, "SqlDbName": "clp-db", "SqlDbQueryJobsTableName": "query_jobs", + "SqlDbCompressionJobsTableName": "compression_jobs", "MongoDbHost": "localhost", "MongoDbPort": 27017, @@ -18,7 +19,14 @@ "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, + "ArchiveOutputCompressionLevel": 3, + "ArchiveOutputTargetArchiveSize": 268435456, + "ArchiveOutputTargetDictionariesSize": 33554432, + "ArchiveOutputTargetEncodedFileSize": 268435456, + "ArchiveOutputTargetSegmentSize": 268435456, + "ClpQueryEngine": "clp", + "ClpStorageEngine": "clp", "PrestoHost": "localhost", "PrestoPort": 8889 } diff --git a/components/webui/server/src/plugins/app/CompressionJobDbManager/index.ts b/components/webui/server/src/plugins/app/CompressionJobDbManager/index.ts new file mode 100644 index 0000000000..86feeb751f --- /dev/null +++ b/components/webui/server/src/plugins/app/CompressionJobDbManager/index.ts @@ -0,0 +1,74 @@ +import {brotliCompressSync} from "node:zlib"; + +import type {MySQLPromisePool} from "@fastify/mysql"; +import {encode} from "@msgpack/msgpack"; +import {FastifyInstance} from "fastify"; +import fp from "fastify-plugin"; +import {ResultSetHeader} from "mysql2"; + +import settings from "../../../../settings.json" with {type: "json"}; +import {COMPRESSION_JOBS_TABLE_COLUMN_NAMES} from "../../../typings/compression.js"; +import {CompressionJobConfig} from "./typings.js"; + + +/** + * Class for managing compression jobs in the CLP package compression scheduler database. + */ +class CompressionJobDbManager { + readonly #sqlPool: MySQLPromisePool; + + readonly #tableName: string; + + private constructor (sqlPool: MySQLPromisePool, tableName: string) { + this.#sqlPool = sqlPool; + this.#tableName = tableName; + } + + /** + * Creates a new CompressionJobDbManager. + * + * @param fastify + * @return + */ + static create (fastify: FastifyInstance): CompressionJobDbManager { + return new CompressionJobDbManager(fastify.mysql, settings.SqlDbCompressionJobsTableName); + } + + /** + * Submits a compression job to the database. + * + * @param jobConfig + * @return The job's ID. + * @throws {Error} on error. + */ + async submitJob (jobConfig: CompressionJobConfig): Promise { + const [result] = await this.#sqlPool.query( + ` + INSERT INTO ${this.#tableName} ( + ${COMPRESSION_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG} + ) + VALUES (?) + `, + [ + Buffer.from(brotliCompressSync(encode(jobConfig))), + ] + ); + + return result.insertId; + } +} + +declare module "fastify" { + interface FastifyInstance { + CompressionJobDbManager: CompressionJobDbManager; + } +} + +export default fp( + (fastify) => { + fastify.decorate("CompressionJobDbManager", CompressionJobDbManager.create(fastify)); + }, + { + name: "CompressionJobDbManager", + } +); diff --git a/components/webui/server/src/plugins/app/CompressionJobDbManager/typings.ts b/components/webui/server/src/plugins/app/CompressionJobDbManager/typings.ts new file mode 100644 index 0000000000..6ff509cabc --- /dev/null +++ b/components/webui/server/src/plugins/app/CompressionJobDbManager/typings.ts @@ -0,0 +1,18 @@ +interface CompressionJobConfig { + input: { + paths_to_compress: string[]; + path_prefix_to_remove: string; + dataset?: string; + timestamp_key?: string; + }; + output: { + target_archive_size: number; + target_dictionaries_size: number; + target_segment_size: number; + target_encoded_file_size: number; + compression_level: number; + }; +} + + +export type {CompressionJobConfig}; diff --git a/components/webui/server/src/plugins/app/QueryJobDbManager/index.ts b/components/webui/server/src/plugins/app/QueryJobDbManager/index.ts index f14f728ec1..d7e8f9984c 100644 --- a/components/webui/server/src/plugins/app/QueryJobDbManager/index.ts +++ b/components/webui/server/src/plugins/app/QueryJobDbManager/index.ts @@ -51,7 +51,7 @@ class QueryJobDbManager { async submitJob (jobConfig: object, jobType: QUERY_JOB_TYPE): Promise { const [result] = await this.#sqlPool.query( ` - INSERT INTO ${settings.SqlDbQueryJobsTableName} ( + INSERT INTO ${this.#tableName} ( ${QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG}, ${QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE} ) diff --git a/components/webui/server/src/routes/api/compress/index.ts b/components/webui/server/src/routes/api/compress/index.ts new file mode 100644 index 0000000000..365af15270 --- /dev/null +++ b/components/webui/server/src/routes/api/compress/index.ts @@ -0,0 +1,96 @@ +import {FastifyPluginAsyncTypebox} from "@fastify/type-provider-typebox"; +import { + CLP_DEFAULT_DATASET_NAME, + CLP_STORAGE_ENGINES, +} from "@webui/common/config"; +import { + CompressionJobCreationSchema, + CompressionJobSchema, +} from "@webui/common/schemas/compression"; +import {ErrorSchema} from "@webui/common/schemas/error"; +import {StatusCodes} from "http-status-codes"; + +import settings from "../../../../settings.json" with {type: "json"}; +import {CompressionJobConfig} from "../../../plugins/app/CompressionJobDbManager/typings.js"; +import {CONTAINER_INPUT_LOGS_ROOT_DIR} from "./typings.js"; + + +/** + * Default compression job configuration. + */ +const DEFAULT_COMPRESSION_JOB_CONFIG: CompressionJobConfig = Object.freeze({ + input: { + paths_to_compress: [], + path_prefix_to_remove: CONTAINER_INPUT_LOGS_ROOT_DIR, + }, + output: { + compression_level: settings.ArchiveOutputCompressionLevel, + target_archive_size: settings.ArchiveOutputTargetArchiveSize, + target_dictionaries_size: settings.ArchiveOutputTargetDictionariesSize, + target_encoded_file_size: settings.ArchiveOutputTargetEncodedFileSize, + target_segment_size: settings.ArchiveOutputTargetSegmentSize, + }, +}); + +/** + * Compression API routes. + * + * @param fastify + */ +const plugin: FastifyPluginAsyncTypebox = async (fastify) => { + const {CompressionJobDbManager} = fastify; + + /** + * Submits a compression job and initiates the compression process. + */ + fastify.post( + "/", + { + schema: { + body: CompressionJobCreationSchema, + response: { + [StatusCodes.CREATED]: CompressionJobSchema, + [StatusCodes.INTERNAL_SERVER_ERROR]: ErrorSchema, + }, + tags: ["Compression"], + }, + }, + async (request, reply) => { + const { + paths, + dataset, + timestampKey, + } = request.body; + + const jobConfig: CompressionJobConfig = structuredClone(DEFAULT_COMPRESSION_JOB_CONFIG); + jobConfig.input.paths_to_compress = paths.map( + (path) => CONTAINER_INPUT_LOGS_ROOT_DIR + path + ); + + if (CLP_STORAGE_ENGINES.CLP_S === settings.ClpStorageEngine as CLP_STORAGE_ENGINES) { + if ("string" !== typeof dataset || 0 === dataset.length) { + jobConfig.input.dataset = CLP_DEFAULT_DATASET_NAME; + } else { + jobConfig.input.dataset = dataset; + } + if ("undefined" !== typeof timestampKey) { + jobConfig.input.timestamp_key = timestampKey; + } + } + + try { + const jobId = await CompressionJobDbManager.submitJob(jobConfig); + reply.code(StatusCodes.CREATED); + + return {jobId}; + } catch (err: unknown) { + const errMsg = "Unable to submit compression job to the SQL database"; + request.log.error(err, errMsg); + + return reply.internalServerError(errMsg); + } + } + ); +}; + +export default plugin; diff --git a/components/webui/server/src/routes/api/compress/typings.ts b/components/webui/server/src/routes/api/compress/typings.ts new file mode 100644 index 0000000000..2b5b8f0afe --- /dev/null +++ b/components/webui/server/src/routes/api/compress/typings.ts @@ -0,0 +1,7 @@ +/** + * Matching the `CONTAINER_INPUT_LOGS_ROOT_DIR` in `clp_package_utils.general`. + */ +const CONTAINER_INPUT_LOGS_ROOT_DIR = "/mnt/logs"; + + +export {CONTAINER_INPUT_LOGS_ROOT_DIR}; diff --git a/components/webui/server/src/typings/compression.ts b/components/webui/server/src/typings/compression.ts new file mode 100644 index 0000000000..6132f99fe8 --- /dev/null +++ b/components/webui/server/src/typings/compression.ts @@ -0,0 +1,11 @@ +/** + * The `compression_jobs` table's column names. + */ +enum COMPRESSION_JOBS_TABLE_COLUMN_NAMES { + ID = "id", + STATUS = "status", + JOB_CONFIG = "clp_config", +} + + +export {COMPRESSION_JOBS_TABLE_COLUMN_NAMES}; diff --git a/components/webui/server/src/typings/query.ts b/components/webui/server/src/typings/query.ts index 5a08635ced..32794636c1 100644 --- a/components/webui/server/src/typings/query.ts +++ b/components/webui/server/src/typings/query.ts @@ -3,10 +3,7 @@ import {RowDataPacket} from "mysql2/promise"; /** - * Matching the `QueryJobStatus` class in - * `job_orchestration.query_scheduler.constants`. - * - * @enum {number} + * Matching the `QueryJobStatus` class in `job_orchestration.scheduler.constants`. */ enum QUERY_JOB_STATUS { PENDING = 0, @@ -29,8 +26,6 @@ const QUERY_JOB_STATUS_WAITING_STATES = new Set([ /** * The `query_jobs` table's column names. - * - * @enum {string} */ enum QUERY_JOBS_TABLE_COLUMN_NAMES { ID = "id",