merge

Author: davemarco

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,16 +1,37 @@
+<!-- markdownlint-disable MD012 -->
+
 <!--
 Set the PR title to a meaningful commit message that:
-- follows the Conventional Commits specification (https://www.conventionalcommits.org).
-- is in imperative form.
+
+* is in imperative form.
+* follows the Conventional Commits specification (https://www.conventionalcommits.org).
+  * See https://github.com/commitizen/conventional-commit-types/blob/master/index.json for possible
+    types.
+
 Example:
+
 fix: Don't add implicit wildcards ('*') at the beginning and the end of a query (fixes #390).
 -->
 
 # Description
+
 <!-- Describe what this request will change/fix and provide any details necessary for reviewers. -->
 
 
 
+# Checklist
+
+<!-- Ensure each item below is satisfied and indicate so by inserting an `x` within each `[ ]`. -->
+
+* [ ] The PR satisfies the [contribution guidelines][yscope-contrib-guidelines].
+* [ ] This is a breaking change and that has been indicated in the PR title, OR this isn't a
+  breaking change.
+* [ ] Necessary docs have been updated, OR no docs need to be updated.
+
 # Validation performed
+
 <!-- Describe what tests and validation you performed on the change. -->
 
+
+
+[yscope-contrib-guidelines]: https://docs.yscope.com/dev-guide/contrib-guides-overview.html

.gitmodules

@@ -29,3 +29,6 @@
 [submodule "components/log-viewer-webui/yscope-log-viewer"]
 	path = components/log-viewer-webui/yscope-log-viewer
 	url = https://github.com/y-scope/yscope-log-viewer.git
+[submodule "components/core/submodules/utfcpp"]
+	path = components/core/submodules/utfcpp
+	url = https://github.com/nemtrif/utfcpp.git

README.md

@@ -43,13 +43,14 @@ features:
   (intermediate representation) produced by CLP's logging libraries.
 
 - **Real-time Compression with CLP Logging Libraries**: CLP provides logging libraries for
-  [Python][clp-loglib-py] and Java ([Log4j][log4j1-appenders] and [Logback][logback-appenders]). The
-  logging libraries compress logs in real-time, so only compressed logs are written to disk or
-  transmitted over the network. The compressed logs use CLP's intermediate representation (IR)
-  format which achieves a higher compression ratio than general purpose compressors like Zstandard.
-  Compressing IR into archives can further double the compression ratio and enable global search,
-  but this requires more memory usage as it needs to buffer enough logs. More details on IR versus
-  archives can be found in this [Uber Engineering Blog][uber-blog].
+  [Python][clp-loglib-py] and Java ([Log4j1][log4j1-appenders], [Log4j2][log4j2-appenders] and 
+  [Logback][logback-appenders]). The logging libraries compress logs in real-time, so only
+  compressed logs are written to disk or transmitted over the network. The compressed logs use CLP's
+  intermediate representation (IR) format which achieves a higher compression ratio than general
+  purpose compressors like Zstandard. Compressing IR into archives can further double the
+  compression ratio and enable global search, but this requires more memory usage as it needs to
+  buffer enough logs. More details on IR versus archives can be found in this
+  [Uber Engineering Blog][uber-blog].
 
 - **[Log Viewer][log-viewer]**: the compressed IR can be viewed in a web-based log viewer. Compared
   to viewing the logs in an editor, CLP's log viewer supports advanced features like filtering logs
@@ -104,6 +105,7 @@ If you would like a feature or want to report a bug, please file an issue and we
 [log-surgeon]: https://github.com/y-scope/log-surgeon
 [log-viewer]: https://github.com/y-scope/yscope-log-viewer
 [log4j1-appenders]: https://github.com/y-scope/log4j1-appenders
+[log4j2-appenders]: https://github.com/y-scope/log4j2-appenders
 [logback-appenders]: https://github.com/y-scope/logback-appenders
 [re2]: https://github.com/google/re2
 [uber-blog]: https://www.uber.com/en-US/blog/reducing-logging-cost-by-two-orders-of-magnitude-using-clp

Taskfile.yml

@@ -4,7 +4,7 @@ includes:
   deps: "deps-tasks.yml"
   docs: "docs/tasks.yml"
   lint: "lint-tasks.yml"
-  utils: "tools/yscope-dev-utils/taskfiles/utils.yml"
+  utils: "tools/yscope-dev-utils/taskfiles/utils.yaml"
 
 vars:
   # Source paths
@@ -27,7 +27,7 @@ vars:
   G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui"
 
   # Taskfile paths
-  G_UTILS_TASKFILE: "{{.ROOT_DIR}}/tools/yscope-dev-utils/taskfiles/utils.yml"
+  G_UTILS_TASKFILE: "{{.ROOT_DIR}}/tools/yscope-dev-utils/taskfiles/utils.yaml"
 
   # Versions
   G_PACKAGE_VERSION: "0.2.0-dev"
@@ -116,7 +116,7 @@ tasks:
       - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          DATA_DIR: "{{.OUTPUT_DIR}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
       - "webui"
     cmds:
       - "rm -rf '{{.OUTPUT_DIR}}'"
@@ -173,8 +173,8 @@ tasks:
       # This command must be last
       - task: "utils:compute-checksum"
         vars:
-          DATA_DIR: "{{.OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
 
   core:
     vars:
@@ -242,7 +242,7 @@ tasks:
       - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          DATA_DIR: "{{.OUTPUT_DIR}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
     cmds:
       - "rm -rf '{{.OUTPUT_DIR}}'"
       - for:
@@ -254,8 +254,8 @@ tasks:
           --output-path "{{.OUTPUT_DIR}}/{{.ITEM}}"
       - task: "utils:compute-checksum"
         vars:
-          DATA_DIR: "{{.OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
 
   webui:
     vars:
@@ -280,7 +280,7 @@ tasks:
       - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          DATA_DIR: "{{.OUTPUT_DIR}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
       - "webui-node-modules"
     cmds:
       - "rm -rf '{{.OUTPUT_DIR}}'"
@@ -298,8 +298,8 @@ tasks:
       # This command must be last
       - task: "utils:compute-checksum"
         vars:
-          DATA_DIR: "{{.OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
 
   nodejs-22:
     internal: true
@@ -342,7 +342,7 @@ tasks:
       - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          DATA_DIR: "{{.OUTPUT_DIR}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
     cmds:
       - "rm -rf '{{.OUTPUT_DIR}}' '{{.OUTPUT_TMP_DIR}}'"
       - "mkdir -p '{{.OUTPUT_TMP_DIR}}'"
@@ -356,8 +356,8 @@ tasks:
       # This command must be last
       - task: "utils:compute-checksum"
         vars:
-          DATA_DIR: "{{.OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
 
   # NOTE: The log-viewer-webui has four different node_modules directories:
   # * client
@@ -407,19 +407,19 @@ tasks:
       - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CLIENT_CHECKSUM_FILE}}"
-          DATA_DIR: "{{.CLIENT_OUTPUT_DIR}}"
+          INCLUDE_PATTERNS: ["{{.CLIENT_OUTPUT_DIR}}"]
       - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.SERVER_CHECKSUM_FILE}}"
-          DATA_DIR: "{{.SERVER_OUTPUT_DIR}}"
+          INCLUDE_PATTERNS: ["{{.SERVER_OUTPUT_DIR}}"]
       - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.PACKAGE_CHECKSUM_FILE}}"
-          DATA_DIR: "{{.PACKAGE_OUTPUT_DIR}}"
+          INCLUDE_PATTERNS: ["{{.PACKAGE_OUTPUT_DIR}}"]
       - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.LOG_VIEWER_CHECKSUM_FILE}}"
-          DATA_DIR: "{{.LOG_VIEWER_OUTPUT_DIR}}"
+          INCLUDE_PATTERNS: ["{{.LOG_VIEWER_OUTPUT_DIR}}"]
     cmds:
       - "rm -f {{.CHECKSUM_FILE}}"
       - task: "clean-log-viewer-webui"
@@ -430,20 +430,20 @@ tasks:
       # These commands must be last
       - task: "utils:compute-checksum"
         vars:
-          DATA_DIR: "{{.CLIENT_OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.CLIENT_CHECKSUM_FILE}}"
+          CHECKSUM_FILE: "{{.CLIENT_CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.CLIENT_OUTPUT_DIR}}"]
       - task: "utils:compute-checksum"
         vars:
-          DATA_DIR: "{{.LOG_VIEWER_OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.LOG_VIEWER_CHECKSUM_FILE}}"
+          CHECKSUM_FILE: "{{.LOG_VIEWER_CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.LOG_VIEWER_OUTPUT_DIR}}"]
       - task: "utils:compute-checksum"
         vars:
-          DATA_DIR: "{{.PACKAGE_OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.PACKAGE_CHECKSUM_FILE}}"
+          CHECKSUM_FILE: "{{.PACKAGE_CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.PACKAGE_OUTPUT_DIR}}"]
       - task: "utils:compute-checksum"
         vars:
-          DATA_DIR: "{{.SERVER_OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.SERVER_CHECKSUM_FILE}}"
+          CHECKSUM_FILE: "{{.SERVER_CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.SERVER_OUTPUT_DIR}}"]
       # This command must be last
       - >-
         cat
@@ -550,18 +550,18 @@ tasks:
       - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          DATA_DIR: "{{.OUTPUT_DIR}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
     cmds:
       - task: "utils:create-venv"
         vars:
           LABEL: "package"
           OUTPUT_DIR: "{{.OUTPUT_DIR}}"
-          REQUIREMENTS_FILE: "requirements.txt"
+          REQUIREMENTS_FILE: "{{.ROOT_DIR}}/requirements.txt"
       # This command must be last
       - task: "utils:compute-checksum"
         vars:
-          DATA_DIR: "{{.OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
 
   python-component:
     internal: true
@@ -614,15 +614,15 @@ tasks:
       - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          DATA_DIR: "{{.OUTPUT_DIR}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
     cmds:
       - "rm -rf '{{.OUTPUT_DIR}}'"
       - "PATH='{{.G_METEOR_BUILD_DIR}}':$PATH meteor npm install --production"
       # This command must be last
       - task: "utils:compute-checksum"
         vars:
-          DATA_DIR: "{{.OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
 
   component-venv:
     internal: true
@@ -643,7 +643,7 @@ tasks:
       - task: "utils:validate-checksum"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          DATA_DIR: "{{.OUTPUT_DIR}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
     cmds:
       - task: "utils:create-venv"
         vars:
@@ -653,8 +653,8 @@ tasks:
       # This command must be last
       - task: "utils:compute-checksum"
         vars:
-          DATA_DIR: "{{.OUTPUT_DIR}}"
-          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
   clean-python-component:
     internal: true
     label: "clean-{{.COMPONENT}}"

components/clp-package-utils/clp_package_utils/general.py

@@ -20,7 +20,6 @@
     REDIS_COMPONENT_NAME,
     REDUCER_COMPONENT_NAME,
     RESULTS_CACHE_COMPONENT_NAME,
-    StorageType,
     WEBUI_COMPONENT_NAME,
     WorkerConfig,
 )
@@ -254,17 +253,17 @@ def generate_container_config(
             container_clp_config.archive_output.get_directory(),
         )
 
-    container_clp_config.stream_output.directory = pathlib.Path("/") / "mnt" / "stream-output"
+    container_clp_config.stream_output.set_directory(pathlib.Path("/") / "mnt" / "stream-output")
     if not is_path_already_mounted(
         clp_home,
         CONTAINER_CLP_HOME,
-        clp_config.stream_output.directory,
-        container_clp_config.stream_output.directory,
+        clp_config.stream_output.get_directory(),
+        container_clp_config.stream_output.get_directory(),
     ):
         docker_mounts.stream_output_dir = DockerMount(
             DockerMountType.BIND,
-            clp_config.stream_output.directory,
-            container_clp_config.stream_output.directory,
+            clp_config.stream_output.get_directory(),
+            container_clp_config.stream_output.get_directory(),
         )
 
     return container_clp_config, docker_mounts
@@ -276,7 +275,7 @@ def generate_worker_config(clp_config: CLPConfig) -> WorkerConfig:
     worker_config.archive_output = clp_config.archive_output.copy(deep=True)
     worker_config.data_directory = clp_config.data_directory
 
-    worker_config.stream_output_dir = clp_config.stream_output.directory
+    worker_config.stream_output = clp_config.stream_output
     worker_config.stream_collection_name = clp_config.results_cache.stream_collection_name
 
     return worker_config