Add data_checkpointing.

Signed-off-by: Revital Sur <[email protected]>
IBM · Oct 13, 2024 · a1f6059 · a1f6059
1 parent d21f0f1
commit a1f6059
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 0 deletions.
diff --git a/transforms/universal/hap/kfp_ray/hap_wf.py b/transforms/universal/hap/kfp_ray/hap_wf.py
@@ -36,6 +36,7 @@ def compute_exec_params_func(
     data_s3_config: str,
     data_max_files: int,
     data_num_samples: int,
+    data_checkpointing: bool,
     runtime_pipeline_id: str,
     runtime_job_id: str,
     runtime_code_location: dict,
@@ -52,6 +53,7 @@ def compute_exec_params_func(
         "data_s3_config": data_s3_config,
         "data_max_files": data_max_files,
         "data_num_samples": data_num_samples,
+        "data_checkpointing": data_checkpointing,
         "runtime_num_workers": KFPUtils.default_compute_execution_params(str(worker_options), str(actor_options)),
         "runtime_worker_options": str(actor_options),
         "runtime_pipeline_id": runtime_pipeline_id,
@@ -193,6 +195,7 @@ def hap(
             data_s3_config=data_s3_config,
             data_max_files=data_max_files,
             data_num_samples=data_num_samples,
+            data_checkpointing=data_checkpointing,
             runtime_pipeline_id=runtime_pipeline_id,
             runtime_job_id=run_id,
             runtime_code_location=runtime_code_location,

diff --git a/transforms/universal/hap/kfp_ray/pipeline_definitions.yaml b/transforms/universal/hap/kfp_ray/pipeline_definitions.yaml
@@ -12,6 +12,7 @@ pipeline_common_input_parameters_values:
     kfp_base_image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
     transform_image: "quay.io/dataprep1/data-prep-kit/hap-ray:latest"
     s3_access_secret: "s3-secret"
+    image_pull_secret: ""
     input_folder: "test/hap/input/"
     output_folder: "test/hap/output/"