From e73c39161c51fb5f6c6f75504283f8c785e044d8 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Thu, 9 May 2024 10:04:28 +0300 Subject: [PATCH 1/2] Fix malware and code_quality workflows. Signed-off-by: Revital Sur Co-authored-by: Mohammad Nassar --- .../code/code_quality/code_quality_wf.py | 22 +++++++++---------- .../code/malware/malware_wf.py | 3 --- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/kfp/transform_workflows/code/code_quality/code_quality_wf.py b/kfp/transform_workflows/code/code_quality/code_quality_wf.py index 540b66880..c24368129 100644 --- a/kfp/transform_workflows/code/code_quality/code_quality_wf.py +++ b/kfp/transform_workflows/code/code_quality/code_quality_wf.py @@ -24,7 +24,7 @@ # the name of the job script -EXEC_SCRIPT_NAME: str = "cq_transform.py" +EXEC_SCRIPT_NAME: str = "code_quality_transform.py" PREFIX: str = "" task_image = "quay.io/dataprep1/data-prep-lab/code_quality:0.2.0" @@ -69,9 +69,9 @@ def code_quality( data_max_files: int = -1, data_num_samples: int = -1, # orchestrator - actor_options: str = "{'num_cpus': 0.8}", - pipeline_id: str = "pipeline_id", - code_location: str = "{'github': 'github', 'commit_hash': '12345', 'path': 'path'}", + runtime_actor_options: str = "{'num_cpus': 0.8}", + runtime_pipeline_id: str = "runtime_pipeline_id", + runtime_code_location: str = "{'github': 'github', 'commit_hash': '12345', 'path': 'path'}", # code quality parameters cq_contents_column_name: str = "contents", cq_language_column_name: str = "language", @@ -108,8 +108,8 @@ def code_quality( :param data_s3_config - s3 configuration :param data_max_files - max files to process :param data_num_samples - num samples to process - :param actor_options - actor options - :param pipeline_id - pipeline id + :param runtime_actor_options - actor options + :param runtime_pipeline_id - pipeline id :param cq_contents_column_name - Name of the column holds the data to process :param cq_language_column_name - Name of the column holds the programming language details :param cq_tokenizer - Name or path to the tokenizer @@ -124,7 +124,7 @@ def code_quality( # compute execution params compute_exec_params = compute_exec_params_op( worker_options=ray_worker_options, - actor_options=actor_options, + actor_options=runtime_actor_options, ) ComponentUtils.add_settings_to_component(compute_exec_params, ONE_HOUR_SEC * 2) # start Ray cluster @@ -148,10 +148,10 @@ def code_quality( "data_s3_config": data_s3_config, "data_max_files": data_max_files, "data_num_samples": data_num_samples, - "num_workers": compute_exec_params.output, - "worker_options": actor_options, - "pipeline_id": pipeline_id, - "job_id": dsl.RUN_ID_PLACEHOLDER, + "runtime_num_workers": compute_exec_params.output, + "runtime_worker_options": runtime_actor_options, + "runtime_pipeline_id": runtime_pipeline_id, + "runtime_job_id": dsl.RUN_ID_PLACEHOLDER, "cq_contents_column_name": cq_contents_column_name, "cq_language_column_name": cq_language_column_name, "cq_tokenizer": cq_tokenizer, diff --git a/kfp/transform_workflows/code/malware/malware_wf.py b/kfp/transform_workflows/code/malware/malware_wf.py index 190ee2916..20850b415 100644 --- a/kfp/transform_workflows/code/malware/malware_wf.py +++ b/kfp/transform_workflows/code/malware/malware_wf.py @@ -65,7 +65,6 @@ def malware( # mallware malware_input_column: str = "contents", malware_output_column: str = "virus_detection", - malware_clamd_socket: str = "/var/run/clamav/clamd.ctl", # additional parameters additional_params: str = '{"wait_interval": 2, "wait_cluster_ready_tmout": 400, "wait_cluster_up_tmout": 300, "wait_job_ready_tmout": 400, "wait_print_tmout": 30, "http_retries": 5}', ): @@ -102,7 +101,6 @@ def malware( :param runtime_code_location - code location :param malware_input_column - input column name :param malware_output_column - output column name - :param malware_clamd_socket - local socket path for clamd :return: None """ # create clean_up task @@ -144,7 +142,6 @@ def malware( "runtime_code_location": runtime_code_location, "malware_input_column": malware_input_column, "malware_output_column": malware_output_column, - "malware_clamd_socket": malware_clamd_socket, }, exec_script_name=EXEC_SCRIPT_NAME, server_url=server_url, From 901dbb1ee1ee7eba96dea4e6a30e3d89396e7976 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Thu, 9 May 2024 10:21:09 +0300 Subject: [PATCH 2/2] Minor fix. Signed-off-by: Revital Sur --- kfp/transform_workflows/.make.transforms_workflows | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kfp/transform_workflows/.make.transforms_workflows b/kfp/transform_workflows/.make.transforms_workflows index fcc276ed6..e6d4d388b 100644 --- a/kfp/transform_workflows/.make.transforms_workflows +++ b/kfp/transform_workflows/.make.transforms_workflows @@ -13,7 +13,7 @@ endef .PHONY: .transforms_workflows.reconcile-requirement .transforms_workflows.reconcile-requirement: - while IFS= read -r line; do \ + @while IFS= read -r line; do \ [ -z "$$line" ] && continue; \ [[ $$line == *#* ]] && continue; \ export DOCKER_IMAGE_NAME=$$(echo $$line |cut -d "=" -f 1 |sed "s/_VERSION//" |tr '[:upper:]' '[:lower:]'); \