Merge pull request #92 from revit13/fixes2

Fix malware and code_quality workflows.
IBM · May 9, 2024 · 8715ae6 · 8715ae6
2 parents c9466c1 + 901dbb1
commit 8715ae6
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 15 deletions.
diff --git a/kfp/transform_workflows/.make.transforms_workflows b/kfp/transform_workflows/.make.transforms_workflows
@@ -13,7 +13,7 @@ endef
 
 .PHONY: .transforms_workflows.reconcile-requirement
 .transforms_workflows.reconcile-requirement:
-	while IFS= read -r line; do \
+	@while IFS= read -r line; do \
 		[ -z "$$line" ] && continue; \
 		[[ $$line == *#* ]] && continue; \
 		export DOCKER_IMAGE_NAME=$$(echo $$line |cut -d "=" -f 1 |sed "s/_VERSION//" |tr '[:upper:]' '[:lower:]'); \

diff --git a/kfp/transform_workflows/code/code_quality/code_quality_wf.py b/kfp/transform_workflows/code/code_quality/code_quality_wf.py
@@ -24,7 +24,7 @@
 
 
 # the name of the job script
-EXEC_SCRIPT_NAME: str = "cq_transform.py"
+EXEC_SCRIPT_NAME: str = "code_quality_transform.py"
 PREFIX: str = ""
 
 task_image = "quay.io/dataprep1/data-prep-lab/code_quality:0.2.0"
@@ -69,9 +69,9 @@ def code_quality(
     data_max_files: int = -1,
     data_num_samples: int = -1,
     # orchestrator
-    actor_options: str = "{'num_cpus': 0.8}",
-    pipeline_id: str = "pipeline_id",
-    code_location: str = "{'github': 'github', 'commit_hash': '12345', 'path': 'path'}",
+    runtime_actor_options: str = "{'num_cpus': 0.8}",
+    runtime_pipeline_id: str = "runtime_pipeline_id",
+    runtime_code_location: str = "{'github': 'github', 'commit_hash': '12345', 'path': 'path'}",
     # code quality parameters
     cq_contents_column_name: str = "contents",
     cq_language_column_name: str = "language",
@@ -108,8 +108,8 @@ def code_quality(
     :param data_s3_config - s3 configuration
     :param data_max_files - max files to process
     :param data_num_samples - num samples to process
-    :param actor_options - actor options
-    :param pipeline_id - pipeline id
+    :param runtime_actor_options - actor options
+    :param runtime_pipeline_id - pipeline id
     :param cq_contents_column_name - Name of the column holds the data to process
     :param cq_language_column_name - Name of the column holds the programming language details
     :param cq_tokenizer - Name or path to the tokenizer
@@ -124,7 +124,7 @@ def code_quality(
         # compute execution params
         compute_exec_params = compute_exec_params_op(
             worker_options=ray_worker_options,
-            actor_options=actor_options,
+            actor_options=runtime_actor_options,
         )
         ComponentUtils.add_settings_to_component(compute_exec_params, ONE_HOUR_SEC * 2)
         # start Ray cluster
@@ -148,10 +148,10 @@ def code_quality(
                 "data_s3_config": data_s3_config,
                 "data_max_files": data_max_files,
                 "data_num_samples": data_num_samples,
-                "num_workers": compute_exec_params.output,
-                "worker_options": actor_options,
-                "pipeline_id": pipeline_id,
-                "job_id": dsl.RUN_ID_PLACEHOLDER,
+                "runtime_num_workers": compute_exec_params.output,
+                "runtime_worker_options": runtime_actor_options,
+                "runtime_pipeline_id": runtime_pipeline_id,
+                "runtime_job_id": dsl.RUN_ID_PLACEHOLDER,
                 "cq_contents_column_name": cq_contents_column_name,
                 "cq_language_column_name": cq_language_column_name,
                 "cq_tokenizer": cq_tokenizer,

diff --git a/kfp/transform_workflows/code/malware/malware_wf.py b/kfp/transform_workflows/code/malware/malware_wf.py
@@ -65,7 +65,6 @@ def malware(
     # mallware
     malware_input_column: str = "contents",
     malware_output_column: str = "virus_detection",
-    malware_clamd_socket: str = "/var/run/clamav/clamd.ctl",
     # additional parameters
     additional_params: str = '{"wait_interval": 2, "wait_cluster_ready_tmout": 400, "wait_cluster_up_tmout": 300, "wait_job_ready_tmout": 400, "wait_print_tmout": 30, "http_retries": 5}',
 ):
@@ -102,7 +101,6 @@ def malware(
     :param runtime_code_location - code location
     :param malware_input_column - input column name
     :param malware_output_column - output column name
-    :param malware_clamd_socket - local socket path for clamd
     :return: None
     """
     # create clean_up task
@@ -144,7 +142,6 @@ def malware(
                 "runtime_code_location": runtime_code_location,
                 "malware_input_column": malware_input_column,
                 "malware_output_column": malware_output_column,
-                "malware_clamd_socket": malware_clamd_socket,
             },
             exec_script_name=EXEC_SCRIPT_NAME,
             server_url=server_url,