vine: reserve a factor of disk when allocating resources (#4035)

* init

* tune param

* lint

* lint

* lint

* condition cahnge: value > 1 || value <= 0

* reserve disk not only for proportional

* lint

* only modify disk factor when in range (0,1)

---------

Co-authored-by: Benjamin Tovar <[email protected]>

Author: JinZhou5042

taskvine/src/manager/vine_manager.c

@@ -2771,6 +2771,11 @@ struct rmsummary *vine_manager_choose_resources_for_task(struct vine_manager *q,
 		limits->disk = available_disk;
 	}
 
+	/* For disk, scale the estimated disk allocation by a [0, 1] factor (by default 0.75) to intentionally
+	 * reserve some space for data movement between the sandbox and cache, and allow extra room for potential cache growth.
+	 * This applies to tasks except function calls. */
+	limits->disk *= q->disk_proportion_available_to_task;
+
 	/* never go below specified min resources. */
 	rmsummary_merge_max(limits, min);
 
@@ -4006,6 +4011,7 @@ struct vine_manager *vine_ssl_create(int port, const char *key, const char *cert
 	q->max_task_resources_requested = rmsummary_create(-1);
 
 	q->sandbox_grow_factor = 2.0;
+	q->disk_proportion_available_to_task = 0.75;
 
 	q->stats = calloc(1, sizeof(struct vine_stats));
 	q->stats_measure = calloc(1, sizeof(struct vine_stats));
@@ -5772,7 +5778,10 @@ int vine_tune(struct vine_manager *q, const char *name, double value)
 
 	} else if (!strcmp(name, "max-library-retries")) {
 		q->max_library_retries = MIN(1, value);
-
+	} else if (!strcmp(name, "disk-proportion-available-to-task")) {
+		if (value < 1 && value > 0) {
+			q->disk_proportion_available_to_task = value;
+		}
 	} else {
 		debug(D_NOTICE | D_VINE, "Warning: tuning parameter \"%s\" not recognized\n", name);
 		return -1;

taskvine/src/manager/vine_manager.h

@@ -229,6 +229,7 @@ struct vine_manager {
 	int watch_library_logfiles;     /* If true, watch the output files produced by each of the library processes running on the remote workers, take them back the current logging directory */
 
 	double sandbox_grow_factor;         /* When task disk sandboxes are exhausted, increase the allocation using their measured valued times this factor */
+	double disk_proportion_available_to_task;   /* intentionally reduces disk allocation for tasks to reserve some space for cache growth. */
 
 	/*todo: confirm datatype. int or int64*/
 	int max_task_stdout_storage;	/* Maximum size of standard output from task.  (If larger, send to a separate file.) */

taskvine/test/vine_allocations.py

@@ -65,27 +65,32 @@ def check_task(category, category_mode, max, min, expected):
 with worker:
     q.tune("force-proportional-resources", 0)
 
+    # note that the disk is divided by a factor to reserve space for cache growth unless the users specify the disk
+    disk_proportion_available_to_task = 0.75
+    q.tune("disk-proportion-available-to-task", disk_proportion_available_to_task)  # the default factor is 0.75
+
     r = {"cores": 1, "memory": 2, "disk": 3, "gpus": 4}
     check_task("all_specified", "fixed", max=r, min={}, expected=r)
 
     check_task("all_specified_no_gpu", "fixed", max={"cores": 1, "memory": 2, "disk": 3}, min={}, expected={"cores": 1, "memory": 2, "disk": 3, "gpus": 0})
 
     check_task("all_specified_no_cores", "fixed", max={"gpus": 4, "memory": 2, "disk": 3}, min={}, expected={"cores": 0, "memory": 2, "disk": 3, "gpus": 4})
 
-    check_task("all_zero", "fixed", max={"cores": 0, "memory": 0, "disk": 0, "gpus": 0}, min={}, expected={"cores": worker_cores, "memory": worker_memory, "disk": worker_disk, "gpus": 0})
+    check_task("all_zero", "fixed", max={"cores": 0, "memory": 0, "disk": 0, "gpus": 0}, min={}, expected={"cores": worker_cores, "memory": worker_memory, "disk": worker_disk * disk_proportion_available_to_task, "gpus": 0})
 
     q.tune("force-proportional-resources", 1)
-    check_task("only_memory", "fixed", max={"memory": worker_memory / 2}, min={}, expected={"cores": worker_cores / 2, "memory": worker_memory / 2, "disk": worker_disk / 2, "gpus": 0})
 
-    check_task("only_memory_w_minimum", "fixed", max={"memory": worker_memory / 2}, min={"cores": 3, "gpus": 2}, expected={"cores": 4, "memory": worker_memory, "disk": worker_disk, "gpus": 2})
+    check_task("only_memory", "fixed", max={"memory": worker_memory / 2}, min={}, expected={"cores": worker_cores / 2, "memory": worker_memory / 2, "disk": worker_disk / 2 * disk_proportion_available_to_task, "gpus": 0})
+
+    check_task("only_memory_w_minimum", "fixed", max={"memory": worker_memory / 2}, min={"cores": 3, "gpus": 2}, expected={"cores": 4, "memory": worker_memory, "disk": worker_disk * disk_proportion_available_to_task, "gpus": 2})
 
-    check_task("only_cores", "fixed", max={"cores": worker_cores}, min={}, expected={"cores": worker_cores, "memory": worker_memory, "disk": worker_disk, "gpus": 0})
+    check_task("only_cores", "fixed", max={"cores": worker_cores}, min={}, expected={"cores": worker_cores, "memory": worker_memory, "disk": worker_disk * disk_proportion_available_to_task, "gpus": 0})
 
-    check_task("auto_whole_worker", "min_waste", max={}, min={}, expected={"cores": worker_cores, "memory": worker_memory, "disk": worker_disk, "gpus": 0})
+    check_task("auto_whole_worker", "min_waste", max={}, min={}, expected={"cores": worker_cores, "memory": worker_memory, "disk": worker_disk * disk_proportion_available_to_task, "gpus": 0})
 
     p = 1 / worker_cores
     r = {"cores": 1}
-    e = {"cores": 1, "memory": math.floor(worker_memory * p), "disk": math.floor(worker_disk * p), "gpus": 0}
+    e = {"cores": 1, "memory": math.floor(worker_memory * p), "disk": math.floor(worker_disk * p) * disk_proportion_available_to_task, "gpus": 0}
     check_task("only_cores_proportional", "fixed", max=r, min={}, expected=e)
 
     p = 2 / worker_cores