keep track of number of forsaken attempts

Author: btovar

taskvine/src/bindings/python3/ndcctools/taskvine/task.py

@@ -423,10 +423,18 @@ def add_environment(self, f):
     # Indicate the number of times the task should be retried. If 0 (the
     # default), the task is tried indefinitely. A task that did not succeed
     # after the given number of retries is returned with result
-    # "result_max_retries".
+    # "max retries".
     def set_retries(self, max_retries):
         return cvine.vine_task_set_retries(self._task, max_retries)
 
+    ##
+    # Indicate the number of times the task can be returned to the manager
+    # without being executed. If 0 default), the task is tried indefinitely.
+    # A task that did not succeed after the given number of retries is returned
+    # with result "forsaken".
+    def set_max_forsaken(self, max_forsaken):
+        return cvine.vine_task_set_max_forsaken(self._task, max_forsaken)
+
     ##
     # Indicate the number of cores required by this task.
     def set_cores(self, cores):

taskvine/src/manager/taskvine.h

@@ -285,13 +285,20 @@ int vine_task_add_input( struct vine_task *t, struct vine_file *f, const char *r
 
 int vine_task_add_output( struct vine_task *t, struct vine_file *f, const char *remote_name, vine_mount_flags_t flags );
 
-/** Specify the number of times this task is retried on worker errors. If less than one, the task is retried indefinitely (this the default). A task that did not succeed after the given number of retries is returned with result VINE_RESULT_MAX_RETRIES.
+/** Specify the number of times this task is retried on worker errors. If less than one, the task is retried indefinitely (this the default). A task that did not succeed after the given number of retries is returned with the result of its last attempt.
 @param t A task object.
 @param max_retries The number of retries.
 */
 
 void vine_task_set_retries( struct vine_task *t, int64_t max_retries );
 
+/** Specify the total number of times this task can be return to the manager without being executed. If less than zero, the task is tried indefinitely (this the default). A task that did not succeed after the given number is returned with the result VINE_RESULT_FORSAKEN.
+@param t A task object.
+@param max_retries The number of retries.
+*/
+
+void vine_task_set_max_forsaken( struct vine_task *t, int64_t max_forsaken );
+
 /** Specify the amount of disk space required by a task.
 @param t A task object.
 @param memory The amount of disk space required by the task, in megabytes.

taskvine/src/manager/vine_manager.c

@@ -1538,6 +1538,7 @@ static vine_result_code_t get_result(struct vine_manager *q, struct vine_worker_
 
 	/* If the task was forsaken by the worker or couldn't exeute, it didn't really complete, so short circuit. */
 	if (task_status == VINE_RESULT_FORSAKEN) {
+		t->forsaken_count++;
 		itable_remove(q->running_table, t->task_id);
 		vine_task_set_result(t, task_status);
 		change_task_state(q, t, VINE_TASK_WAITING_RETRIEVAL);
@@ -2795,10 +2796,12 @@ static int resubmit_task_on_exhaustion(struct vine_manager *q, struct vine_worke
 static int resubmit_if_needed(struct vine_manager *q, struct vine_worker_info *w, struct vine_task *t)
 {
 	/* in this function, any change_task_state should only be to VINE_TASK_READY */
-
 	if (t->result == VINE_RESULT_FORSAKEN) {
-		/* forsaken tasks are always resubmitted. they also get a retry back as they are victims of circumstance
-		 */
+		if (t->max_forsaken > -1 && t->forsaken_count > t->max_forsaken) {
+			return 0;
+		}
+
+		/* forsaken tasks get a retry back as they are victims of circumstance */
 		t->try_count -= 1;
 		change_task_state(q, t, VINE_TASK_READY);
 		return 1;
@@ -2810,8 +2813,7 @@ static int resubmit_if_needed(struct vine_manager *q, struct vine_worker_info *w
 	}
 
 	/* special handlings per result. note that most results are terminal, that is tasks are not retried even if they
-	 * have not reached max_retries. max_retries is only used for transient errors, or for modified tasks (such as a
-	 * change in the resource request). */
+	 * have not reached max_retries. */
 	switch (t->result) {
 	case VINE_RESULT_RESOURCE_EXHAUSTION:
 		return resubmit_task_on_exhaustion(q, w, t);
@@ -3125,7 +3127,6 @@ static int send_one_task(struct vine_manager *q)
 
 	timestamp_t now_usecs = timestamp_get();
 	double now_secs = ((double)now_usecs) / ONE_SECOND;
-	timestamp_t time_failure_range = now_usecs - q->transient_error_interval;
 
 	int tasks_to_consider = MIN(list_size(q->ready_list), q->attempt_schedule_depth);
 
@@ -3140,7 +3141,7 @@ static int send_one_task(struct vine_manager *q)
 		}
 
 		// Skip if this task failed recently
-		if (time_failure_range > t->time_when_last_failure) {
+		if (t->time_when_last_failure + q->transient_error_interval > now_usecs) {
 			continue;
 		}
 

taskvine/src/manager/vine_task.c

@@ -129,6 +129,7 @@ void vine_task_reset(struct vine_task *t)
 
 	t->resource_request = CATEGORY_ALLOCATION_FIRST;
 	t->try_count = 0;
+	t->forsaken_count = 0;
 	t->exhausted_attempts = 0;
 	t->workers_slow = 0;
 
@@ -226,6 +227,7 @@ struct vine_task *vine_task_copy(const struct vine_task *task)
 	vine_task_set_scheduler(new, task->worker_selection_algorithm);
 	vine_task_set_priority(new, task->priority);
 	vine_task_set_retries(new, task->max_retries);
+	vine_task_set_max_forsaken(new, task->max_forsaken);
 	vine_task_set_time_min(new, task->min_running_time);
 
 	/* Internal state of task is cleared from vine_task_create */
@@ -306,6 +308,15 @@ void vine_task_set_retries(struct vine_task *t, int64_t max_retries)
 	}
 }
 
+void vine_task_set_max_forsaken(struct vine_task *t, int64_t max_forsaken)
+{
+	if (max_forsaken < 0) {
+		t->max_retries = -1;
+	} else {
+		t->max_forsaken = max_forsaken;
+	}
+}
+
 void vine_task_set_memory(struct vine_task *t, int64_t memory)
 {
 	if (memory < 0) {

taskvine/src/manager/vine_task.h

@@ -61,17 +61,20 @@ struct vine_task {
 	vine_schedule_t worker_selection_algorithm; /**< How to choose worker to run the task. */
 	double priority;             /**< The priority of this task relative to others in the queue: higher number run earlier. */
 	int max_retries;             /**< Number of times the task is tried to be executed on some workers until success. If less than one, the task is retried indefinitely. See try_count below.*/
+	int max_forsaken;            /**< Number of times the task is submitted to workers without being executed. If less than one, the task is retried indefinitely. See forsaken_count below.*/
 	int64_t min_running_time;    /**< Minimum time (in seconds) the task needs to run. (see vine_worker --wall-time)*/
 
 	/***** Internal state of task as it works towards completion. *****/
 
 	vine_task_state_t state;       /**< Current state of task: READY, RUNNING, etc */
 	struct vine_worker_info *worker;    /**< Worker to which this task has been dispatched. */
         struct vine_task* library_task; /**< Library task to which a function task has been matched. */
-	int try_count;               /**< The number of times the task has been dispatched to a worker. If larger than max_retries, the task failes with @ref VINE_RESULT_MAX_RETRIES. */
-	int exhausted_attempts;      /**< Number of times the task failed given exhausted resources. */
-	int workers_slow;            /**< Number of times this task has been terminated for running too long. */
-	int function_slots_inuse;    /**< If a library, the number of functions currently running. */
+	int try_count;               /**< The number of times the task has been dispatched to a worker without being forsaken. If larger than max_retries, return with result of last attempt. */
+	int forsaken_count;         /**< The number of times the task has been dispatched to a worker. If larger than max_forsaken, return with VINE_RESULT_FORSAKEN. */
+	int exhausted_attempts;     /**< Number of times the task failed given exhausted resources. */
+	int forsaken_attempts;      /**< Number of times the task was submitted to a worker but failed to start execution. */
+	int workers_slow;           /**< Number of times this task has been terminated for running too long. */
+	int function_slots_inuse;   /**< If a library, the number of functions currently running. */
 
 	/***** Results of task once it has reached completion. *****/
 

taskvine/test/vine_python.py

@@ -203,7 +203,7 @@ def next_output_name():
     report_task(t, "success", 0)
 
     # should fail in the alloted time
-    t = vine.Task("/bin/sleep 10")
+    t = vine.Task("/bin/sleep 100")
     t.set_time_max(1)
     q.submit(t)
     t = q.wait(wait_time)
@@ -266,13 +266,13 @@ def next_output_name():
     report_task(t, "success", 0)
 
     # generate an invalid remote input file, should get an input missing error.
-    t = vine.Task("wc -l infile")
-    t.set_retries(1)
+    t = vine.Task("wc -l infile_for_forsaken")
+    t.set_max_forsaken(1)
     url = q.declare_url("https://pretty-sure-this-is-not-a-valid-url.com")
-    t.add_input(url, "infile")
+    t.add_input(url, "infile_for_forsaken")
     q.submit(t)
     t = q.wait(wait_time)
-    report_task(t, "transfer missing", 1)
+    report_task(t, "forsaken", -1)
 
     # create a temporary output file, and then fetch its contents manually.
     t = vine.Task("echo howdy > output")