diff --git a/taskvine/src/bindings/python3/ndcctools/taskvine/display.py b/taskvine/src/bindings/python3/ndcctools/taskvine/display.py index 95da12d924..db174298d1 100644 --- a/taskvine/src/bindings/python3/ndcctools/taskvine/display.py +++ b/taskvine/src/bindings/python3/ndcctools/taskvine/display.py @@ -110,6 +110,7 @@ def generate_manager_table(self, manager_s): "tasks_done", "tasks_waiting", "tasks_running", + "recovery_tasks_submitted", "tasks_exhausted_attempts", "workers_connected", "workers_busy", diff --git a/taskvine/src/manager/taskvine.h b/taskvine/src/manager/taskvine.h index 63e3e0369d..bc3f2f7ac8 100644 --- a/taskvine/src/manager/taskvine.h +++ b/taskvine/src/manager/taskvine.h @@ -176,6 +176,7 @@ struct vine_stats { int tasks_on_workers; /**< Number of tasks currently dispatched to some worker. */ int tasks_running; /**< Number of tasks currently executing at some worker. */ int tasks_with_results; /**< Number of tasks with retrieved results and waiting to be returned to user. */ + int recovery_tasks_submitted; /**< Total number of recovery tasks submitted since the manager started. */ /* Cumulative stats for tasks: */ int tasks_submitted; /**< Total number of tasks submitted to the manager. */ diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index 44bb1f0427..f385fbab06 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -2234,6 +2234,7 @@ static struct jx *manager_to_jx(struct vine_manager *q) jx_insert_integer(j, "tasks_on_workers", info.tasks_on_workers); jx_insert_integer(j, "tasks_running", info.tasks_running); jx_insert_integer(j, "tasks_with_results", info.tasks_with_results); + jx_insert_integer(j, "recovery_tasks_submitted", info.recovery_tasks_submitted); jx_insert_integer(j, "tasks_left", q->num_tasks_left); jx_insert_integer(j, "tasks_submitted", info.tasks_submitted); @@ -3490,7 +3491,7 @@ static void vine_manager_consider_recovery_task(struct vine_manager *q, struct v case VINE_TASK_INITIAL: /* The recovery task has never been run, so submit it now. */ vine_submit(q, rt); - notice(D_VINE, "Submitted recovery task %d (%s) to re-create lost temporary file %s.", rt->task_id, rt->command_line, lost_file->cached_name); + debug(D_VINE, "Submitted recovery task %d (%s) to re-create lost temporary file %s.", rt->task_id, rt->command_line, lost_file->cached_name); break; case VINE_TASK_READY: case VINE_TASK_RUNNING: @@ -3504,7 +3505,7 @@ static void vine_manager_consider_recovery_task(struct vine_manager *q, struct v * here. */ vine_task_reset(rt); vine_submit(q, rt); - notice(D_VINE, "Submitted recovery task %d (%s) to re-create lost temporary file %s.", rt->task_id, rt->command_line, lost_file->cached_name); + debug(D_VINE, "Submitted recovery task %d (%s) to re-create lost temporary file %s.", rt->task_id, rt->command_line, lost_file->cached_name); break; } } @@ -4841,6 +4842,10 @@ int vine_submit(struct vine_manager *q, struct vine_task *t) /* Issue warnings if the files are set up strangely. */ vine_task_check_consistency(t); + if (t->type == VINE_TASK_TYPE_RECOVERY) { + q->stats->recovery_tasks_submitted++; + } + if (t->has_fixed_locations) { q->fixed_location_in_queue++; vine_task_set_scheduler(t, VINE_SCHEDULE_FILES);