Skip to content

Commit 3df9700

Browse files
LuiggiTenorioKLuiggiTenorioK
authored
Optimizing legacy code (#265)
1 parent 5bb8d89 commit 3df9700

1 file changed

Lines changed: 114 additions & 73 deletions

File tree

‎autosubmit_api/autosubmit_legacy/job/job_list.py‎

Lines changed: 114 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -17,35 +17,47 @@
1717
# You should have received a copy of the GNU General Public License
1818
# along with Autosubmit. If not, see <http://www.gnu.org/licenses/>.
1919

20-
import os
21-
import traceback
2220
import datetime
2321
import math
22+
import os
23+
import traceback
2424

2525
# Spectral imports
2626
# End Spectral imports
27+
from time import mktime, time
28+
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
2729

28-
from time import time, mktime
30+
from bscearth.utils.date import date2str
2931
from dateutil.relativedelta import relativedelta
3032

31-
from autosubmit_api.autosubmit_legacy.job.job_utils import SubJob
32-
from autosubmit_api.autosubmit_legacy.job.job_utils import SubJobManager, job_times_to_text
33-
from autosubmit_api.config.basicConfig import APIBasicConfig
34-
from autosubmit_api.performance.utils import calculate_ASYPD_perjob, calculate_SYPD_perjob
35-
from autosubmit_api.components.jobs import utils as JUtils
36-
from autosubmit_api.monitor.monitor import Monitor
33+
from autosubmit_api.autosubmit_legacy.job.job_utils import (
34+
SubJob,
35+
SubJobManager,
36+
job_times_to_text,
37+
)
38+
from autosubmit_api.builders.experiment_history_builder import (
39+
ExperimentHistoryBuilder,
40+
ExperimentHistoryDirector,
41+
)
3742
from autosubmit_api.common.utils import Status
38-
from bscearth.utils.date import date2str
43+
from autosubmit_api.components.jobs import utils as JUtils
44+
from autosubmit_api.config.basicConfig import APIBasicConfig
45+
3946
# from autosubmit_legacy.job.tree import Tree
4047
from autosubmit_api.database import db_structure as DbStructure
4148
from autosubmit_api.database.db_jobdata import JobDataStructure, JobRow
42-
from autosubmit_api.builders.experiment_history_builder import ExperimentHistoryDirector, ExperimentHistoryBuilder
4349
from autosubmit_api.history.data_classes.job_data import JobData
44-
45-
from typing import List, Dict, Optional, Tuple
46-
50+
from autosubmit_api.monitor.monitor import Monitor
51+
from autosubmit_api.performance.utils import (
52+
calculate_ASYPD_perjob,
53+
calculate_SYPD_perjob,
54+
)
4755
from autosubmit_api.persistance.experiment import ExperimentPaths
4856

57+
if TYPE_CHECKING:
58+
# Avoid circular imports
59+
from autosubmit_api.components.jobs.job_factory import SimpleJob
60+
4961

5062
class JobList:
5163
"""
@@ -576,17 +588,31 @@ def date_to_str_space(date_str):
576588
return ""
577589

578590
@staticmethod
579-
def get_job_times_collection(basic_config: APIBasicConfig, allJobs, expid, job_to_package=None, package_to_jobs=None, timeseconds=True):
591+
def get_job_times_collection(
592+
basic_config: APIBasicConfig,
593+
allJobs: List["SimpleJob"],
594+
expid: str,
595+
job_to_package: Optional[Dict[str, str]] = None,
596+
package_to_jobs: Optional[Dict[str, List[str]]] = None,
597+
timeseconds: bool = True,
598+
):
580599
"""
581600
Gets queuing and running time for the collection of jobs
582601
583602
:return: job running to min (queue, run, status), job running to text (text)
584603
"""
585604
# Getting information
586-
job_data = None
605+
last_jobs_data = None
587606
try:
588-
experiment_history = ExperimentHistoryDirector(ExperimentHistoryBuilder(expid)).build_reader_experiment_history()
589-
job_data = experiment_history.manager.get_all_last_job_data_dcs()
607+
experiment_history = ExperimentHistoryDirector(
608+
ExperimentHistoryBuilder(expid)
609+
).build_reader_experiment_history()
610+
last_jobs_data = experiment_history.manager.get_all_last_job_data_dcs()
611+
map_job_name_to_job_data = (
612+
{job_data.job_name: job_data for job_data in last_jobs_data}
613+
if last_jobs_data
614+
else {}
615+
)
590616
except Exception:
591617
print(traceback.print_exc())
592618
# Result variables
@@ -604,8 +630,20 @@ def get_job_times_collection(basic_config: APIBasicConfig, allJobs, expid, job_t
604630
# Main loop
605631
# print("Start main loop")
606632
for job in allJobs:
633+
job_data = (
634+
map_job_name_to_job_data.get(job.name, None)
635+
if map_job_name_to_job_data
636+
else None
637+
)
607638
job_info = JobList.retrieve_times(
608-
job.status, job.name, job._tmp_path, make_exception=False, job_times=None, seconds=timeseconds, job_data_collection=job_data)
639+
job.status,
640+
job.name,
641+
job._tmp_path,
642+
make_exception=False,
643+
job_times=None,
644+
seconds=timeseconds,
645+
job_data=job_data,
646+
)
609647
# if job_info:
610648
job_name_to_job_info[job.name] = job_info
611649
time_total = (job_info.queue_time +
@@ -660,7 +698,7 @@ def retrieve_times(
660698
make_exception: bool = False,
661699
job_times: Optional[Dict[str, Tuple[int, int, int, int, int]]] = None,
662700
seconds: bool = False,
663-
job_data_collection: Optional[List[JobData]] = None,
701+
job_data: Optional[JobData] = None,
664702
) -> JobRow:
665703
"""
666704
Retrieve job timestamps from database.
@@ -690,60 +728,63 @@ def retrieve_times(
690728

691729
try:
692730
# Getting data from new job database
693-
if job_data_collection is not None:
694-
# for job in job_data_collection:
695-
# print(job.job_name)
696-
job_data = next(
697-
(job for job in job_data_collection if job.job_name == name), None)
698-
if job_data:
699-
status = Status.VALUE_TO_KEY[status_code]
700-
if status == job_data.status:
701-
energy = job_data.energy
702-
if job_times:
703-
t_submit, t_start, t_finish, _, _ = job_times.get(name, (0, 0, 0, 0, 0))
704-
if t_finish - t_start > job_data.running_time:
705-
t_submit = t_submit if t_submit > 0 else job_data.submit
706-
t_start = t_start if t_start > 0 else job_data.start
707-
t_finish = t_finish if t_finish > 0 else job_data.finish
708-
else:
709-
t_submit = job_data.submit if job_data.submit > 0 else t_submit
710-
t_start = job_data.start if job_data.start > 0 else t_start
711-
t_finish = job_data.finish if job_data.finish > 0 else t_finish
712-
job_data.submit = t_submit
713-
job_data.start = t_start
714-
job_data.finish = t_finish
715-
else:
716-
t_submit = job_data.submit
717-
t_start = job_data.start
718-
t_finish = job_data.finish
719-
# Test if start time does not make sense
720-
if t_start >= t_finish:
721-
if job_times:
722-
_, c_start, _, _, _ = job_times.get(name, (0, t_start, t_finish, 0, 0))
723-
job_data.start = c_start if t_start > c_start else t_start
724-
725-
if seconds is False:
726-
queue_time = math.ceil(job_data.queuing_time / 60)
727-
running_time = math.ceil(job_data.running_time / 60)
728-
else:
729-
queue_time = job_data.queuing_time
730-
running_time = job_data.running_time
731-
732-
if status_code in [Status.SUSPENDED]:
733-
t_submit = t_start = t_finish = 0
734-
return JobRow(
735-
job_data.job_name,
736-
int(queue_time),
737-
int(running_time),
738-
status,
739-
energy,
740-
t_submit,
741-
t_start,
742-
t_finish,
743-
job_data.ncpus,
744-
job_data.run_id,
745-
job_data.workflow_commit,
731+
if job_data:
732+
status = Status.VALUE_TO_KEY[status_code]
733+
if status == job_data.status:
734+
energy = job_data.energy
735+
if job_times:
736+
t_submit, t_start, t_finish, _, _ = job_times.get(
737+
name, (0, 0, 0, 0, 0)
746738
)
739+
if t_finish - t_start > job_data.running_time:
740+
t_submit = t_submit if t_submit > 0 else job_data.submit
741+
t_start = t_start if t_start > 0 else job_data.start
742+
t_finish = t_finish if t_finish > 0 else job_data.finish
743+
else:
744+
t_submit = (
745+
job_data.submit if job_data.submit > 0 else t_submit
746+
)
747+
t_start = job_data.start if job_data.start > 0 else t_start
748+
t_finish = (
749+
job_data.finish if job_data.finish > 0 else t_finish
750+
)
751+
job_data.submit = t_submit
752+
job_data.start = t_start
753+
job_data.finish = t_finish
754+
else:
755+
t_submit = job_data.submit
756+
t_start = job_data.start
757+
t_finish = job_data.finish
758+
# Test if start time does not make sense
759+
if t_start >= t_finish:
760+
if job_times:
761+
_, c_start, _, _, _ = job_times.get(
762+
name, (0, t_start, t_finish, 0, 0)
763+
)
764+
job_data.start = c_start if t_start > c_start else t_start
765+
766+
if seconds is False:
767+
queue_time = math.ceil(job_data.queuing_time / 60)
768+
running_time = math.ceil(job_data.running_time / 60)
769+
else:
770+
queue_time = job_data.queuing_time
771+
running_time = job_data.running_time
772+
773+
if status_code in [Status.SUSPENDED]:
774+
t_submit = t_start = t_finish = 0
775+
return JobRow(
776+
job_data.job_name,
777+
int(queue_time),
778+
int(running_time),
779+
status,
780+
energy,
781+
t_submit,
782+
t_start,
783+
t_finish,
784+
job_data.ncpus,
785+
job_data.run_id,
786+
job_data.workflow_commit,
787+
)
747788

748789
# Using standard procedure
749790
if status_code in [Status.RUNNING, Status.SUBMITTED, Status.QUEUING, Status.FAILED] or make_exception is True:

0 commit comments

Comments
 (0)