From aabf1211b081c8fb82dbe76be11e0890ca8916d0 Mon Sep 17 00:00:00 2001 From: heinold Date: Tue, 8 Jan 2019 16:11:49 +0100 Subject: [PATCH 1/3] Work commit --- build.gradle | 3 + .../roddy/execution/jobs/BEFakeJobID.groovy | 2 +- .../dkfz/roddy/execution/jobs/BEJobID.groovy | 7 - .../jobs/BatchEuphoriaJobManager.groovy | 7 + .../execution/jobs/JobManagerOptions.groovy | 8 ++ .../jobs/QueryJobStatesFilter.groovy | 8 ++ .../execution/jobs/SubmissionCommand.groovy | 7 - .../jobs/cluster/lsf/LSFJobManager.groovy | 128 +++++++++++------ .../jobs/cluster/lsf/LSFJobManagerSpec.groovy | 131 +++++++++++------- src/test/resources/bjobsJobTemplatePart1.txt | 1 + src/test/resources/bjobsJobTemplatePart2.txt | 32 +++++ ...onvertBJobsResultLinesToResultMapTest.json | 42 ++++++ .../lsf/queryExtendedJobStateByIdTest.json | 41 ++++++ 13 files changed, 310 insertions(+), 107 deletions(-) create mode 100644 src/main/groovy/de/dkfz/roddy/execution/jobs/QueryJobStatesFilter.groovy create mode 100644 src/test/resources/bjobsJobTemplatePart1.txt create mode 100644 src/test/resources/bjobsJobTemplatePart2.txt create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/convertBJobsResultLinesToResultMapTest.json create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdTest.json diff --git a/build.gradle b/build.gradle index f25fcbf8..dc3f393d 100644 --- a/build.gradle +++ b/build.gradle @@ -46,6 +46,9 @@ configurations.all { } dependencies { + compile 'com.fasterxml.jackson.core:jackson-core:2.9.7' + compile 'com.fasterxml.jackson.core:jackson-annotations:2.9.7' + compile 'com.fasterxml.jackson.core:jackson-databind:2.9.7' compile 'org.codehaus.groovy:groovy-all:2.4.9' testCompile 'junit:junit:4.12' testCompile 'org.spockframework:spock-core:1.1-groovy-2.4' diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEFakeJobID.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEFakeJobID.groovy index 42af20e2..7de70c22 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEFakeJobID.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEFakeJobID.groovy @@ -13,7 +13,7 @@ import static de.dkfz.roddy.execution.jobs.BEFakeJobID.FakeJobReason.values /** * Created by heinold on 23.02.17. */ -class BEFakeJobID extends BEJobID.FakeJobID { +class BEFakeJobID extends BEJobID { /** * Various reasons why a job was not executed and is a fake job. */ diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobID.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobID.groovy index e37083fc..61cfea3b 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobID.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobID.groovy @@ -53,13 +53,6 @@ class BEJobID implements Comparable { return id } - @Deprecated - static class FakeJobID extends BEJobID { - FakeJobID(String id) { - super(id) - } - } - @Override int compareTo(BEJobID o) { return this.id.compareTo(o.id) diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy index 53ef9cac..c1e161cf 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy @@ -35,6 +35,8 @@ abstract class BatchEuphoriaJobManager { protected boolean queryOnlyStartedJobs + protected final Duration maxAgeOfJobsForQueries + protected String userIDForQueries private String userEmail @@ -83,6 +85,7 @@ abstract class BatchEuphoriaJobManager { this.isTrackingOfUserJobsEnabled = parms.userIdForJobQueries as boolean this.queryOnlyStartedJobs = parms.trackOnlyStartedJobs + this.maxAgeOfJobsForQueries = parms.maxAgeOfJobsForJobQueries this.userIDForQueries = parms.userIdForJobQueries this.userEmail = parms.userEmail @@ -104,6 +107,10 @@ abstract class BatchEuphoriaJobManager { } } + void setQueryJobStatesFilter() { + + } + /** * If you override this method, make sure to build in the check for further job submission! It is not allowed to * submit any jobs after waitForJobs() was called. diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy index 57df38a8..47d657e0 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy @@ -18,14 +18,21 @@ import java.time.ZoneId class JobManagerOptions { boolean createDaemon + Duration updateInterval String userIdForJobQueries + + Duration maxAgeOfJobsForJobQueries + boolean trackOnlyStartedJobs String userGroup + String userAccount + String userEmail + String userMask /** @@ -84,6 +91,7 @@ class JobManagerOptionsBuilder { JobManagerOptionsBuilder() { trackOnlyStartedJobs = false updateInterval = Duration.ofMinutes(5) + maxAgeOfJobsForJobQueries = Duration.ofDays(14) createDaemon = false requestMemoryIsEnabled = true requestWalltimeIsEnabled = true diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/QueryJobStatesFilter.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/QueryJobStatesFilter.groovy new file mode 100644 index 00000000..46a6abcf --- /dev/null +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/QueryJobStatesFilter.groovy @@ -0,0 +1,8 @@ +package de.dkfz.roddy.execution.jobs + +import groovy.transform.CompileStatic + +@CompileStatic +class QueryJobStatesFilter { + boolean keep() { return true }; +} diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommand.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommand.groovy index 9155590e..d589c782 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommand.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommand.groovy @@ -61,13 +61,6 @@ abstract class SubmissionCommand extends Command { passEnvironment.orElse(parentJobManager.passEnvironment) } - @Deprecated - @Override - String toString() { - // TODO toString() shouldn't be used for such specific things. Remove explicit calls to get the bash command representation. - return toBashCommandString() - } - @Override String toBashCommandString() { String email = parentJobManager.getUserEmail() diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy index 8d545fec..5d59318a 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy @@ -14,12 +14,13 @@ import de.dkfz.roddy.execution.jobs.* import de.dkfz.roddy.tools.BashUtils import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.BufferValue +import de.dkfz.roddy.tools.DateTimeHelper import groovy.json.JsonSlurper +import groovy.transform.CompileStatic import java.time.Duration import java.time.LocalDateTime import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter /** * Factory for the management of LSF cluster systems. @@ -29,33 +30,38 @@ import java.time.format.DateTimeFormatter @groovy.transform.CompileStatic class LSFJobManager extends AbstractLSFJobManager { - private static final String LSF_COMMAND_QUERY_STATES = "bjobs -a -o -hms -json \"jobid job_name stat user queue " + + private static final String LSF_COMMAND_QUERY_STATES = "bjobs -a -o -hms -json \"jobid job_name stat finish_time\"" + private static final String LSF_COMMAND_QUERY_EXTENDED_STATES = "bjobs -a -o -hms -json \"jobid job_name stat user queue " + "job_description proj_name job_group job_priority pids exit_code from_host exec_host submit_time start_time " + "finish_time cpu_used run_time user_group swap max_mem runtimelimit sub_cwd " + "pend_reason exec_cwd output_file input_file effective_resreq exec_home slots error_file command dependency \"" private static final String LSF_COMMAND_DELETE_JOBS = "bkill" - private final DateTimeFormatter DATE_PATTERN - + static final DateTimeHelper dateTimeHelper = new DateTimeHelper("MMM ppd HH:mm yyyy") LSFJobManager(BEExecutionService executionService, JobManagerOptions parms) { super(executionService, parms) - DATE_PATTERN = DateTimeFormatter - .ofPattern("MMM ppd HH:mm yyyy") - .withLocale(Locale.ENGLISH) - .withZone(parms.timeZoneId) } - private String getQueryCommand() { - return LSF_COMMAND_QUERY_STATES + ZonedDateTime parseTime(String str) { + ZonedDateTime date = dateTimeHelper.parseTime(str) + /** + * LSF does not return the year of submission (or whatever). The assumption here is that if the time parsed + * under the assumption of the current year is later than the current time, then the job correct date must + * have been in the last year. As LSF does not return the actual year, this is the least problematic assumption. + */ + if (date > ZonedDateTime.now()) { + return date.minusYears(1) + } + return date } @Override Map queryExtendedJobStateById(List jobIds) { Map queriedExtendedStates = [:] for (BEJobID id : jobIds) { - Map jobDetails = runBjobs([id]).get(id) - queriedExtendedStates.put(id, queryJobInfo(jobDetails)) + Map jobDetails = runBjobs([id], true).get(id) + queriedExtendedStates.put(id, convertJobDetailsMapToGenericJobInfoobject(jobDetails)) } return queriedExtendedStates } @@ -88,15 +94,16 @@ class LSFJobManager extends AbstractLSFJobManager { } protected Map queryJobStates(List jobIDs) { - runBjobs(jobIDs).collectEntries { BEJobID jobID, Object value -> + def bjobs = runBjobs(jobIDs, false) + bjobs.collectEntries { BEJobID jobID, Object value -> JobState js = parseJobState(value["STAT"] as String) [(jobID): js] } as Map } - private Map> runBjobs(List jobIDs) { - StringBuilder queryCommand = new StringBuilder(getQueryCommand()) + Map> runBjobs(List jobIDs, boolean extended) { + StringBuilder queryCommand = new StringBuilder(extended ? LSF_COMMAND_QUERY_EXTENDED_STATES : LSF_COMMAND_QUERY_STATES) // user argument must be passed before the job IDs if (isTrackingOfUserJobsEnabled) @@ -109,49 +116,76 @@ class LSFJobManager extends AbstractLSFJobManager { ExecutionResult er = executionService.execute(queryCommand.toString()) List resultLines = er.resultLines + if (!er.successful) { + String error = "Job status couldn't be updated. \n command: ${queryCommand} \n status code: ${er.exitCode} \n result: ${er.resultLines}" + throw new BEException(error) + } + + Map> result = convertBJobsJsonOutputToResultMap(resultLines.join("\n")) + return filterJobMapByAge(result, maxAgeOfJobsForQueries) + } + + static Map> convertBJobsJsonOutputToResultMap(String rawJson) { Map> result = [:] - if (er.successful) { - if (resultLines.size() >= 1) { - String rawJson = resultLines.join("\n") - Object parsedJson = new JsonSlurper().parseText(rawJson) - List records = (List) parsedJson.getAt("RECORDS") - records.each { - BEJobID jobID = new BEJobID(it["JOBID"] as String) - result.put(jobID, it as Map) - } - } + if (!rawJson) + return result - } else { - String error = "Job status couldn't be updated. \n command: ${queryCommand} \n status code: ${er.exitCode} \n result: ${er.resultLines}" - throw new BEException(error) + Object parsedJson = new JsonSlurper().parseText(rawJson) + List records = (List) parsedJson["RECORDS"] + for (record in records) { + BEJobID jobID = new BEJobID(record["JOBID"] as String) + result[jobID] = record as Map } - return result + + result } - private ZonedDateTime parseTime(String str) { - ZonedDateTime date = ZonedDateTime.parse("${str} ${LocalDateTime.now().getYear()}", DATE_PATTERN) - if (date > ZonedDateTime.now()) { - return date.minusYears(1) + /** + * For all entries in records, check if they are finished and if so, check if they are younger (or older) than + * the maximum age. + * @param records A map of informational entries for one or more job ids + * @param reference Time A timestamp which can be set. It is compared against the timestamp of finished entries. + * @param maxJobKeepDuration Defines the maximum duration + * @return The map of records where too old entries are filtered out. + */ + @CompileStatic + static Map> filterJobMapByAge( + Map> records, + LocalDateTime referenceTime = LocalDateTime.now(), + Duration maxJobKeepDuration + ) { + records.findAll { def k, def record -> + String finishTime = record["FINISH_TIME"] + boolean youngEnough = true + if (finishTime) { + String timeString = "${finishTime[0..-3]} ${referenceTime.getYear()}" + withCaughtAndLoggedException { + ZonedDateTime _finishTime = parseTime(timeString) + Duration timeSpan = dateTimeHelper.timeSpanOf(_finishTime.toLocalDateTime(), referenceTime) + if (dateTimeHelper.isOlderThan(timeSpan, maxJobKeepDuration)) + youngEnough = false + } + } + youngEnough } - return date } /** * Used by @getJobDetails to set JobInfo */ - private GenericJobInfo queryJobInfo(Map jobResult) { + private GenericJobInfo convertJobDetailsMapToGenericJobInfoobject(Map jobResult) { GenericJobInfo jobInfo BEJobID jobID - try{ + try { jobID = new BEJobID(jobResult["JOBID"] as String) - }catch (Exception exp){ + } catch (Exception exp) { throw new BEException("Job ID '${jobResult["JOBID"]}' could not be transformed to BEJobID ") } - List dependIDs = ((String) jobResult["DEPENDENCY"])? ((String) jobResult["DEPENDENCY"]).tokenize(/&/).collect { it.find(/\d+/) } : null - jobInfo = new GenericJobInfo(jobResult["JOB_NAME"] as String ?: null, jobResult["COMMAND"] as String ? new File(jobResult["COMMAND"] as String): null, jobID, null, dependIDs) + List dependIDs = ((String) jobResult["DEPENDENCY"]) ? ((String) jobResult["DEPENDENCY"]).tokenize(/&/).collect { it.find(/\d+/) } : null + jobInfo = new GenericJobInfo(jobResult["JOB_NAME"] as String ?: null, jobResult["COMMAND"] as String ? new File(jobResult["COMMAND"] as String) : null, jobID, null, dependIDs) String queue = jobResult["QUEUE"] ?: null Duration runTime = withCaughtAndLoggedException { @@ -204,13 +238,17 @@ class LSFJobManager extends AbstractLSFJobManager { jobInfo.setResourceReq(jobResult["EFFECTIVE_RESREQ"] as String ?: null) jobInfo.setExecHome(jobResult["EXEC_HOME"] as String ?: null) - if (jobResult["SUBMIT_TIME"]) - withCaughtAndLoggedException { jobInfo.setSubmitTime(parseTime(jobResult["SUBMIT_TIME"] as String)) } - if (jobResult["START_TIME"]) - withCaughtAndLoggedException { jobInfo.setStartTime(parseTime(jobResult["START_TIME"] as String)) } - if (jobResult["FINISH_TIME"]) + String submissionTime = jobResult["SUBMIT_TIME"] + String startTime = jobResult["START_TIME"] + String finishTime = jobResult["FINISH_TIME"] + + if (submissionTime) + withCaughtAndLoggedException { jobInfo.setSubmitTime(parseTime(submissionTime as String)) } + if (startTime) + withCaughtAndLoggedException { jobInfo.setStartTime(parseTime(startTime as String)) } + if (finishTime) withCaughtAndLoggedException { - jobInfo.setEndTime(parseTime((jobResult["FINISH_TIME"] as String).substring(0, (jobResult["FINISH_TIME"] as String).length() - 2))) + jobInfo.setEndTime(parseTime(finishTime[0..-2])) } return jobInfo diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy index 3fd71656..66935ba6 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy @@ -20,56 +20,12 @@ import spock.lang.Specification import java.lang.reflect.Method import java.time.Duration +import java.time.LocalDateTime import java.time.ZoneId import java.time.ZonedDateTime class LSFJobManagerSpec extends Specification { - - final static String RAW_JSON_OUTPUT = ''' -{ - "COMMAND":"bjobs", - "JOBS":1, - "RECORDS":[ - { - "JOBID":"22005", - "JOB_NAME":"ls -l", - "STAT":"DONE", - "USER":"otptest", - "QUEUE":"short-dmg", - "JOB_DESCRIPTION":"", - "PROJ_NAME":"default", - "JOB_GROUP":"", - "JOB_PRIORITY":"", - "PIDS":"46782,46796,46798,46915,47458,47643", - "EXIT_CODE":"", - "FROM_HOST":"tbi-cn013", - "EXEC_HOST":"tbi-cn019:tbi-cn019", - "SUBMIT_TIME":"Dec 28 19:56", - "START_TIME":"Dec 28 19:56", - "FINISH_TIME":"Dec 28 19:56 L", - "CPU_USED":"00:00:01", - "RUN_TIME":"00:00:01", - "USER_GROUP":"", - "SWAP":"", - "MAX_MEM":"5.2 Gbytes", - "RUNTIMELIMIT":"00:10:00", - "SUB_CWD":"$HOME", - "PEND_REASON":"", - "EXEC_CWD":"\\/home\\/otptest", - "OUTPUT_FILE":"", - "INPUT_FILE":"", - "EFFECTIVE_RESREQ":"select[type == local] order[r15s:pg] ", - "EXEC_HOME":"\\/home\\/otptest", - "SLOTS":"1", - "ERROR_FILE":"", - "COMMAND":"ls -l", - "DEPENDENCY":"done(22004)" - } - ] -} -''' - final static String RAW_JSON_OUTPUT_WITHOUT_LISTS = ''' { "COMMAND":"bjobs", @@ -155,7 +111,7 @@ class LSFJobManagerSpec extends Specification { void "queryJobInfo, bjobs JSON output empty "() { given: - String emptyRawJsonOutput= ''' + String emptyRawJsonOutput = ''' { "COMMAND":"bjobs", "JOBS":1, @@ -184,8 +140,9 @@ class LSFJobManagerSpec extends Specification { void "test queryExtendedJobStateById"() { given: JobManagerOptions parms = JobManagerOptions.create().build() + def jsonFile = getResourceFile("de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdTest.json") BEExecutionService testExecutionService = [ - execute: { String s -> new ExecutionResult(true, 0, RAW_JSON_OUTPUT.split("\n") as List, null) } + execute: { String s -> new ExecutionResult(true, 0, jsonFile.readLines(), null) } ] as BEExecutionService LSFJobManager manager = new LSFJobManager(testExecutionService, parms) @@ -263,4 +220,84 @@ class LSFJobManagerSpec extends Specification { jobInfo.timeUnknownState == null jobInfo.timeOfCalculation == null } + + static final File getResourceFile(String file) { + new File("src/test/resources", file) + } + + def "test convertBJobsResultLinesToResultMap"() { + given: + def jsonFile = getResourceFile("de/dkfz/roddy/execution/jobs/cluster/lsf/convertBJobsResultLinesToResultMapTest.json") + def json = jsonFile.text + + when: + Map> map = LSFJobManager.convertBJobsJsonOutputToResultMap(json) + def jobId = map.keySet()[0] + + then: + map.size() == 6 + jobId.id == "487641" + map[jobId]["JOBID"] == "487641" + map[jobId]["JOB_NAME"] == "RoddyTest_testScript" + map[jobId]["STAT"] == "EXIT" + map[jobId]["FINISH_TIME"] == "Jan 7 09:59 L" + } + + def "test filterJobMapByAge"() { + given: + def jsonFile = getResourceFile("de/dkfz/roddy/execution/jobs/cluster/lsf/convertBJobsResultLinesToResultMapTest.json") + def json = jsonFile.text + LocalDateTime referenceTime = LocalDateTime.of(2019, 01, 8, 14, 29, 2, 25) + + when: + def records = LSFJobManager.convertBJobsJsonOutputToResultMap(json) + records = LSFJobManager.filterJobMapByAge(records, referenceTime, Duration.ofMinutes(10)) + def id = records.keySet()[0] + + then: + records.size() == 2 + id.id == "491864" + records[id]["FINISH_TIME"] == "Jan 8 14:20 L" + } + + def testMassiveConvertBJobsResultLinesToResultMap(def _entries, def value) { + when: + int entries = _entries[0] + String template1 = getResourceFile("bjobsJobTemplatePart1.txt").text + String template2 = getResourceFile("bjobsJobTemplatePart2.txt").text + List lines = new LinkedList<>() + + lines << "{" + lines << ' "COMMAND":"bjobs",' + lines << ' "JOBS":"' + entries + '",' + lines << ' "RECORDS":[' + + int maximum = 1000000 + entries - 1 + for (int i = 1000000; i <= maximum; i++) { + lines += template1.readLines() + lines << ' "JOBID":"' + i + '",' + lines << ' "JOB_NAME":"r181217_003553288_Strand_T_150_aTestJob",' + lines += template2.readLines() + if (i < maximum) + lines << " ," + } + lines << " ]" + lines << "}" + println("Entries ${entries}") + def result = LSFJobManager.convertBJobsJsonOutputToResultMap(lines.join("\n")) + + then: + result.size() == entries + + where: + _entries | value + [1] | true + [10] | true + [100] | true + [1000] | true + [2000] | true + [4000] | true + [8000] | true + [16000] | true + } } diff --git a/src/test/resources/bjobsJobTemplatePart1.txt b/src/test/resources/bjobsJobTemplatePart1.txt new file mode 100644 index 00000000..7e9163bd --- /dev/null +++ b/src/test/resources/bjobsJobTemplatePart1.txt @@ -0,0 +1 @@ + { diff --git a/src/test/resources/bjobsJobTemplatePart2.txt b/src/test/resources/bjobsJobTemplatePart2.txt new file mode 100644 index 00000000..f76286ac --- /dev/null +++ b/src/test/resources/bjobsJobTemplatePart2.txt @@ -0,0 +1,32 @@ + "STAT":"RUN", + "USER":"icgcdata", + "QUEUE":"verylong", + "JOB_DESCRIPTION":"", + "PROJ_NAME":"default", + "JOB_GROUP":"", + "JOB_PRIORITY":"", + "PIDS":"15197,15238,15242,16672,16750,16751,16754,16756,16767,16769,16771,16774,16775,16776,16777,16778,16779,16781,16783,16784,16786,16787,16789,16801,16802,16803,16804,16805,16806,16807,16808,16809,16810,16811,16812,16814,16815,16833", + "EXIT_CODE":"", + "FROM_HOST":"host-submission", + "EXEC_HOST":"host-cn037:host-cn037:host-cn037:host-cn037:host-cn037:host-cn037:host-cn037:host-cn037", + "SUBMIT_TIME":"Dec 17 00:35", + "START_TIME":"Dec 17 00:36", + "FINISH_TIME":"Dec 27 00:36 L", + "CPU_USED":"59:32:51", + "RUN_TIME":"08:38:26", + "USER_GROUP":"", + "SWAP":"0 Mbytes", + "MAX_MEM":"40.8 Gbytes", + "RUNTIMELIMIT":"240:00:00", + "SUB_CWD":"\/tmp\/some\/cwd\/folder", + "PEND_REASON":"", + "EXEC_CWD":"/home/a/testfolder", + "OUTPUT_FILE":"\/tmp\/some\/annoyinglylong\/folder\/for\/a\/textfile\/andhereisthelogfile.o471773", + "INPUT_FILE":"", + "EFFECTIVE_RESREQ":"select[type == local] order[r15s:pg] rusage[mem=61440.00] span[hosts=1] ", + "EXEC_HOME":"\/home\/icgcdata", + "SLOTS":"8", + "ERROR_FILE":"", + "COMMAND":"\/tmp\/a\/project\/folder\/with\/results\/roddyExecutionStore\/exec_181217_003553288_testuser_test\/analysisTools\/roddyTools\/wrapInScript.sh", + "DEPENDENCY":"" + } diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/convertBJobsResultLinesToResultMapTest.json b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/convertBJobsResultLinesToResultMapTest.json new file mode 100644 index 00000000..03504df4 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/convertBJobsResultLinesToResultMapTest.json @@ -0,0 +1,42 @@ +{ + "COMMAND":"bjobs", + "JOBS":6, + "RECORDS":[ + { + "JOBID":"487641", + "JOB_NAME":"RoddyTest_testScript", + "STAT":"EXIT", + "FINISH_TIME":"Jan 7 09:59 L" + }, + { + "JOBID":"487642", + "JOB_NAME":"RoddyTest_testScript", + "STAT":"EXIT", + "FINISH_TIME":"Jan 7 09:59 L" + }, + { + "JOBID":"488155", + "JOB_NAME":"RoddyTest_testScript", + "STAT":"EXIT", + "FINISH_TIME":"Jan 7 09:59 L" + }, + { + "JOBID":"491861", + "JOB_NAME":"RoddyTest_testScript", + "STAT":"DONE", + "FINISH_TIME":"Jan 8 14:17 L" + }, + { + "JOBID":"491864", + "JOB_NAME":"RoddyTest_testScript", + "STAT":"DONE", + "FINISH_TIME":"Jan 8 14:20 L" + }, + { + "JOBID":"491867", + "JOB_NAME":"RoddyTest_testScriptMultiInMixedParameters", + "STAT":"DONE", + "FINISH_TIME":"Jan 8 14:21 L" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdTest.json b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdTest.json new file mode 100644 index 00000000..06d5fd3c --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdTest.json @@ -0,0 +1,41 @@ +{ + "COMMAND": "bjobs", + "JOBS": 1, + "RECORDS": [ + { + "JOBID": "22005", + "JOB_NAME": "ls -l", + "STAT": "DONE", + "USER": "otptest", + "QUEUE": "short-dmg", + "JOB_DESCRIPTION": "", + "PROJ_NAME": "default", + "JOB_GROUP": "", + "JOB_PRIORITY": "", + "PIDS": "46782,46796,46798,46915,47458,47643", + "EXIT_CODE": "", + "FROM_HOST": "tbi-cn013", + "EXEC_HOST": "tbi-cn019:tbi-cn019", + "SUBMIT_TIME": "Dec 28 19:56", + "START_TIME": "Dec 28 19:56", + "FINISH_TIME": "Dec 28 19:56 L", + "CPU_USED": "00:00:01", + "RUN_TIME": "00:00:01", + "USER_GROUP": "", + "SWAP": "", + "MAX_MEM": "5.2 Gbytes", + "RUNTIMELIMIT": "00:10:00", + "SUB_CWD": "$HOME", + "PEND_REASON": "", + "EXEC_CWD": "\/home\/otptest", + "OUTPUT_FILE": "", + "INPUT_FILE": "", + "EFFECTIVE_RESREQ": "select[type == local] order[r15s:pg] ", + "EXEC_HOME": "\/home\/otptest", + "SLOTS": "1", + "ERROR_FILE": "", + "COMMAND": "ls -l", + "DEPENDENCY": "done(22004)" + } + ] +} \ No newline at end of file From cd016ae9d31f136c624442e5c3554ad6d35cb402 Mon Sep 17 00:00:00 2001 From: heinold Date: Tue, 15 Jan 2019 14:43:00 +0100 Subject: [PATCH 2/3] Changes for review of #137 by vinjana Update gradle wrapper to 5.1.1 Updated the dependency for RoddyToolLib to 2.1.0 In GenericJobInfo, change field tool of type File to command of type String. This better reflects that e.g. LSF can also execute commands. A tool can always be seen as a command whereas a command is not necessarily a tool. In JobManagerOptions, rename maxAgeOfJobsForJobQueries to maxTrackingTimeForFinishedJobs, as the new name is much more expressive. In BatchEuphoriaJobManager and its subclasses, rename the aforementioned field as well. In PBSCommandParser and LSFCommandParser, do a bit of code cleanup and adapt code. In LSFJobManager: - Add the stripAwayStatusInfo method for time string. This gets rid of the trailing status code in some timing info reported by LSF. - Change the return value of runBjobs to return a nested map of type Map. This makes some followup code much cleaner. - Changed convertJobDetailsMapToGenericJobInfoobject a lot: - Fixed a typo. - Got rid of all the "as String" casts. - Reordered the entries and grouped them logically. - Extracted code duplicates to custom methods. In LSFJobManagerSpec: - Renamed "queryJobInfo, bjobs JSON output with lists" to "Test convertJobDetailsMapToGenericJobInfoObject" - Added the "test queryExtendedJobStateById with overdue date" method which tests for a NullPointerException. --- build.gradle | 3 +- gradle/wrapper/gradle-wrapper.jar | Bin 54712 -> 55741 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 2 +- gradlew.bat | 2 +- .../jobs/BatchEuphoriaJobManager.groovy | 4 +- .../execution/jobs/GenericJobInfo.groovy | 6 +- .../execution/jobs/JobManagerOptions.groovy | 4 +- .../jobs/cluster/lsf/LSFCommandParser.groovy | 11 +- .../jobs/cluster/lsf/LSFJobManager.groovy | 176 ++++++++++-------- .../cluster/lsf/rest/LSFRestJobManager.groovy | 2 +- .../jobs/cluster/pbs/PBSCommandParser.groovy | 10 +- .../jobs/cluster/lsf/LSFJobManagerSpec.groovy | 41 ++-- .../jobs/cluster/pbs/PBSJobManagerSpec.groovy | 4 +- 14 files changed, 153 insertions(+), 114 deletions(-) diff --git a/build.gradle b/build.gradle index dc3f393d..15a643fc 100644 --- a/build.gradle +++ b/build.gradle @@ -63,8 +63,7 @@ dependencies { compile group: 'commons-cli', name: 'commons-cli', version: '1.2' compile group: 'org.apache.commons', name: 'commons-text', version: '1.1' compile 'com.google.guava:guava:23.0' - // compile 'com.github.eilslabs:RoddyToolLib:master-SNAPSHOT' - compile 'com.github.theroddywms:RoddyToolLib:2.0.0' + compile 'com.github.theroddywms:RoddyToolLib:2.1.0' } task writePom { diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index ed88a042a287c140a32e1639edfc91b2a233da8c..457aad0d98108420a977756b7145c93c8910b076 100755 GIT binary patch delta 49817 zcmZ6wQ*b3*w1t~aC+XO>ZS8by+qP{dyJOq#*tVT?Y}-zD?BxFUo`+lKRIP{gxT@By zF~=BRTjkI0#-H&37~aO)IX*BjFx3P>WvIZWJLex@@wf>s?ywbW=Bqy;!NAU8k_5GI zk`#X619ddeG|<0dGfWNfAx#8>B7*>^j1kSt>UB-h-NZi_e%Oj`AdZnl%bU-S4awYK z`F#HBc`8=WtMr3us##xJ`HK3A;K9>D%7jBn^n5p$ zLn^kqA)X<0E23X7lRR;4ZW1d0{ZMhO_~flevvLEx*CfM=Ab8Jc#^{KWA(&|_FRV_k7eWLV{aMw;DMs? z+R5g5>U+v z>6ymtJR{klAX#fPvE#BkN>`P)Ph?yNBpa?%rhH4nZ+yCqZlKd6(p33^ou@bq{*Y9n zw-+dZk+*Ld?~sFe?zNza+3r`e@8X-kFXJm_`B@|$R3xq<(i)CB{+$q6dyBB-!*;2= zeH-&MOxpz9XFI7 zZhet2dg2@0{qE9(e{a1I=@812Kp1mpXy%k&ZN@q-311g&xb>Ee0bu!5)MTs`WROhd zraD7jq0%`6-s|@7l<&phd7HF7ThlvN+`@F&*%JFFp-JsbD4--nY$j{Dfx1+%?njle zj_6&ZnFDXi3BDnnNp>S*2K$hE+;ZzHC&RZx%SO$Rh6LJ*xqyvTdb@h59?Mx#_={Y> zw2n91LZ)wh_ScH2L>)2J#jZ9McnKXSX*^LvYinI6#u8J~_TmgvVXn%kv(3rJ5r`Z6 zlms~`kF&`5(Oh@lZSuc#z!ni{2EmQ^WvEl}!`Ws`^05~@NA zPfLVLjrGeuCvCD^M*;cSiZ@tn(ZJlrYHU&lR`Z`lR7ufDF;zyRmuSJx^F;B^`E$4f z9HI>u@iwQLA}8rhh!l)Vb!?1cOStYaAqRV>_Vd@{Aq87&xKGhyzX#n_PRN4~a{@8`x&ub^f`fJg+JDg!J}?COz3+?x z(I`mBqtwa}WavDp-}e8Op#ip8hJIIZx^8P(|&g2Z#R5fI@vBcT*6pG`T= z^;80%@GLlt zpLCtV{1<#brmJO9VLl!jgr=pO12WsyTtBa3CyIAydNcr=ln2CgHplVAeeeNPl=%sP z4h9Ad{hxFSfmtej`4#+kn@@6p;s7?&x-$jBtw41L3hWB&dIqkBuA(L;s`>!12>V1L zX!e(JjuDW0p7n1=(fV*jphlR9$t`?bGB&ZaQAcz(ul_pEY&qI*QL?kY{7?!a1p0`~ zx>R5}XVJI9ax#2oem`%vXBhPMenU7C#SzFFE(b(u;knRE^0Y;Spku0NvTo=graEm1 zf%n8<;CWz=*>9&?z^z3rI4m!eXZ6^hP{xIG;?59tFCDE^Ix>`P>sk_n2U2x#h;;&5v~MH*~>)fg~G65QNWaU!cX{{ffcA>ugJVM zl7v!;{P~ug%8pF{!Z1uA)GR0Oo06WLv`)aclTfrwm=7$Ftq<{b<6<4l^sxp(bc1n2 zHqFB`w>X1c(Am+Oo(gQk`43T~zdnG098;Fb8~#0x)lyqKw-ZTr-6w`&-;eOUBbPh0 zIf9)`L}W{_8|Ik`%)PQnT_Q=I#1_G%kEj~F(NwGNVBuZ!(D^`|zO!~-q+ie3KY-ZU z=IQK|F1&O@gKVHur9OZB?-@cnm*Xfx1Osy*00SfWpBd^Mfdb}e!5E^iczyX>HM42Q z1Zz$o-5CclxZ+0lV{&q$rU^pli;=>OJQ2qfg)GJL^1xBc?Q=0yX9YJ|9gtJT<_yVa zZ*y_d;U})TX>XQP`vB~c2(7Ym?Y)cTvbdCbpPy!Hnv`3)J<3&4$yEH3E@KtS_X84Jn(XW%pWPBGy$y0tWAHhh9>IID?gq=_y~FXGJ{sw zwITVy_|oP8s>s+I0B&1JJ_x?H`vlToMv*yp^Q`}V0olPR^^x}SZezGV;J!G5WnKVi z-!QwXA#D|Hi3k&8p~8D}cv4cw!$7EdF~$Sq~f8;+Cy#6qxwA2Wu@Y6vbJR~AXd zY1mB00G3p-lN*m$AcvYZ)7Z#rRZiJ{$SncsP$i$xGcefMu5BH8QEvgKP^0 zd2W4)TK!EDX0Gc-D*VV-3YNh3<%^ zo1(=%yA;Hot6UihJ)a^V;_n%_?G#ChkDS1o63C%Urm>F^c?0u( zVDbS$vjRmk<8@sQenWRK?g`b;FVDad{m%G*=yQ6C4Md__fXb0fBLGJx_Jrs3CiSd& zt-Z47Bg+mUzstYeNgI>c!!=lY%ki8n=a1O_fLz|R;Ej4ff-C#4K|&m*0+y0-DI9Jx z`E`Y&yDr@EXI8MMY$A8cB~kzY)AOka=sPO2CIy>zo9H5cb<`3nyMF5)^ZS*#_&~u- zTtaO~HRrT|w|sKwPoZgT>2*`;&PCD4htpdFvdNacIE&FN5i!OYoVj}&U7udEyBUrS zrT93GdV%sEYrp|qD%R5P>{}`}{+R1kG~F6$$M=SPYXxifxp;FCZFnn=qN#Xypa?g$ z!#bfGRClo97QT){ri}tFxzWGsL%A%EwX-3RF3`T>wB}WsdNd?7%A`XVrw z&B|h9nXlX^=LZ0w7G5R&y2*9?(k{16GD_rOHK9h~n#-0+-Wt1!d&VGvhGiMEIZG!I zHCQU|l62RR+4))5I{K6dBi39646eYIgdDh{2o}21a#v5joIPEw#5zrx3LKU_-;AWB&o(F!hyT# z8)V;UyHRLSpsiC(#A_H%+yGe#y@P|AjJ_~L_#2kuHyG9^HyGMQuA7Qf+%-MXZyG@8 z{N`hN&VU~k@l<-;5jhF-w#K-r#^Ps*+`v-n>-t0eHM2b{UXwjq{UVs;UAy(;R9C51 zicT{mrbES`I-H32=jJ$`e*e)gXX9p9&OUuEJk zmcMO9w;E%ynpKiHui)=1!ly|^7Cj0fO=F!^T=O^$U4yX_(zsHT*stckibN(AH|Al? zRoPWcv285SY29nmJZ%Vz)F-C`YGg^v+1S(36tqU`LG=7~V`cNsvLg&q|EvO|*Ei%_ z#-11vR8q51wo&teS|F0ynK2g`O7fJr;`wg8NFH40U6;ivnB)@`SfS9iRM~NnrrX`Q zZaWXneW{Mjcl(Xj&vL1-zp@t>8$+B0ou`Drv6T2s%NYh#8+cM^yV-GA-8%gEJqTQ{oR zcPZMDzsga8ck`mx3~;f92f03s8jjVBQ%|d^S0lnFQ1KIk+S7oEd$_feCQDo55b)f|djs^w%r^{L;DRi1Dvl|N4M;prB%M7EUk zN6K{-0E!E&-!;@q%}t9FOCnoG7}RbjY;?P4cTFW{qG`CCsUf00&QqS_nITSId+}Km zVQ-=x7EggR4BYP+iRcts%0`(V9Gi|B#o!8SAMgBXr*a&!2}jk%S)HSHOeMBixbEYT z#d?`Q?P2a>tJfdfzmsbm-$60^7W1YFxGEit87f%>Tz^e#exFeILj4umczP(yuU8;Gdag@>(1@Ky8Z1yEP z^X-r|Pa>JMKK6jGaU^4sB4_7NAwFZ!SYiGMjKa#9S1F8SJu7L)xQo>tO6!+5<=5(o zFY!w$y&JXr5D2DaR^+GQ1;4JYI~nTYD=wY%QLp;&w`npVbz9~WS=rI*kENnbZgtF7;5|8|s=TiW1V>+b6@M}~^+%3D1Tm8bvpw+Ahg!Q7W8pVA8 z3hfeyt4Z#%zgvK*l8oJoj&X(5dDGMpsmDQn%ANH3vV!nPue7pFd~yqg7K!^D>|COn zTI57>7e-~R254K9(oMhBpE6y7Ur4&-zFTV4*-*Dssw!L+SSpw4w{tCSsWldFo|ick z3M<-=U@NjVYKUj?VRVJ$TrMym+6^axcY42quH+6|0#58if7*{IZw5kV)?sEf(d)QS zVVY@aB%9@^H%dPf&Nr0hCerq4@u%M@6D%u(%M2c?E$T}Qas=-ROb@79 z68zp!3MW>5?ie$_1VhXTo}W_;34QG9vxjs%F_Ogi5!hN)kq`}oHSRusMU=+o^^85$E$1{qS>0g-Em5M;h8aV@ml1mVj!=YK(1_LM$Nb%!Su@t< z)#Jp)L#w?PK{uz;djQqk{0Df&U{-@Ae+gC2m0LE9zJ#Wqc4`PCeJv1$L8wV=^4zRh ztBIfIp0K-=A?itT$}K%1lb(CMrZ9u8l|L$r8%s?eu;^_%Co^;Q*9QSuyI@*-fcKW= za;Csi^@pczTPm{C@~abvm06TZPPj4uM!t=pyo&HfAK$#x&io{6Bf zW3Ogi^ZSm36fTI>AO8keBGS4G(-kIU6Mq2Ng@uJfi^STwGpbul(drR@V^v}qy?2-7 zDRBPpmeJ5}qjR2DH4QAS?3FuTpl#9+d;PCA_l2xKzxPRYB680~cl*DOLpMBhsmQ zo+!RB?LnW$Rjh4;Pp=14r@yrvzKStEKk-K^2g8iBFX5YjFe~p3T$@^GFrwCNt5~)6 zFwYiF!4P2vBYkGqycO<=_Uer;9&g&g#2jGOchF zO_F}}B+@pcQQeqI6BsEYFvpB!NFuRC5E{Kv`)o{@sU^El+>nK1Qbq{x~d z^*gO|0iL0O-Ct2J6i-ml$Z2aTo__U~7g6gq^ni5znLV~-C9O6{H8xX&YLldLP#5gk zD*R27pi?B8cdCwlU&;rhTLjtBM4a;6-Tsi=<>O48|!1_(govKx%xL?&X#Q;9$%A$Nuydzeb z>hUK^$p;xfk|Fr2se4D`meN-&|T;VfdR`~=n$*J%Uf9zO4fRSe6seO=Ik2Mk56W@neL5d z(GylPQpKs93x2y9b_)K&`rj$nD?qj&=@UxCcu@}(2S|2Hm@ zecc@^tn+vF%KI%u-9TT^Nxnk*Pz9qi#lft!PR5iRCd_E?E&abky~x07uDmCkoje2R zK}9w0eZ5RJqmIvViE7lT5s317-8%giN~3&NQdEY6Y_V!Htp?0*^nuf?iL5RzUAEXN zdKqHZ*;ssGEcy3Yo7I5#{}aXwgTgjzpuoU<;laSz{@*Z;kpz!J0@T=W!PCI`CeWH+ zuXOXZs!*SjlZTa*w=LdFB+Wt|ZB}_rY^y(5rXbgop?Vsr)T`GXW_d;zh6{1D2?zxr zg!3~@&u3GW0>SS=rtjf|_Wl{|0S&;zr|+z#;P0T85WZjj-g&*=xp{a0Jo)~5@&Jq7 zv*6|rKH2zt0NFfg1@u&sK+l_Z6q3N7sT?d_x-JNnsrHnzlM~C3g#scQK^!wOXz>mROX#1Ytj7F<6huwQURA_p`~gOfvYG7n62x{%bDKtJKyw zDl?nNH&#-w1SXZ3FiOV7j!gX_W=@7VsN2lm$Aebj*lBPPTcKIzRp9owUv088b0Y2} zId8nzFrCvFKWvUXs%9Qy=`igarU>XJDHQ1*7F%#C(nEk}j-*JTZlB|!G;hSBWo?w^ z`1KVh0A4+=UshDv?l*FGno+5H{vca=m^%0((n7Ut2h7=UmT=P^+;CgiAjL*RI+U|# z9Fs2AN~I#Ye%zg$?1M=Ow%Va*l*>#JlpVxk_f;B9WB1h(w0JcgZVjgzuw2)6lzZcNE>X?Nts!qKiiNsyX%SB zC53=s0dA<<5<(KL1%%DXc5>9NmOTdiB5O2&!8;vw`x)jmBuEqsjg2S524d|Y1+6Y$ z^QbjmCO*LXVgTTM5o)0yx@&8QkKeBL69Y6-0{wlr`T(HJ0Edbr9nYA)c87c2DW&Re zee!_9LHjAjVuBR~EIz9@Ry`KA`GsOi?1eNvAX7DQW{>QK)lYzk$PT9?rV4nR0CXxi zio;r;2T1Sd&fqfwZ_cay{45aP$x_S%1eob%PAXJc%V-(ZnvNttc4rI>HlQ8(Hnmdv zeRk)2&8M2H*|V~DD?gfLM7YV0s?8oVuX8G&N-(?3{(Qvw(L@{q-++92L=y`i) zqcerfU)Xga5qqJOd;D*qK>wk;mBjnGD@yWa+hgudp7=wR81ra-@R4}k5xNruBp1;B z<4PThbU?P3CTu7U!DB`}@F{W2;yfpNh!mbrfUHnU%I(n>yGpz7(wgtIp)H}~5hz12 zV%L;+c3{V!8M$14ftjm36AMUCoSD`+)e>9L6=~_r5kzQ}R4l!m2d-b^UXbX!~#^p;Hp>@DUw`FF@EE#~5Ps7G6toK28|7J*}4(G#0duL%pyyc&GFHDE6a}M}dzROt6ix87~(Ud|J<7v4c z*7-vlqyHL>OXvf?Xj&l=xEvULS_BA+o*mjbU7#r_5AHR@o&g=W1@Dh^98yy#B(Wk~ z32>E@jPLl_tj2I6csu%Xl3Pn!Q-fHbKJZH47!VF@E(z@<_?cIoWZBvs_?yMux{oytBcA1Nb$J=_biM#q<;wAE}IA-qQUiU;Z|eUPpD-0c4ZcOc`w zMI2LNZCS9D`O6rgIot8U6z~w$)!x_RYZdf{I3h4uIvyPQuSi@=KO2cR^ONX*Z{hgM zV0y3rctvPZJ`pEyLw-XLnGcTKJvv}9Sq2tenufe-2y%TMPOQLCkW@EWu!vWTQVTMu zo@_L~z+fo~qD<27Qlz)Mg8L#^h8XG4$o%Q>a`N}ZmcY0F7dUeeB&1TblqaQ$Za>66 zDk`jSdh)Gqm4WDNAPtNZBuvX}onTZ+D3DBHmAjia58vS8EB6+gX z!U+$>;ATje)nUg?!FS-3s`A#y?3$IfBky2`cb22;zx9vBmA+km7(npf0#cLzyaB~b z*~Z}Jw&G717+S_qHNe}+*e7th7So40sK-6EwVGV7>7F{mmE>^n$y8k!X7xI#VWZNl z(4A4t(60k^zb`Y6Sf3x34BK_~wljT(RBC)Q8Im5Rm48kn$52^gGH4jo+sbFhMmu04 z0w$fA#Rur1ym|~*LTGVX&oSC|S%)%Xq$2%kz2`=mofNFdi$iez?XL(x zfjeMJ9UQ_`7=0!G+zJKbCn<{ssxI0^Q`>FZ7>>99_g@(D)g`V1HH zA#6L{$?weDrkuI6gW>ad)?%{wcxo)JDwO<73_rGAOc+oGhAfI#6~g5+ln~`|1N`8q zSt5aZV3+-5k=8MPe!wfo8bZfiWc5N!{jsWk#J6jI+)-pg9#^UaX~=PhZ9+%v^N~ zU3xmpO_N=4@r!f-x6`OPZQs)H`RxBgf7t&&`-^Pcp?HJ^18c(u1EU2?svpHmvLwX? z(!U?x8{uV8fXAk{nAuWQA&Y>S;5j05dZL7pAI2l;5MM;Q8naiD$#KsY6|g!-gH0Pi+|t9CvqTyYI}D*ym#K8ymuy&X81p_|8N)wMKW+z7vR^X zPS_mYX+zekDAD#8dv*r`sM|K8e7Zb=g%Dj*!&$eOOxx$f8|YnnV$GseBRq$D-L%^g z9yFe=7z3d)hy-H-WP~9DLV={g zprc6p#hhji3Q(`GaOE zJW=yhKwgjriuEW4s!nu0hr?+U{Tf{_6@j&5c?M}=3S)x~l8g#ow{l~rI{Lmk{T@Xm zrAK#qO!jFzH(s}X7#_R7VEs>r09+`L z$k6Bs9!$4Riv!?+c$Zme_twqj z(`f>B%w6;?9dqxOVEtH&D<8kGKL_I|xOL`zins3H`Tp>91|CwSpwg5Br=&3TAxX*b zKd1n?N;2Wi1}DB;<`)yKn%q6hqT(%lyakiN`PFVtM4XhcsZ}S|M2HJGm^XO}dJ|g* zbydWmCwLq$>f<&HqYt32=(=;L%W1fN4z-~S*6$~ZYRM)uT*fO`PXxs^y<_Yv`lw7Y zGUc_X%>v{GBq@!QP}f}c0jUY{FId3G=d0%YRSwml_((RcpJ&_bFL11t6A3aP)z3<9 zWBeE!xLG_{liyU;ay8NCZ=~!F$uu+=6D1;(b=pwgot~7`7E(aMx*OHWc?I*B1d3&g zv%Nx}2Pd8^jE_{``NUr_o&)B*0J4_9+;Ke6@)3%iZ3)atw{>%=#IgBtFq0D%8yI*B z+KA~z>-nbS+?zJRG5^|9#2VKW=t_4zVEviuO1=hORSwgt*+{t!8p`}ySma5jm?{p< zX%q)J+SCW3?U#X$X!eVa5P$|by|MxJj~~FILD-ye-)t!~Bhv`&tG^r+V|7-^QpI}; zFO7k{mM?T);@KwP#iNNRL5TYAz~s8n-UA}suLvN`OKa%&Zm;b4S>R?dm3RgxZte&h zZtk!PE^3>~APuiVP)H}bxE3^vjA25QoL)n%L?OLq+8_||UXi^^vCG4F&%1UFRG2K*{@06S2cc!9c1n3YqO#$^k)oAO*Nh%&H1 z$?&IF(S{tc`E!<_@g}qcqD1tbAh|o@*HorvjLllT-a4F3^_FZR^$s6SSWyUvj7%1~ zkUYaas(N~xDvAwzJ!UJ7agOY3)>(i{3yM@Xf0FuHo%!OJL!_7VQ{2cD+O(_EAT5gm z{17m9L=&AagU(XSZ&0@8mX*!!+^lAB#7pzR^g8VRp&^(3K2nKq4wFbK7Ki0RxWepp z9Ol5$lpLothAzt6t)Ar$oT1qwJwy9|+w?GgjUm2uHQeZEyB>}$P49L!zo?Wn;EJtH z&$+Em!QPsBbs~Y@s(Fm?w32D4xPF3>Hyx;`wG8?i_}zb|cM%B{jU&hXnu?(tIjrwN zWORZ)PI=jJ&+6Vu%DVFB!b*R%6_PsC4pL5FHL7j>NK`9Z$o=}Zp7H8nBpalI~wpn(>^!T%qFaZKy<{Qd(MqjPxNKD#bvViC=8K2+Lj`NzSr zkY`l!#>u=T#DGwPz<8WG$T zf4>#q2p%nJXrC~$miY2kY7&-1zQ7a-OW_$^I*d|r`TW*LS@G$euQ+aiM|EH2J(Ra6 zvIo5h-CQAvj-pLE@L3^0Ga2ZjE%RBNxP0;qp;BhWkTtZaK3t2e%^Bh)nvpjUh>rQ@ zweRrw)-9K$Ne;kjPOZ%y+E-tyg(|Hx{b#@*qFvJyMTL^yH%~*4cjrS@=L=3(QkR&m zUaSbP^I$r5){_F%78JX=8vMj04yUjj;;tHDWH%woamQdRc32)DegKkl?qb>=UKC?^ z7_!M9z-~rWdj{_`S~mJ|UB5)Ugk2Hy51brQe`?3Vh`z``os;m>ODG9@Js(gP46Xk{ zGYOx2Ubz=Y(jEV|2Np|$7{a>2l^HuAQ-1M+E0efkb?$*rg`Jkvf*-*5F?FKNy=iop zIqX2d#~t#P|F_l%ydNmVKA&NwXo6_D&XS@Ee=3N;eU?ekqI6H7-WDL;_vJRkDV@S} z*wpuU_EBL-tikg=jc%%?A)l(Y>ifvA&wVFsEuY2~-#WYR$O1@Hu+&NuGWBej5Ajw5 z;*Dewq3G|Cvd!shX|&Q?r>y&oW~hMW*A=HCXu1&JTbEcR&i`~>2g7WHzQ_P*)bSPv z{@7Q)y}{(o>%<+ohV&%)%^iJc&W42N2jN*xmGog;|j32B2 zf-TL&9rir%di5sH=rU&Jk%XjS6VKh>EHK-f_9#5duLgnQUC~~Q;aWmFUA$Kb;-0x9 zzK|ZgLB+7{)F6JUpVD9+$guji`0avUiRF2HIcIH1Ye#}3LV1}3+nsEXeN#Q?-e$y9 zb5xDx7o16EZF8qp-jgH8@fe_DVb~M@IBDq zJPr>PY@0kl6ouop!7N6ym*2Tz1k!YnCis>oJ$ z4L{0=tptd!$x@cCZa9iOn4Hb!oUCz+vexNbOpK?%DYk|mM-qg_l08I&@jRXq_unah zX;Xa_g@4{laz0X-dST}$gS<{#nBhEN=3j|+^3~@wP`nju|0u@piLM=TEOkxo*D$;P z6Zl^x(h>Pf%~xa?U~)RtdN|Tr+%(%Vo_;&b~pmOl8$-+tPqioZV3N?+JvGw$WZLa#vc_i=-BO2<3j(9!z}l}f8KAq;T=wQw=> zveZ41AOatgHG=qLp~8KT6BH8X|Lqr)W4a&;|F{ zQGXDw?eA_Hu73y?M^|tf#|Lij90=CGa?fLa;J+QbdCEpVm~?)KQ@iM-FgJU1`wJ4d z2*Uj;+H?ER8Z-cDzygSgGZrq;t8K%sJPE|!_1SiWj$J(wdFB}YBI0=W1M7V(9}&C? z0Ji`HVA_{Ml1GzwIeq@sTRdEq7VB|}WHQ~4%3a}D*02+-H@+aZJ|{Y=`_IMf%gm@C zvJ}>ud>?;(7+VL;Hr&)~u`@#Hf0+x3Y+XSeyfT9FA@@Rl>tzZtuluxi8?v`GqCvDR zmgSRtv=zB%^f0=4ma1YGzzL&9t<%c*!LW%;R1UZw(?>(?9tFFaVl!N2K1p5Jd`T>@ zRquG3um2U!b9neBUp49+%$x?7=vt@Zx=BlbdXUg*6%ejaYs|-l@Jc+Q#R;3v60ax4 ziYo2C)=da|{L!kMd5Fn6+QasD^z^uR06|Vzmz40f*K7Pc9iZg^Ncsu@7FxG(chy`= zV}*+EE05{3KE)3z#Bdu+jjI|ueC^a-Bx4E4RcOg=J&9iVQxww_%ASV^9gsJ&*}m)j zC+MhxD@S0&Z*FPnJ3``%JuMs#AH3hRnk|y_9=hPE+NiqH^XmAvDTwYLJ3DGUG=!Fa zS{`)edTcu{Ws=2!jAa=?BQWj3i2kqkp?y=~EiL1}Fb#XFH}5kZeR`VwXGWKAUUBc4 zgu;9IR%Nbqe&b4@p+5mAS7wCC9WupK8f-B0O?i2v99ao7#K>{trb#n-e;xgn=FCsC zOMdHvdZ@QM!DTQc;s<*nJBz4I_bqRe5Rh{qBmE3-WfbHCX7fgspr+~xUV?tJ|6;2S zPHmxEO|JXE7pQ2%r8OFcU#?=xw}ZOUMRsOcZF5G{$bn(sAPhj|k8!lbI@$YOg|EvO zR8H~Pr1AL>e6WXFLzuLD0?KW$)X4_dqtF$}M^%T+sw{r>eBoUVMn?LzTtiUVmpSOh z_r;iZlPk#!bSAvPNbhI-?VZ;*q3BcKCSOyUTNASbP7@Uk+fobgED9tWn!@wC z2&nZmMWo5xV#bWi4nXcUmW&47Xsud=#i)Jd%|K`YJ96mdlCmWJP0;`TEv3#cDbFve zo=c&kPoc3*Eb%2Yq;}#5RxRHxEcQoD;a&X@b zAJxHl(P9sUzHmziBxEj>p1OSC9$T>38TIw+pFl(HCNO zC?07d5NHfN2)!^0i(jA%LAE$gO>`MEKZvbi|K|;H9(FO6Z(q+-xIgE{xHxbScH}!` z+XR+-hqA7d8aH)X`wuvLugyUIU*l#=2L?v`KV(y$0|ltMv?M^)*fj=P1t)>O2#09+ zH9+|z0Xa}K=}+*mB#J&I)7U@T=`IqOW@`p@o91dA2JMv!%VIS}RN_dRtw^0_HJfU6 zyXt2D^(UQ&%FM4BAGRbZR_K3mo6p;N(|i-(^v~0c286t#q8J|uSuOXDR^pOu%9%>@ z(bM8dWK%%wSNw|-;b}F`?3kAniOs`fUK~88n9(UT&jR+OP2P;swlnTr+8B*Jv`zNZ z(yU_hOtnrqw?_w${HM(TN$L z+iU{!F}hT>QkPbtLzhQ`E$wWR_ddt4SM*GY(&&7FKhr!YDc+=>(gt*oU!d#Pf}p(k z7UY^Y4@**fQwQ+2RGAhv?f|-7QvB1YCBl1C_ST0Cur~Jg=6UQ4_S#f@9a))NE5p@y zw*QgMfvKxYkK5mj+qAa78h-yHL1Dby59gi1$Sr^!-CU%EwKFln_M3FFYN6TOD*;=P zUZ6qiMBlCf0k&LgCSyQe4ZAv|H?Q_@!A$WpaEDjIEK`uMz$~e1gpdbIH^Hdk2kayZ zuc^oiH#ZvqQ>~qDXNmh-M{{!}E@y3xHn201rt})$E=!$oR@m++MtqUTT4#v8iyJ=@ zNBf^`7@xvEMo>!{;d+E?YEutSZLw-B8Z^RmTuTz%zT&r$kvK2rnsK3q!Hx1T zHEYE_?{ZF?@UHwd?UhGzhB*Vk+C1J1I?PWeK3RgSjKpC$H5IU%mUb}XtV0FxwN))~ zwul(9a>8kR&kkfmMrC19n+@N~dV#+gcP?>ZnA~w4bTqZZt=F-Qyhy zmCXG>`OgLY9oP}_*xI!(X_r{KdmyQ-Yb51idX`X;eVnMu%*s28`v-7^$v#EV!c_76 zH5n*mzzg4Butc7R5iBFyQZnjlzI65u<6ERaVh}~!vZz(ef~6YZk53{SFIF&E!VhG)WC%OsL=|7AfkOnmOU!Q z^J^!(7yCq+NPWTtqC3Ynr~?go74=U57|iy6nOwyZzjg`OJ#PE9Qt6dnQ0aHf>+?X9 z=k8mr9F*=^8qCRig|HB%YJsUCe+{NAhMNCG1YT-#5k8(#E{C1vpln&3G8~7~8TKR0 zd$EZW8)%0xL6(aHbtKvLmHf(Lf7N1>ve6SP#AM3%jUjVXFBb{{>4B8Up=bytP)-dH z`UfhXn6wY6939KXVuR^(sIho7(3v{5&=NtJM(JpmKSY^kBiPAMOUrHD*y)|zv^I(A zsGWkXCTC$-3y=g~Dv~!Cq_x&`mGA~q=R6@QvtJID|Vp%KR_p`_t825-T%U7V@E^s=hPPHb9#%zWE@ui_ah ze)gDfnm>5|(avwpz`vb!7Z}RRoHLuB+M5*-Z=y3pQQ6Scj!jo3BSS4Rj8%n4{+2}5 zN?B%dl9zm?SUV|3se~C0@}^bXkZrkS=rDyd^I-)YA#80ODgyT=(7$>H(#VG12NG(U z42M+J<5?ul%x3}#*)7=m&EUc@w=&DO&Kk|9j4LM6I};s$Y9)5OZDSwDN~JFl&2Qbc%Mgk z@_nFSnuoi!z6OqT#Mh?=M_!P93UKT>GWS}=3u$M#=`389{3+Hbst)k-$ffATv{O4j z}gMo3yT zBGqdJppHbv#IZ}nQazigHDw@Jz=k#%>o+%W5q@=Nu7r=66fjjv(xUZuok133xv3Nb z1X78l#YJjW{a8)iO4+Yv79CPb@&!k9%rK2apj59>0PD4PKlsZWYsZly_lWjI{sEUi z=Nf2nmpaHzD?XUJYwpNC%EsfyHV*1=x+x{y0q-I(qi8R@0hhi-tQKIGiOsEpD}?Zc z==BWg0EHuiei6YVsg%6^QZ;|^crHel*ANr$&}i$J@(=oNZD@kjOA z^Oy&X!tG_prg=K0gI3sY`6G>2e)9dLA9_(R*uIO7CnwItF?*UJ<^6#vSg6SeA`2d}(`!9V$Pl{C`1Oi~vz3ijiLQ zQg+ZFJ~F-&*{9;UWsjaI=0DXyELL@o>Il@cBc;DKA4pg>e;JSVmOm)GrHvEaVbFf( zIEaCGovS1wvMy2b#Jx0}K1O%R5oPa-hmRr*oUr9a<-NB6#)ek`^Nt6*`!)~AK%Bqd zO_|A7wk6(9V7)dQg?oq>@WAjly~yLhb#1R{-Vr?5y70m)LBD8PYJ6WY7XR6zCXw&e z!01umn>qUardQ7z)1Ku|@oD0gsRdc-F8{(d7ACi)`U9IZgP+arlLKZoa5>uhzG^AA zMWzLiV>qCp`U&bQYHA8Cy0En$BKiT~0F{rd%DVZn8SMR;^Kj0n7JhVJ2Hq^pAT zxRs1Dmy`-Ygma!w4R%Yk(Ed$lVa5|0ZsL)_)(u}IMk=Lux5P>caP5=ygHJfIrY!qo>KfUmhE_l zF&0(_K7r4SrqrKt-xp|$oO0;yEZmBCoO+?_Mbh2mu=H4W8KZ+Jgj-Z3t8GN z`PSv76ag%vB-H-V6+6rEU-+9EaBC*MX4^iBLZ6966Cz@~$qcmhYIykxrwwQ8u|HS) zuWp$7pK5i;07-Jl51BG2NE}A=NyT4W5^{1SbPeCMVrR1-P`;#-3yYwh@9f%Sr}y#9SSZ`t=i8Wx&GJ)?NGD|X>NUbD+!=;5ad7sg~YFd z7I~^!cUp4N|A(u0?9QwU*0$5>BputfZQDl2wryUqZFX|S>e#kz+qUuayZ3lM?0by0 z{=iyuRMkA|tfPK;p76e*Gv&lp)3*6L9R$Zd<8oDHDd09t0x0A+Jz{$|WKO*;82Xn( z@(hNZy_674F&-1!smxa_yxPJRd|5k6JqW18$lgaZf49YnUog49@UKA~i1NVEbh4(( zgCWdYLLctiLg>aO5J!>g3JDN0ZwFJbZwFV<-pjqz0iVMjNFE?L-bdx2|HwqPTbl4? zD#3{%%s_TT1F9OlFE*1^PUDBur77ORyt6x5o61J^A2_8WWxOeyvxfNo3F>fr^@z;q z0ZvdJbVU)rA*ReTe5xAUWNur7s|qW65`xg@7G2;(gw(iI92` z$?_jp7CGui&Nj&`^GOl2sl686XVX64#R_ro^cDUs&c z%Grfu$D_8R?CxCo(upS%_j%TLI@XDT=*&eq3cDN1IX2&Zc#P{a@2vtCm9El3vvO1L z81Z%}fK9F(i6u*3Ejamt1vT5e?46JjoeOfK5S|}*$`p{T0gVOp1$wR(wV3`vS5QeM zAWRtoLxa~f=sY{*`rY#o^+tK!6KduMKVib(=2@(%*TOZ^wkhFj|JGt zu?xaN8(90*P!`HG*#pcQ*@O$jNpfNV!hAL~z*e170r-x()(?G2mU%}OGZZDP^6H!k zR9rM?_3@XxD*1(s`OX#&#f=kb04*VM`XjTK@%%+Eb2VAT01ytHWkVM84h66D!6+_+ zC~qeZn+A{HgqK%GD{V0KC$ts5AqTM@hVz>o(H&p4U*z4lfT$=N0!7BA>oN$rzQ&0R z;HrXQ+}m%U6@ zo?F5bvU!WhhZspKZ>*R|3}+9KOIR8w%^PEkQsa4BRuMY=@fDcdP70Juh&YU@*@ef3 zD2Npky(KK3@c*9by(fK{Qhu3qdKoRa4j4p+dgC7Jz^GzxC9VigppqRb9 zp?&=+EU9G7VZ+5SE%;_MaA!7}`J*>xO+BSKCsMNDEW$u%>%=drD%X^4%4i|aHUU*Z zZ+`*Arm?FM+z1v@*p#C2vNTlzFf6iv284AV-{#^?Z<42tPQ&cD~}$#iK#uady%b z=2>Pg(MhY#Cue5llTI0Tj{EGaPq@7BhpGzaPr>Ia==(R)PB>25(an4#^w@WOIKln< zdkR4eVBNm{LLKYe+_$3wvaLrB&-%Nn3J-6)rGZ7zo#ivqA!iZ%B~&NpNgdhz5f&N3 zF(p(Db{<3{UEl1J>jRHmeG2dRABD`nrztG90Ix%}M=T>rFg|yfuKS#{0*JzyE~M>G>yhhVwq) zstS7d?djO0#cA5l5fBYP2CI&;uvcaX+0U*QgTN@eX$S(CF9|_mDb_Jqg@NuKcjknN zWCFINclRtSIvZzzay_f0i4D8nH3rRH+)-%14jWTJasw_!2a3Z82eLb)K#xt-Q2mY# z86?idR=J-;UX*JhOv%*v?JijO<_Lp>y-k<-yQvkn_ml|NXYJzbm+|sxuSbr29lwO_ zgfn9T+bOY*=Hc>Wz}=zBn`I!#?l~CW$YTPjc@>9Tync-Vywl`E3uR7y72iM__(jjY zcqo0tZ61&;e4{zPpeTK#%Z~|)6PE=E=1*ncSU$ls*Mb9geBWqj`Ji7fe!k(j1y?Cc zcR?ptPPn3-K(AB}q_IuInSX?T!6=7Au_|}vDM9;IDy|>@u2Rg4GfPSgwZ{@_1`Um$ z{l3LAHGk{{ti>*%`ZQ#S_|~Kw&q&)OOd!S>?B_mIi-xu!`GU@W%(^9O!wnWu)@L5A z+vg#X6`3V0wpLt|fYiYdS!n}`>z!Tz1)l;I&nmw?zr~B93T5;Qdm!_v`G)g8nMBV&8jb_ zB}+S|S(9FCIp8!hE#<>ElriPMXb}E;3hZ{ex8<sE|J(Vy>E?d2|K~Gh{l`8c_+PQHMiL%iZ9iv# zI{b~0rfO&W&$O~a?o5rL%h3_V#6&71d6QUUQ0G29mQ+>WP>C9&Kd1l{H{XL1@Fy7a z_@j&!%O*2Ff9y2%?M{DsEg%5+VUdR=0o!cU61dLvTQQ~}w2jrhu$5-A2~@Q=Hc&V8 zWON0Q{wI%thKHF(=zBJiGFK_!>14j{J&HbVOvxxByKN4U`YG=HvkqHpQHAa@6Ky9|z?fjP zxXv@AGdyfr)p(;j!|MzMy`*EsbPY?zGWVXmdUkY3QG}W6(O$?goro!bc8kvj zN`8a9VS-Jjn>Xa_&jD+J9;?A?NME2Xq>Kp$61g`TQe~nvK9>ElXOb5Bjhb@GPz2h6 z&!n$Di|%{q48=e{IGq`>5Pi>I`umUo>T&Jj$#U-0;yic<-^5l*!>05%KEiaSG5`gZG&rV` zED;5m5eRK2R$e834=A_;hcr5A*X0|hIENFUY#A#mnjf_ne1PFU2Sijd^FJvLKl zauNobscB}g!Kr>>;eTg<3Ks-)&Om~WnUc(^c_evExCIo@{L>y1-DZVF3{@ zV|VnNw|et)482el&X~?n$0ZeyWLbxdA`mLgIY+!g7i<5_8fvKDn#lBF-`loD&cZ>f z11x#77TjDI?b4&kd3_NtwtngJgSg#HtQ}Y4RK~a5Wp0ZC{)6GclU~_F|7SGN{)bT^ z|Ns9h8o<&RM;(pNSO!}Ye>m2NqR|**km)b&DmoOa;08noo09cvueRgd5}r1SC$eW@ ztat>RDOG%`FO?!?a-GPku0^5Z&(A4)BOZ4%+SMxAbp)^Lu9EjFE_cu8NrCVCApwx= z!EF*;r+-s&e4r@HB*}Io@o9!4A!O^|ny9M0cmN?VuGLyG>X6+Lomjy^xz#SR{^H2L zkg>ZcNIuG6QTrK;{Aj%;w;7NwK5B*Lsy^~Z(-IEI`tiF>k;lJz35GC8!gpm1eKeE1 ziTl+DXO>0Z+9K_PZ{?7>(S~3QeRu+|ciIELjWD-i_TQae#LyV1IJtE-=I@itMr$V* z>j1{+wBTB6b+F#V)oIe=QoRvT&i{&J6`H7f6=#|>GZqz^dl=`_j40^*`Q^tfFY0RT z)&*WTjaDUaTNLn$gOk+y!sGGKK$5g$GlGmvjnW+M#QH~@Ee%?%r=q@43lFmi;WI-P z`x(&_p?T&U80gIXAQ@;-ln~m*wGz4UbO4ooCH7NbS0)FT{s_Zq6O}c9Wv4RBWK%%5 z#+eF%+I4&>N~~XAcPZzQ$*WOFE=y2i{~ZYam(QH@eZLmj-sGIgzPNdLLT7bJ%-!Y2 zR_WIkvO~n`njQ@_cBUm{DPc!>S~Qj}DAv?mB}LO(2B0q5T1)yF3 zO#fjM@7ZW`%y3%dDjf}^*jxrF+TCq}EBHOf)*sAWYEyq>;0x!xHJ|BTR-72zejdopz9k+P97I4BwQ$dtaz>d0c=(@T z5EF78^}Y(P>amz44R%*oiXQ}rD)E-?_H$P6g<+H5)P)Y;28YHOtBGLGcdUEUS-&9Z z&DKP+ZQTXY4vm}fTd!dI(Q*s)Dqtlihvtr2p$aVDz$ExZsH0~4K8rE11N@5ima)wk z$D{sa_y`VGVEYAaTfN}nrEkf894NdB-IYnnHz9msc8)=_(6z^yRP0*Z_ zT~DRZ4R0_$eXQ~*Q6ASbRqPBbX_hk0Q>W&nt5cI0y-&NO_37WVh4PeIlt#9@gV8^^ zT~VKKuCTCe4C-qV_^}-50_f_$V@%|<2XV07NQDYsaK=@s_kE%lF<4}3>s;}I0V;mu)!(50SYvCYocqucS|FO z6T5#1*2eqh5Ifw^!x5xKU{azqH-W7NA7?F$(5Ai(ioBPXE`#%MEJ+p%%jY~W=We^H ztb)t4YW#Pkxoqs_)e)l*8>ax7vW~WOvAE&+7{{XHcU1#k-KG1YpHCficz7Lk-A>KY zv9-J4w{^M(<=KBl0O7Q|sNIQEcTtsCn;sy?c{@QmY=-gY2$z}^uf`~axJUTj#KU1N zUIOMJ&WuVScbOGl0mi{BGhrZCgA$aMe~|ejnWl^kHaKn=*~B=AA&^ITEXuXHgtd0u zRsC7DXRfuluPonFMUl0HjX;ySCclxT4t;nMD)U++1<9Fy1H>H2nX)U1UHEH$x@vww ztIyU2jX0tG)v-it5Q}p-fp;j)>x_)O@73Kbcz5~t=vtQgQ8ok_k4Ib?xR*vwU1sv| zK{`gyrwBEW1fWEpfrZvY7%n%;;Gbq%H!}_1)TrIS(lcYo@wz5y9b=}V)V#c98(WZL zj!9q!5?(d{c9o=wtnN(jf|L%Zdprq+LMPE&R2_YauOcM;R!F*i2?Na2Tk7eXgUt(u zYNPza6gK z%zo7#o=};Uf6Fc3Q5ApiHK@mJVh%et%pJp%x95%kN+Re!_yxa3P6kbVqF??-PvahW zKCvEx)3$9zFRa8FjGBI99wH{Irf|yL6~0$i-xnt9nKOH}1qOt9>5DVA=Ga{Q-RWGv zvPF_tTCSJKXtc~X!Ti)rlbPVr7C9fNiz-&$hps{%9sa~vdub%wM&ZAz=cQuz#w>Yf z7D>zm1i<)$5v<{#!SJf~e!4#FKVUvENq)0>tqA4jKLG#y{evBZ&l`be1U4H|vLQX^ z(HbmMKVAer){UeJgu(M`e{s>v!-Co{VdmBl+9L}_sV=bO!IEpMfN#Z_1-4^W8JTF; z9jB?X4mvqmpZm6_?M-C;3jfC|q>$W`Dz*yY5-YrJ-ArQJOBz{17NeJFd+FXqKV>V{ z8B4wA8t^GMSF%>~$0`3EZf)=jJ-q9rCYqJ&PN-ep-^vWifw z2u(x>B|{zXTOq3)HCmD5z)A31y_U%3&Z$N4J)g;tH@}d<9c$=%R;ZJ|ytWf2B+?>q zx@tZCm}))#n(E|uf9?i+fbWR;U`Sb^r~SL2_A#KoxI_vpdKdsQ}>f z(0kBkpccXlahhCRbc5Nb5g@-OWL|h;G#29%BXxqWTj?95V$xslq1_ZQ0DRA9_mS+?+S z+5|s6`&qLce<+j!gnGP3wSbMDV*tiS4?kPY=Htf0X!;~gDZ2W+)zIv;Xgvzs1F+Cy zS{^j{)ir5xgz9xI_KNl00r9EXWi#6-l;IiYq)e&iN#r%a>UA>p$VHUsy0Xz+RDz0xJglavd2ne4RYTe$Gr=aEG+6}^Q%leovH=L1BpcNqDQqDo)ph!%uQ6s+#-Y2cBLaD1Ga2$bTh5mk2+f} z(QLkP^#xus82934V$I-Q1x>cAeCL*%7o!?$(7t@_aF?x9T0_4pdz1T=69y&f2(`uF zV*@)PvozElVJy*bUNj7LR{t5*)v7MAlx`v-6>c*kryaEiF>=Y+vZ6|9r#5YNuP#QD(^aRW%&t2#N9 zh94x-2&bG~|U5S-L7<6Y!c0XfPS#Kpno_x?vkd_iAzq7pXjrC^LW%1v;soEDy$Do_v>wWTE+Tz zeJ!g8N!EYo&Q2zS)mSs>z3KRIsYlu)3|b-YNsaC)d?Cypj`3d9^t58%m`nQ~BOP>O z_c)7uat`ZFoz0Bq##e(KN@?}sf=g`o_=a!mN)niMTEh~HmI2UzoD72uL%;Q}(=*_yj#S?G^z2yl}>lS>E?W`oURmFjj)O&r+mk-VIRrZf#3t z)-8zN1g)Ka{L08L@?YA0xA12C-2e#C(M41jn_!q}^$4$D~$h zgXZKugLy0c3}TM`t;@(e8?5YU_VN`-^m^Um(CAoXilNX%yt zzU+AadjJSV2RDw(SgSC|E(gu+GYtt`#_t0t2^lYwuu`_%AdurT&!5}1Gcfm)skeXp zXa2REuMDB)pDrQr>mQGq2Vf3-SlgGD$p!NPMYBZMErf<9hLTj3MH2x>-j1isK9p=t zY_A~qqn0Bj@4ET<0*q#Hhe z*MLy-r;4lUv&wm+rW2uK7W&!l`hE%i8CNUHgqG9A2jWm`-C9d7j-bKjJZe}$-e!6& z<|C@E^YdHVX1G2O|N@Jw3N+C&8_uivvFuW_F8@u~ z<>I1#7fs1S{OG3p624tuSa7z#>U`q&Ok6J7FD6ENb{X4j-|cd8C83JJD^tRh2f2(H zO@kkDRQcBB8$ErzyV$*^rCY7~+;qP9w{tnmz5>F;AiXzT7|^!+_QSprvv7*}A>|dq z#Bjx*Hv=?-ND?Q$fJ){2Rh4jfki>>P_Tm$S{H8jw?^_-B&4O6mHyHh`@`h{oz7In< z7gYRMf+w&-cY@`CV=}jA!seT5V8Z4DwSiGS*zwMrl0Zy5_(y|hxMAUbJgiw1W2k^& zo&%yz7|scnIG}le9NnuNE?q$I+l`-=9t>kI!1>k|6JrnU;;O*T`3>~}29<9@KJQ4L z6Nu>=0fjo-IWvT^;)q5{9wbXGbk)a)CMgR+RWNk%Ll{8>M}b<170W~%%1IwCf0WR< zwCn+76gr-YY$Oksxf+bBObEg>UTlH(5o8o9z7Cm`NGga|JuuxWJ{YzxGP4n%(DMiL zUofdLLRGSnpKOxK5UYZdS@?wDKbTRh%>FwHkXWx8iwp$<^8XFdQvUCimtq_ZkfQz! zKsm;nJLMK1Jj$^tm2-Q% z|J}nwK>q^-L4oy!b=QWAfhdTJ! z)5dBn$`w@~sUVG=y7@Tileb zr!7`*p9*cUR&V)`(cLmvJN}wpif9$*$ym_-JXuM*xFbwPL3xKj@Cv6H4<8MB7jLrL zW=h08OR3qDbT=a9S`nB5s79OsfA>&vjZd7W+MakqjpHrCG$mLnpiWDDijh_Q6c2sS zKXAY{BXdp9({LgfXQsfS~cNjn>ojXJY-$-EIr9w}RocV`EQo6^JH%zoF^92^ z?{s>PxV1ZtmK27;q?a zLr4nt{9zoqrE50dyUQ**W|i5tKcK1=hk<1_!9q?I<=_UbYun^!otgYGngvdy3auKE zkBT-jo=$;6rz-;kkaQjyn+b_7D?<6x0Ewk3ZK9h+yCx*=Zte`e&b39dkHf5&c+1=h z&}r^GOR#fMM)!caJ^4-|tE1-?EXIw&@2qY4h;M6MVaC$e3bR}*%uPLCT%n(=bCnsO z%G2C8vCWBtRqk^jBxBu_&I>jz;D6zug3(u{6og+Fa!#}X=m}mq2MIOS@_Q9H;e0-* zZbs%@&10IDOn0t}NuL{Q5jl4{U$)T|l5{B&$pC5uL;e)!a}pDIBodkG06X#r*y-Ar zM~)*JO{x=7H7^$S>LMdo28c;nzpvMbXgUU|iL`pJZPqs~i$Z@;dW@)=U!kEsyhA8Qj%*b<#<|p>bE~T@`>Y; zsQpF~$2gHQ`G9*86Sv0XOWqNqP%(P>CWgv`3b@e#Dl`F8_D3R;v_i_0*x+2I$755l zUoRN4FJvfZK5b;4HS@G4bA2)!`NSnYxK~~Ni>s)f<$y>HPy0nP@-XqI`F_{X{ngkuk8MG)RZ4L zyJkXV>Rom3@|lpfj8+ZG+IPFT&Q6H0o_owVMslrVMOk-w#+kcp^{caQT{mao$)4xY zFDKT7Zc3e>QKsOK0oQp^l~*L&^Y@^VzLG1zQ{YG%a%vXGGiN>ptz%G&Q2DEvG{g>% z$tjkT?0Snv-EPkx5$fIMW*I_@L!Bnu4Y^^$8>!sL>!?)21*okfJp3iq6a6Y&J z1=}n#UWc}Zx>45>Vj(~ck#9uw83CXc>yF-rMEmkg6?B$m_jJ9!&^!FJAKE1k-#d>* z_jbK~v2U7rH@ums2*oiSS%H%z6^m>nR31g-nJ2w2!P3w3As{&u14lLC{un4Xi8314 z)}6r)w0=;oj&n!2f1;c9hU!+t@+W|P=;MRT9f2j9mDwWwwAeZIs#BUtoZ6Rw?9JWm zO)31y38NV2l?&V|v}n4iRqkV(PR?}){21RpI2=5VfmJwL=WyLt!8eH;);~fEu7bZ( zXIe5UHD$grnhmf%&F%68l96mm(ys`|Hc_8*S_s-1BQ`xyy-&V6gKIF39T4>LUMPZL zem3aGig42EaEow+{-`yQkTx0wU@4R7wBf3L6UAFW|B0r~_-dTPEmHA2v^<#63f2^J zhT&q^62Z6a9P$+5az=8kw~Y zk?>$-D;%#URxj(;9pUza8;|J9SWFHd(HxNoGK^!|Xq=6*_U4GohL1N6&>lMIw2s@P zZl=aiHWS9JoR_R=t%J5PQGP%2`$c3p|tqneX4x(>EL(z9P>dUtnaMAF&g zAv3?+_@SU0q8XxjZXz`Vkj=WGtJy!yyA-B#PA1#qaNXSg9OTn|ch_?4RO9i=Dc|BZ z7B3_e>s>+OJAEan-4Re{bR9yRh!Tx*unn84jL)*ieyql%&@Y;Ws@ylyhq{U^4v#RW z8?w+eVi=JBGheg>fnnLd)2=ww$dO8LP|H$eDk%V%KY$3GAH+Ndkis|xAyVa${Oc## zL=nzGe0zraw;b#!IqDVnGB7`u(!0!}=>nb=tSY~lh;iNvt#>J7@mLWjN`-k^vIxwM zoE9-O(DDoxA%Q_Y3EaE}89{}aPxxV&*UyIPY)*NhXmBw^eLlFy&t@R&l$J12R_L(; za-0*dlB|kF%$~fIwEgZc`Ru?IRGXX)XOAnOvq=5r&@nD7E3%EXwN26^Y6tMrorGkX z4Ys%;P?c0`DhW7WBW`N+6A?&`Q>IoZMns;oU?uvCG&eA8sJU!wkOnJK0O;G!XJDNj zAlQjx-olGlOl^2DJl^8+zHW6s+S(KV0KOm;#a9JEdAQx+L~D!%hwpp5kvuf}%xylu4`JlbU>%K^LE9o}OhI2ROoJ5-Vy*-l zTI=;p^a8dkFeCL`#?D}ulKUQUi|C}Wv7(ptY1iw@Uk2IGNFBN zYftl7T1ja6qn_1`*7dj6u#5PPwvQ zwk%H>s>k_5DAhGTA5x_J-Vc0bJBQeYy_M{2W87~G?C%R!Cx8vk^&izFcnS-@xy;~_GXBoM7;A-HfrMy63pNxCd1G^#jo1)E1Ulo8Qv zZvyNeGUH0C>s?Df-{o)H+r{^-P+GIEJUt?^H9-4K1SkRjrEt@) zAdjAi>6aREqETQMB99xRf#)tNVMgdGZ>VnXX_r*Hk7fnJN zk^VwdY{a8U`>cV$qXSObMU(^adW&*2X&UobSiKRi%q|)1zS!t6FvD9xj`BQH-uyhX z$-p8{;?nJ|1{@wyWIl?ocf6m-nKW5Ep;u%De`!NM0BGv%iPAYR3=)+lmvsd=&$9SU z8s^}<{kCNuXaEw}&yve>54e45az^@n88mxBihUZiI3esLA9IwpXx?oghca%rkagRDgxtzJl*uVfptVy-?enV6a8 z%}kEMu3J>e&#>dxq)h!UQE$*}OC$YD9%4L}Oy-!?&j3wVZRhiv((-CiZsrXcExAco z_B#O~n=-`E-xN*S57-nsm$HdoP@M$%_K5_-2}xqPOuBl(os+ns^z}%OK=KeT_8WYe zDlK++-rJ@*6&lSz3%_k_SNbfqEl!P9Bgp1<*UaZv=CW=a%f+jj6qk0>KMN*t1@a~9 zUG=+?Jb*4%Apmx0uK+V9gC{lva!Ou?6-vNqZjWDK2tolX1UO4H(@h&1XiM<8Kk023 z;ZejI@2GjAC!V^MxD4rK$*UFZ4Qv}@?XLl)M~;OR0`#)JnPSw1w>M8 z38)iHUk1ZS`2)zIiluVM0jo&A=!1Wvm8fLaE#OE<`Qu^x^RPw_G4ocDC)T3(>qpql zV-`w`Tc}-AyL4|TrzA42*UnR}dmD&K310*L(W~9@@w>*|#xDqKZB!sifxCRWGuof` zE*^<$7~mk?4*AS@Q3AsT;JXy&VA^E?+)bwynsU2M>BnE-|JPsR`*+GSY1igAU_%X> z7`kt#FZF8PGF{vZd32(*`mlVb@7E1*fRs+P0n*Gd$=m7LU18*X_#DTdg;j>qU_96s z!xaM#^f(*l&lBWX*cYE~p`NJN$)n1k*a{yuGAjLHI?``n`W0LBxOD6prNWdNV|W~-zu3`E8Axv{LGMu)yfTnSSzt_&J$_Z@Uo^eKU`J z7joM-9VEwk1cKH9OVxy#iY+KGV9~M(Qm(|mvp8vsocVgLMS88w;LVg0=)eK{$Xqs{-&mguG`)T7Wi}dc{dXqy@u3RCzo%Xu5bL9oQiH zU6-mk!Ep8z84R@gmZixW)3O?xQMMFOh{>v`u%c%vI&RP$4&S%uEI6d2`@5zkEl!Bhn2Qa>IMmxCo!63x&ovSK`wjbSRd-%N`Q{kIH;wL74G(4`-0 zjfR`0<9t)^f>0UKPIo3i?)M^Np5a6IzZ@x@b zPqn2llG(ZF4VN8JSsG$YuT2U&Y+X9nonV7k-b!)O0Icxd!%$A5PK@09-{DaEEDgT8yX;uYq= zt9x)NN43?$iea5I^)uSPpv<6No1fd>#uj50{o!EEjm)19aME|=Iz873QCCR8ac@aw zz_Hr2=!oGDsLCJR?MSbiNhiFUG1}MtN{qqKsOnOMQ1_V z88!#%8l4QogJ1791wJzs?ELhh1lG5~GV?3nF!3ACj^(L%1vEsk5e5c!dIZPvX*8y)KKpePR@&>$+px|a49^s;8-%p`@ zkmY>JUtJD^EC=a)29SesN#oe5`SKQ25bsn+&Hl`-mwc0@yG|Cr+~STYp%c%+`HOQrqEIr%-_psTZ^})8 zfdaSmDlroy^dhw!H*{rnAMLo*_y|+N&@7lom?Ak`dNa-lG}1`2>%kY@ zl^6ilxG~-dj0_?tvnG|W64BRXz|cGymAq)J8Ks?DiNNd zQXdl9Q@w;(G2fg2z*y3bbwL1a)B5>+XT4AA*6pnd zhwFSsV#4Vsa2-3Qy9=?Vu`~VXlW>K}l_Mfws!1`*=)9aVPN#6LP7qFwlV+N3PJb^^ z^rO-okni)>+{$m*k&OBV{lCfd<1(_k=-&pv!+$Sjl0yJ#(wZwWpz+_n$F^|mAUz-@ zWMm*k_@$!u99ZJnu8s!?0$Fj3%P#DF7{c}U|A60zI7KmyM8l_#%$&E>BBAOhSYs9j zyXJfnmuhvF?&zWBW2RCZQ|J*k^yH9;$tPmy#4JsL8VBXb0=t8ejH{ND>?(6yZo!4; zDAH%jxD0H^bVSUb>Pdx!4>vho`W_etI{<&+@`(Ng)|o%)+kfm288btF<^MTD50n15 zi2>wMln}q7hNCA(p+XBu-E7xYx&ki}FNwPXr*~lt@`cEl zzF2ORd?1Xo>$USV3}A?Er@5TXj(nU~Ggj~Kru;$eHKbk!$o19fQ23m31?j1MSrBLOUV zxEA;cnPZZeXK=o~iE=c}hL)jIOhYpVGqF_2TEUNXEbY|Y2`J#2(-i2mD)aXuDcSVy z(=;`szp8DH#P+CfPR6mC__;^aLT6=-nBv*s+>r7dbXGuk-_hSAhcmEDo zYoCHLLrl+iO52O2;;eZ0hqxDRjiMwhYktQ~_$$Pl?VM%el-^p<>Ig7{I6`L7D@24OOP>fN_ib67ItDO}~UU`5V%o>6^jsyrH1$ z(wxh$fE)T3I2LnpnaYZ9Y2Klz5VBAkPsviPCg!9_rKnc3J)>R_J)415ktzhrhk)I-Tem zN(uSZ!2)O;{3D5nL~n>l(d6RAJq+s_%(*(RGA@BHNFVfF@uFn20|+}|%u{;PPIHMr zjWaK%CZ@j3%-rqX@0XhiK+GK1{bQAWT1LAju~rpW5JpJ-RH<>1Jx@0r`Q71_6J<4u z#NMwsn#5K??z!0b7Z7=_q0nlxvRLGs3m&?FC&Brp8=6q-@qEiH;xt-;%v%CWv;HWj z&A*>TuPLA00qs7n!!3;vBLx_@WHh<%e6W2zh`II(&o z+=>?kbvb;J>0pP-o9SqtL?j*u1+%7;%Ne6!>#SP-h;hK;abO-9pIYxV-fTs{t8;j) zBt|lo?3tuZ*7UIb*i2B=afHRU{n|B(^D-Rk_SQG760+=BXKDJQP)7mqRN+HHsiGmcd zX8K3?hsO-!#9+pshmVZia3@`kIx;8y9jcoP4?^>!X2!+#hcl6))Uni@WS74Tkrz|t zs^Y?)df|j`IZ$l{eQqM6(5hiXJSkf}L?7Psc~oLhA!CrQX7eXjl^C;ZBPSIs_;7I{ z4JJG@Hb0b`UQrgPY#T++5dR&w$o*>J+cS$}!cidT!TiH73nS1W`~REr3WtYxe*BaE zvS2_!c>m`ZEBfLA&Xw22QNNbZD@mdTTGE4?3gOv)`1Ech8f1}zXKFh-Xi+qfca91> zqScg%en$CM5mh9}M4EZDVJp|w7n<%KkZ8b0%WREpK&?=X=IF${ zaYW5of(1IjXPY70g;D*+-Duthv)rcE-}0DH`5G~W#M-LZlR_a|f;F-1sd`4(g~|NC zT5h`uwhMBx)0hyOtvFHaws(zO8Gus1>f(xWO(_i9aYmX}`SE0qNBHf5-fAs0BHb{7 zH#gT`3dF4_?m?*`KgV6F1D>o zN#0!)EPG5e_|CAkdRxZvQIEw%N>ky%xK};0FqdfAz2Sj|*+(to@!YA!dxzf8>^EG) z7|_xyCgEPEWjp*`CL=Ma#&ZH%v8nsewg+}rRoIO)R;|CU59oEIgk%FW}%tn9(+R=wh1 z-6!KzSmG?QqWx6DgvTr`S9QtS_2H4@P|@oItlO=b$0NZGc^lS@3VA^wl)j5_sBQ_L zS~9G3T48(TK&3cKPi9KjnY7(>02-9Ni+JV^b&;;GIVk(_zoD)P`o78R?l|?3ims@I z<*;BvL_JF~2>6vQdvECf+Rd#g6d)^nfa>!mYy^B$XuaRy^C?8~vB&W@`4hJLKaa8l zWW9rjDPKcb3an(+l}_4g*53vEGuVdwpN<0xDUFKLa$E zefA`R-HFnW8(@c^s(CjD1*v?L-{4B5zkc6h3G-n3v_?5dbtGUX=n-vup5aQ*@WIHt zu!yonDW&#fY6~aoipS~}9XSpHyLXTQ>jl-uph=?I6U7!sB8XYxBvSkmQh!j3%UtDn zem#B$;f4($VL}IpX4~R53u;7&U!4p;J)UF4u4bcfUI{;eI$I?x@C0xj;eij@{J_X& zaGAYzgd&(}5w?O?YW7K`Wx^gNo0T(9$Sd{~Z-E9UY&%sA*W-#>50097DLd%LS7-I3 z8~tyWpn>zbjS`i? z1TEJF+c+PN40)$bftm5(5H(SEkq??J3iJ4)u5c5Q3z=blk%&DOQAtE*ZAY*QdEqpl~Mboi0OtIQs^b*}Kl=alkim$BP!#PyOC*)7D}FJ-w3 zAL`Ep-!`4S5@TcDY6CLO5UHj=4_B6Ic5I8O8-fC^MoPvcv#_k1%_4D89Y%Yxb$*>Y zKoxD5;mR6~LA<}60KQk9FCqy07Q`YUqY5^OWs9#(C@h`;AA&OTxs1NOSxgKjAFy&7 zm}1j*D<2__XaHwTjUfttE*~c@eqfx1P#@ttTD`y zYEn31^&)utYG4MGBbNd_%hIY(8k$&WH%jy-KN5+r>By;TBza}+5oUp+` zfyPyRXh76VYT)S%Az(}U#RGGR==Y`>aLx7uY#iccdjQd?2`;u z2T9%zryu>VAKiLpI$oAm0B-~Vit^#JP)#wZZ=@lhBA}3h0b*I>ZWD;3Q#N`VmEkVa4NNOi~8I20m4w;9lX@^n=d(wqq3lR0Qd!{PoBA9Ud zkB0~&>>bB=brYg|$LZ|+xGN#_PD%2+lCWp^2fE?sp+AAo& z(;FYkz76^7i1^}RJp&s`^j9*W=tON#XfK;|tLD7*eM%4^-Mi8L zL@}*5EjDxv2QRcgm-*skh_}&N2h*wn@5anB`Kol|7xUh`;8Y-6%(U^`|e|?1X_n1TF(h-&WP(D)GEh+V-GQQB5w4 z^Z535>;beRVollQTvrLS5Ja)j%WWJ5yc_ELKES=ETA#(+(V;Se-`?7(oJ*k?`{+wopkT6tTu^#qRy)05?~rqr|rQZ7t?R( zh=SJpEL!?fyCvTDC0-7@Y)-cJ8%Vq?Clv7>G=O&ECBs%~3Gv=Tzf`#SW~2gJ<{VQI zC#gbFk zpUY#^co=sD@viq{VuZh!$DrI4#0Wc6v_GrLNR@d{Ml`3ePR(O8yfVB~?9ceV#5>x@ zt#tJ#I^`0*v%ZtcCifEU)B^zUu1s|`$1IF9!8Qi!!KhX`@@@!KdfIwL zsX5gRhEk2{Bj2oKacaH& zVhf6FQY|g!sqnU1bg{3uPXbrp@G}WgI27A~GA<14Dps$u?OmbgkL}inTWZGF5ZQo4 z{y3boUT0411=|k$w_)V7VL^oD6ueU}0p8!Lym(?Lg3h{pO2lLPPyK52mUTupo%2N_ z%X!8*m{H)rFLl;dlV6^zrUvP@^eu)M$}OE+y^&q`T4Vw^iq~HsKBk1d{0EYSeR1z8 z@Y0(gbLp3wfxlMdlF67y-eqJO2B3679sF!PaAZ|Fe^!gr{H^Ch{%may%UdR$1Z6x= z5S`a}n-RY2RhmW4O9?M*UKRR3rEq1|X$ zM8zUA^pf3F8R^|+8K?oyK$+xF8(0FqYuSrZHzW!XLQ(0?U^*wvraj!{D1e6wb6k;5 zc5m0GMl?zW62Vo7R@4oH12J2#dW8B*aGcNQa-;3lturmR6xJrsR-W*OQgzQ{2Yus| zffL$%+Jx^=s=uKbvLuIgvdZcN{PZ(?m(M8TiA-~iNfA6(3q3bZSZmMX4A}1zK`4>D6{yl>9aB#N{+ru+7 z#Xf0E0IoM~eCy$R`BxK?b^Wwt)x_8$EYeNMoejuWm~kHePnfijRpp>D);9peE6Eg( zAwo#^zZO&}2D=MOK!dY;AVZ7ZPv>a|a9G5!N zn59n(XL&O73fqCd@HW=re&I}S|t`t1?(ppSXlvDD4y7_E2yl&H6KrEd52BR2ImdG4ce;DCZUQRFAxZq)V6tJ<|w{ z9KJ*LVni+7DI~S-Jp!(r>>A#zO~d_pRc}w+AKSXgSEDWyZj5uV2@?%ynoYC((KWg1 zvh8ES_UXkk%&k%?4NV@tfGiw4Cj21{rKAm2`yA47bMZru1FC_1a^#Q89Ut38Ue5i; z(dgq=QBwF=c3`%guZfO~1HO^-$rDHNE#*#C53!Eil-5Q*KG44Cq5V=;O=BB4aMH)U z*C?apo*GGAPnsj=Vz0#e5SE~sw(-F=n~f!&6%va9w>3iU!96%!M*L!M&$uq^9 zdAgt%Bp#J_OR&iq?f@KqPTl4k^!%H(!&)i4+IX&~o@)xgo=xiJ=NCF$r{0Nu<3>KKkcS-bd`kV(-amAlP;Fdvj0ne2kbxn4p`nIL9b zhN2zbn9$iF)XGDI4nTyT%@2-tl;*9zxphLMW2e63+0>A1OUyjBUgLL|!VX*-`D?Kk zf7r*d5p#$EE_;?iS{Z6uu{-ma@b5k*K{C*#g!|zIdHl}u%>`rbad`SjDp*D+D#`oj zzb_Y3_+NscUe5ModRcl)MAcfc#fX#4IjmNRSEmciGKsL(7 zmR0Pu*GiGhr9VztOslXZw=P24sgu$4q7%7n$dos-dc9Ajw#a^-GN%HUuZ5Bg}uEDN2-in@19#i*%%XdE0gkPcIx zo~As)R8v%?&n+xumm_Pl29UzAVK$mdsKUGmUCTKcG*<_&)aE#~env8hF23kv-PV>t ze1fe1lAmy5KGEFCkBIW4@5cDoExfqdQiV!`%;~BX=hsrt;jc({h{=S(+;sZd^A9;_ z8BF5OW6hSl597!<-MVZ0#6wMP8(c$Qf;(ztT0?soT)Hw&4Ad0fn*m>vC_DWS2b8<- zqk4`)KeU{OHc&HkbaT9ClTJz)P;Furr}8|6nUbJV#8LS4;21a#OD$ZK#n%5!cX+{4 zolum+J<6nvbzM2Nt|%vsOb%N}Uzm+o!?*ve-PakWr>$I9ZUQ%f=n|Mw)p;XdZvnSL z-%{zHZc>+~SGi@8&Iceq_U|eW)GedswR$*GT2%`h15{K8bs(Ddx#VWq%{WauqqJwk z;rpK9*Deuv-a^&hNKU*?m}#_Ev@cfdZnDU)OifMAG7<|8cO_$V{4~0{cCu;QE^&9R zv1MUODwrveB$-G3aOewJk(}*$7lKwmiVrq`*B~1LyO23g?GEU3eZV61bH>?Kde@oN ziE3m}=L({jFp2QBj+(>Moq@c}S%gJ9hDLmuj5d=~SI>J28VwikrWxwHBNCVs&!u77 zLm$FpBH#aUkzX52cb11nRVk#q#{FrBvuO_xn9V`Q7WE&Qp}p^{|7fu2)ithW@^ zbb+XSNPMfX*5^~o1H4#KJSLqX?3(*LIgiv=MQe%^FD~@L^f-F7U%%&AB_epL*(sch z45C;6HM5pZ`F*kjSM3qF`l1T_klX^ugs*DP!> z-khVc*hXdp4G<5R=P!f+iDXJM z7V?6;o@W;N&x%1{8vPKmWY`i6GSV*mvD9RmJiZ!(YG^iO%Xr>g@_>zdZoDLD^LSAj zewYsf)s@x+{MAR@yWMk-jCY8dIqmkBr!`}jqqI-g^6{h{J_lKG2JV?s?O$KwCO&G} zZa7Y)UU_{WO|6XEP$sDHH^%$2#ir5E<@OR#=2hL1yQ^q)IEy>zZLCtL1scTkOEVG| zs7EZo-J=hv`}R`$CIY6{IlLB)e<}+vYsj?_e3|l4=qL!aff~bE$M$$ciEXcS^G=R} zWds{pE-t>!rPyX>k>7!un$=?`XufLkoU8o3t}wyP_*1Ebdk>*rW>Iy@y3K9BWQ<>U zJ<&f`ZDRDZ$nb~U$ovf5E|Zl3vF4c>t4HYRJd9iYVeDq>n;O7FkO@7Pxz1xu)FocC z$shOCKiJRoZh3m?=V)hHjBjRGDImO5QiE&CJ2 zgw1+gQ~O>0xu%I@>XTW9*~EzEmpvM;!o1P)wDR%{fpSMQ#T z?nKivqz257gKdOtJ)7Nxhv@}-_(+$(#b#c$$mA{b%|7YyoC z>LVVeSVY)ch#N0U?Qu6e`nH-q^BSx)X8z-K+;@@4C4zy|otyDdmZ0M~KEq>k2j|~< z_9)YsPk!NE!h>Ws89!4R(=7kJ*j+gMDDf54J~aaw()a=DzZF4iSMLwrhvQDkx=^1~ z(4xOdO{fg!eZ;qskAbD4`S4q>Ce7Ki%jOEXD1Hy)iJ1Z-l1eDxy4cBG57ibZ2AsZ{ zoc8@OHF@>+{Ns%}fLGvDLRda$@LL>5lo(?POVK_)15zU&2WBV@0&7~o-7bIaZcUgE zA{_BDZo>p%pjTWfc@WR??G9GBm z)Z5N9(TR3Vzg0@#W`-wX94QW~fuCk#2)h)8jR*s3Z}<+|(wSZ9z5y7I9>TRcNl&d> zXA+`Wd7EU#31AA%)avtC*@$*jPodhujOByR;Ej^;=Y94Jz#S}849l|&XJGi2`MTL58B zeg)N&%yJVzbP;+XPRNqWzeyGn?)Y$jr{HGZPF9kxx|c0%q?1E0Q1 zlH%Kshea451EtMWTSF_jEYwf!rB28#Fwk{H4aXrQ4}CH5e-h~jDY6$BQT@dEdlAc0 z)TtpE2Z!(7BD(j)0ma6!_k!~6faD@hYw_t zsDz&W(mW4D2x_%V#sU2N0Ra)1y>SQw z@NO_%SiC@>PD3EPusg7{JG9bv@(sVbd`<17=nN0NGe;13MT`FB1#%JR0wR*E0BA>7 z_$d}Ja@q)<+xt<8Y=Tq%htNS=96Cuax{^dlD!+oM*nCIm2HF_oCTToI8)eKc)6yha zxX(ewL7*aatil78uWVW3kYXM+HC zvH`PIDP-$GUaz10Tz3!*noxLnV$QHagK0ATwG{Zkfr7MWdweNvenkEC-@aEh%=DQ*-CW1@A#v(5k8aFP%QWstf6}p1_s727)z&6ut z9n<0Rrn}^CY9rsdL73lQziIZ{7_+uMie`R0mg@PDbDh2M{{G_!P5|ULJ2REWL~+LW z5w0`O&n6>c!3jRQGR)pGqa2wdP>lpJ`u$&KfLFV}%)VEmeEtq_DV;|M>lelzU%2J4 zmMk-rrs>3?bJY+=N289uE_T&e!7(|xn{a1VHnpr89QDX4xaQq5W=BIG`sqPi7 zoEgCknMRI9iMnbC;$3KN3wvE925wBoa_Yg6z4t*3eE0n4r z`$@|+Z_=8r>~as_WC4mUBuxkp>_mO>-WHI%rcIz7CE#tSlUucBIPzMald|*mmo5K| zl4<-H?851~?D$I$Mk+WG%eRc8xLL@Lxair*!2T6GhN7S35#EQ88-v73F<# zva+c+JJeUAOuK98H)LsxLK=!`uayw;8aYc%r!do0y&KZJ8$e2GPCA^!!u<-(S)(^u zc2((ITDJQnTpNr?%!mK(q8+ESk+wj-VgnS(lLM%r3VI~T^s|V-qTYr^(8$1sq@h67 zC(O^&!B>mv)OAp(Ik~clD^QIyKQS{EM@gk12=}HC7$I=mZ=Nt~yDOE7g(#CUZMyDk zw9K4#mpM>f1A6>mjNw~22NRZN!|J(oV;jfnGrkAQ#HyASwc-zPf;U~r%tY*kiM)`X zrUH;WQXnCoAvd@-DdkcpBz25^*PNo;3h5D26DRa6?8j(RiVd{6?24iYYJz9rLW=b; zr5S{Q%x2R`?yN0mZKVdqMGvpwn#LzAjWcTH`_$5A+usx%M0=%)%xx8Cb$^_42Hqs8 z+} zpK~MzI+!P-?dw(9Y~vRdN>*quR+D$V>4#HWMxB+4=c}h>TrMEj`H>dwGFYi>d_O6! zWu^8h*=l}Y0;C)MxR3$#k1s=I$#ALJmO_UqL6q85^I>s2Y_fYjjFI#1`$`cBo&d~U z)TD6s_}UX^3kuXmP(Ku=FR;ezsmx~Oz8#m5vMgb4fGuu)Otq6RdtTzgku|1pn>Rt^ z@_~XNYEW+NIBauP(8XGRujM;L%QvA8Hyh6$ z(6blRZ{#F1WaPw78kIZLmhYM>U`e~X(!_@r7vUgOYbc212|aE4QRE1sICD+>V2bqS zk*`h{(&(dk<#(OCai{MjLOePOxXld}Qdz2&UY zPm%7oUu#M{Q&OnrgLeaLaxf|U&?aImMjl9;QQUs^jm)UKd@&euBjd|kx zK^Mw#OUFTCK_CJ1i!|yR29WruX96#~l!=*DCp&Knu_nAp`0qy2OgC#08y%$eMEf5; zL@+Q1XG>;aWFZB>+E{-m@m)4EgWKD?*?g@*(Aaht@;sHbx$aO+2nNmdNJ;H9+T1otW>o{>wW+i?e5?9y`zKDHxeqXzrHw zz1-!Q9qD)3uF|h3B%zgpeFP$}C)EKHP#pNNFJa%D@5Kn~65&b+rdIS5rPK)S*wSE; zX{YDgo>~v*(i4_MZk$2k3#HK%ZY5rUh2*n2rduLJPt0<$@`$T%e~x(!m8o0ucZ?p) z`ZT9XyjmUrRXJ5YRCe(}ZdGQatWxxKvabL$aP@mR70_)_x}K`+ES+ABBcpPRgXo8O zdz>z>g39!4@u!i`-lDQ@ixnyOpa>oP5ogL9yNrW0cy9^1CD|0C$|WXt4VQMN#sfso z%<8dh->Ovnp6rS-ea`H%=Bl3)z9o+kJq+qSIMxjSZ7Fu>33S3k=`&BL@hS_v+A-B0 z1~qram3j@$@;M!UUOlg4LW48&5~kyEx*iT?jNEXJvj*Mer$d3JR!tyPpbAN3dcFCs zv4H(B^uDGCrP!Rb8X0!fH|FT86ZqkvNGo}uThQ1PGMQ^+h5`jmUE9L1 zX&iX~k8pz^`W9Y2&I**N6Z>*9ES0>%vdf5zQeCqs%ewh9gs6%*4e{~v!xw|9E!{#! zH#<0XiG_(8 zzk*a$^wvrQW$=pT%L#%EipOl=o1}swuW%Xl6WiK*-|>(o*T33<|Im^!y;fEs^hUG^RJax$Ofz#?KjU%68jXl1ko z%tv-Z7f938ODE{5j=S75L^o&30_T4-iGZg#t19Lw~MI@E^ zhPF31R#dgf#X?!8+Tz?QUL9ixAeI*l{+JoVaj1;*lagZAws2vL!0llgi4M+m zOWp+2)tvs?nauvf8FmrSO+!2@%^H;J8*p+e`yn)I#6V0R`UV!>W^Ed?PMfn$x$evA z#Q@*n7$mJ&gd+rvAr78&x;14mR9k4RJ)G{;s||rYVn-z2R#MKGQI3#eb6H^!03HV! z9c>0Gn~GgDjGfIA8xcm8<6_S2YZRhxGLJB^?XlN~d8Oi-uhHOk)M&f?5f&oHt5#J@ zF_bRDxf(a0JcsyQ#eHY$TeKB!m%RJN5UeP|wKIjt z{pxX;%YNUr+g~;&kqO1QCfRuv6Nc-EAUhAm7#%iX5sd|A&w2@=YFOCKcj*eH*HsM^ z!`#-1zFo{PyY6F;%&JfwXsPa~Rv?-V5hY-3(p+@6I}4iW41|GVqEXA$0bJN8>JpZ! z81a{=WmLfE=}vn3u4iYkLodjMNd~GK&hd6wAUjTLWLqMJyC~}3hQmbuH|7Z`hcl~l~^0MvQ~c1HG64ocU<(ua#90lRy4|&8^lPs0hY3c z`GgQx1iJ7|)~*|M0t2zt>Z(oG-G0Wyu#4nY(vISh1i3>tai%nSSR%^k{SxG%{z^9S zZc;6}LI-^36nzEHZ_7&!x60k zM;XL<#A6YB1mH-&3iNG9gYQruGwg}x;&CNz_dX99j}Fic4Y!%*fY_+tgfyY7DN*aI z1aU3k*1Cc{K38^{TbpJX3YdEPN;=UP5+aUF8rvv2(O<7etO0aOzCG1D{~!;0{Zt}$ z*r67P7LmA}}hE6oZPG_l(_k>-ua!?i~Ko@U)$DV?$ z(pPFC%Md5oFF3?FCrySMr94*?XB=`gD8@L%NF-2?9j2XqTk+7)Ud>+F&1QT#uyWue z{y;B5m1=9~`wMVXGjZ0R>@!rlSnhnD+Mp_)n?QZKNNO5M`Itl+8RjcAL!desX*fR+ z>G9~%Y`nYz$U%e&QM!@LV;5i`x;G=S7qS%IgvK&D_F^NfoAJF*%S+#;{0R>e(C1yK zeJn&)q-e%^p$W7|&_U&9Yjgw!L4jGZy7=|Ac5w+7$pW0BYUcZfzRl1Tjqw%~-+yw` z{tl&)UwJ;#A)M9L;D>8IK8w;=K$2Ca*MWt+9AvIYbNIsv?}&0nOfZcv=vLB2tN*#C z&wB!+-am$s){R~l!5zw{xgyPxA<;J(E2T9#m|)50eecScCs!gt01kZ-#d+4qzI%A1 zr9?c()DY0=qqiBkRG8$wg6mkQx4M4hY|!}Ec)LFbX@;S5>zn80!xrHJ$TGcO zQ(G&A8n)!?a`BXnT5JmHDjdggxX3Rfk?G}60|2j9#jN-;#KwUV862je^UN&t!s0=Syi2SeEQzieIS78b!#x&UI{r>@{(a0niOBg0A=p z-q!Kc(nc>Gp?53J?Az!Fl*0aQ71ZzgIKc2zs2%sl5wU|r$Y>1&zZf)!BLM?uT|CxCCcJw}NOOnEx25)uz!tKyi1UI>G8`3&6#)4c@5li{CU?-2yBOa)T@85Gq`%1 zI2{t@XiqT?F^kw^*Zx84w#^TPHn}SF0?W=&=a;Be*UhyO+^vp0Jp=f@EtymR{v}|T zxMpXfKK$+R*9f-&R?+Mxx7BB%AdJBU-HY-`_Or0gLx%mFNy;F* zc&xb<$;^A@_JBUD^~*RnKld=v^%Vg@GT5RTy?S+ZI6xa;E^{>6D6DGiS%}xew$VMo z6nRUab<~h+rNsFyUoj@Z&9ka2FmG^CkcR9hhw8Pczi{af8bzN#28zm`w2hG6LX)Ef z9mBLbLY7i;{GErRHA@V#Qw5!s4J4G9>bly?(asY3+tKh#N>x8$Uc1USWBUOC!q!Y0 zeF|UNE9dlpTFGVlnr|PE=IP#5-LY2AGRWMdigVF*mOSlpFZkd0gcwShl4@!iyHOUx z=jG5&mlLOKH$7}E%$XHyR~;lGqROaM`W^)2jnU;LFwW|b>79f&e!_l+Y~F{s&&SjH zOuk@@VQR-tJd7<6i*UQL8f667C+n0Of2>2wD}o-Gqa!yS_xv_SYnArdRhpLXP~J9; zD#RSaU7EUZK9s2q0%t-YV$5*xQ03kK1OFlK{pdui`t~6Afw(>`pqKLwuLdZ*@68)V z;NqN~O7}oAWOa5pfZ`~;(ZPZT%E=%Uj8R$6mYhQLNoiko#rMbxg1H5tx<;C^`hSWO z_|i0``_pAdz*0R*i}sY%+4tn;*?O&Pb%K}V>_F#YfAnhuZH6D=<#2q46UG6#Ku{hF zciH&M6D{Y|km;CfOS%8>+d7C7&Jz!o1+&%a@68;T^Mvdn z^fhE=2@(7>dx|({TC7~T^%4?RYSZ3_sa1o^Kc!6>|#& z7iRF>!qj$oro!;aVBw{D?JiBuY2P)x}(kawXsDBETn`TOf$D@k0kCQyut zomx^o1$r`W+P;GUu(>o#1KLocPon)9O0QEFz3CV|7SUe`!$I{bPRh4AosZIjc1Rml zwp=?Evmskd6BV69PuB3`Ok#s!ETQcR*0Z_~ac^?K4pR2BBX;QJz(L?)Vc*2aK! zOOip}%}}~U`g&&aoun=p-CbElPY5sYE|lXVy621-ShoQ{l`0qgr7s;m!zEQkVo{Ff zfkv;UA1V^3;c)n8Yr@I-l+QZVowMPb*d0y3#;`(ds^kMx^nDXh+h`0zPu$*gC7 zag;<@wRJlN+a+@ccdqSAf5VI4ndIsG9;uFKI6kU+l04!G$Fd-YN7-0MVw0XHdSn@L z+62UN=5{#%_z!Q94zD@j8U2nqz5&l28Oi3IH0BMf>8GG7^4r?z*mc`PSRl*XQXyw%KCkZ`>Sn-tx< zN=#}}jC)LU13fz-p1(sQaV|gEMW4W8ql=^Ln~@!<$bY+WmA`W08_ODkCp4cjrDS0D z2nihl92ofFx?rxKonbVn<@yb8kNBGlXX3FD5C}w%8wJ)n9#%F8*~->tv20ajed#K@ z5H*Azu>KJS?}p=`QnV@Y64x{E)aD2_P>IPi>o2^$m+PQ5=K;&ybIACnG}!17Y1uP? zzTGT$%lz&48)mLX9saA-yjO~0vEPbVeM-z%Ksdvsu59-s1Al75kJNkqu|hyz*WDA8 zXWZv+VHUg)Q_c`M%@1;CCOd4uWqo&|0WLR!gN4hy1CD8>ZQY|zY%V?UQKeo~vzIW+w&^oti?NK|mC4)+*} z1-zA8YcAb^hv3sOtjhK9QE8Nl44R%ER!D!I@zLZ=w(;r-lMuWi9o{3~rXp!n{w*l@ z2xpV8G#4hk=UihDE6h3}WElq#J@}R6<-_9!Z#rkPMQ2zOS9d49MHjBxGERD|*_L8i zamVNtZZ-0~g(;4_+#*0xD5k=w*9TKV6TrbvQHK3=d;NrRY*AxTi>%nj^Eg-4(wF^> ze{I6{c-rh;=`{_(Me$*vKj&AEfk&#z#pLKq-Xex|WD^7wJikwU4o;vYi9nl@$rqES zkM07{E;o7=0aACZluRSqWZAE?WWQil5NxE-luRnLM=bSm@*Hs*NwA z+rOrl@S_if#z67qVCW!P2rQs*6CS|YgTNBYUvgvgZZP_c(jBO-V5*mTh?@jDrh|>iq80a?xhaj7J4QUGy_FYro!6gjY-~?Kv(eHycAO>; z5uM6_|JHdh2`4G)+mSr4wXoIb@q?K8JfJI`)^m0F`+_E?++wq5eSUs5-(rd{?dAaS z1Nozel5&sf@G?%WX%0Y!25U$2mI!*l&}$XDa8zTJORi)a{rYF|@6U7JV(az4^iRS* zcP(5sv8y{KYDm#U#9&_f&S}o7fGC4lfeLu3AGy_dQtRQrsZK}v`h*_qWRY()|GaOp zS7A=Btl~7ZnUizK#CB^zSK=3x3~IGwXi;5XL#wqac4E)s#HIl3eD*M1mRG``kZj(} zu-hJnN6$rRhV9&Wk-wguu_vp<+?>l=X;pjs|l-sYw2mz0@dS zitwyc(`SC{MRX|_B7AP*^ZV|%aQ{-alP+KFR|^QQ{Dqa#7*M)&j49sSgB|D1OXvbV z@;*G=*2mLULv1raw9U!LdnOFwFZ}IzNPOhofFKyI+&G=oZZg>A8*pFc0X0gqcj zq@58I%)zT;R+lD0A&ys6VbxpH=6;K1jn_{DO1JFLFk81gnCEm9rc2BuQd)0e=2!XjhIURg05e!)aM450JCpK=%>p3 zV}jY#ok`l+yK4~D%`s1prXbOK>qiYexV?M%>`Y#XOqb#~caE zp7q21K9BOY4RmyddkXu(OvtPEywU^xGKEFm(;wJ9qu!-jUl1)al4GG@zw&Z}SKsW2t4oxLwO^B!79JY7w#ERG zC9_wU_NmKnZ@05gIzm^o^Rrw9E(V&|YSFAej_C5|cbQ6S$>Or~U?3^hJ^f+RMksP%I>pRUCyM>@FRb&uJ*2@Hr0ce4{B z3T;mBg-CL2GiBeA_^3obD*aR--{f73nucq?QI!8sdz|QQ>fJw8O(lY> zwi(_=kTsua38P;n{Z-Rl%M`chE(VdQHG-GB+NV-G6MHBfYgL<0CEVe&uLl zGsBDw5A31K!>CHBv!0+MeP^^A2WZVMJNHhg9Tk2gVbemelL^nM^m z15IVRGWw$VzB`%WyA09PJz@WE6ZyS>_zoiIjdOZd$4G>op!GB5Sr?bWu^rrHR zTSWY-i9=YVvY%R(U*^aq$CqF>LfZzom9uBzRu@U_(nw(1Kq~9cgBx@IPIwALmk~$t z#&XT6GjJqqW~8+_`J^~NGmMFt<7&XAV9x?%xP6*`fq$&0AqSfRXxUvIF>Xva)G_8- zF1v5{5CR)O;!xory{E7F!t&HdzjO!gKW!a|mdaJci!97E(=FZL{=mnkD46?Aa7~+> z@>9W(6lFsxKI|%n-W}pP)w*PNT!n$#C^SF0DsH}4_XCzcu={JjLFH1Hoqhp;Mot(P zNxohBnu$+hHkRiSFy^ewtZas?vJSqsCzQNLN@nvC;V*H~nCF+M`O7GX?z8%W6hGtW z2<83D6J#sl=gSc7g&~KC>%vRa$foS}_=A}}oI&ISvI3HR$7dQsK@qATquGaYwlj|l z%D8+(x}PT$ucBwly~=8jr*$tM1zA7wm1L40L{=OWEXY_40Vvk~pq2m_Ni*7?BY1g)0sQSMhL0Bi*)Ztf+mw=kwG8R z;;K}`_l5fVfc;MxBl^Ha9HX22KM}xlK=zs12mw>pe&p~x6{~v+B)etIS<(kDRBuF*$48-lv2m=4J;e!Kjhp~W4;P^n= zNmN2Bm3O~F(Ajy=PW}%HbY%4b1SA3?4&xFoPq&K@fU2xKC|ZW&FOVF}ACSQ?Edj=v z0%tKuInRLz42<+IP%E%tn1j$OP`z}v z6&V_kXB3t2c@Ouf5(L-)?PPxeqnZ9(QkfA27y$x0itPS01-+}lq5d1_#{37U$O!|S z9mOTYGSFqD0EsEs(u0A0{L3OP_x}ST0o{jD3IEnM{a>WF+#g__G~%DT{abDFe}Pnr ze}Mm|J&6lk8$%`hTf*>v!EDO^4~zk197iSmuSgTfL0JA52Nt9C2WUUeNbrx&$-f_T z1`ut62xzQ>@#m_4t4RE>wPd3|#;7NRAug|0Tf%{eArZYY7b676ifr8d*{N0se=(QVLX?vdE0t4*{f$Njh zg#Y1X!~=mWK_E~N!~bmZ&L9x|A4~tx6aHVA!#}7h7vQb~4NznemGCc8s(%omqdPC4 zQy@9;d?}spzs_j;Xeu8U||1gLH^l-BmP|VbcTWOzbR~> znkeuO0(7(&4`iKHB>X2o=-*a$lYp#gw4jV3Y@pBrD&c=qFF=;K|FU$H@(0K~_mS|w zLF|9DRR0(Ckp2f`kb?Hd+J6J3L9N=~t;3H@;M|-9;eW$VL4d#S?m7o#i5X}z|B>*o zxWE6s8rXLJpGVL;PfGaTfG<#m7yb*xR`dt-ae;~O-@!5e;A~4lATnUg0yg14Qyl+x zt>IQ+_W}>$e+NULLNokh0(7*{3uI~{1JZOM{_($mcOL&>9|wT8i}Hm3={)|^yS##2 yo%f%!preT);P*a$kgL-{*tTt(6Wg|JJI^E&+qUhzaWe75oY=Ol*uU`?dzm35)V;HK92{ntF_0}6PnUvE` za26RMVH(#SqQp&4+?z;3AWpN;@7L(gdQu@3O^lnWO-u zQ);l!gKSqPvoLp zbTKh!Nko-atLXeFpXe#k#Ryp9um@RCd$g*{X4=M~vR>HYOe*Rd$^30>cTFtvlm_=b z_vfd$%DPX_=rK8hK==!*%~0Q5lj{ulQ1Gr~v|lPP`+QDW&d5)?Ox)|~fNbo_RFVo* zXz#KTwDEf&Mi{!e-@ZYZ;>(F&f zrfEhRo?sP7b`tb=!vwx>vVv0m1{=u*^fUtr#&jz~&~OaB1mC{3V5Ybj^?wKKQCWN? zUD|A2dct@D`#bKnG3@unqVme~j5#fo`lVj9f6Oqwj=vdHg@hXjuR|1}>OJm05H{f!oiv~ZBR zNJG>#?k(&=gLZ#j1CA=)3jaRxyO;sRX724{A+>?XwLA#{N(6v5{z7%S@aHNU;Zx~I~eh2W0h{58%6G`ppynHIwLy~bb zJQq>4--NPaIri~nk^2vNk`#8-C;v1=ryTNz>?k~{(%{pd`EIvJqNGz6%tL!gT2nN~ z%zi#iD6wW_B3uI1rU2w_X{D80uiD;E?-Q#SdBg~6Zzwt(IKqmZ9dd|)FC*fp0p@>S z_ZZ=9g(l3%N-U}p2Q#??2uXT*KViOQQAv-U`bUwtRJa{*1rz0vvIM%LJ}VS?6r^)( z59oleD5VNZQ74ACB_~ZNlD+WrT))VCK`YCBhrj>>gMmT%&m~2`tb@%hi~oE5O$MMi z0bP1e=3w~cu)Ka^XHq5s5qmLvSQ(kBo@{vJox&jumvez5L83|j`saOE60w2bIHsch zAnx>ufJ#b}&eqyFO^kEd4M-CZdk3hOV~(Xb@zU>91&@Pdgr5@IYb zA7P{FVEV=p3R|;!m0*%^7cK8UkG*cahkPT2dkv-`3i~%8!|yoH?OM>w^69wVNf<2U zc10467az$LYYDC0#*7dj)}E@tenXE7C~!dz zs%|0=pKt93DLcK0{6KAE!WUTCALo8|VJTY=Om85hjEDN0=NPU0j#yV}>TDQVsNDL~ zt1FnNv{Rk2zSNr?#n9+hQHtg|^ddayU}~xvk`7^lvCxaAhgri7{NCbELn8Cy_C4|j zeCU&9G9A_x+fNfg;Wn=hQ>b_XAQi{;7zOvjU{wwAcAk5`T(PhyagQWaqRJ(XA#515 zlxsQ~^8uny3H3vYx;Y{E2VUL*MXN)@-yRQLHYf%kj}!IJD+3`@xT@^u-wt&4lJLlH z$a-5>1=Yi(V)Gl(Nkfa@!)pBlf{Rj#L3E+bq+!(T5&tr>^5PO<80{SDfGSxRG}H;~ zCp&~yT*fE6BVVnfm`KhyJAw@6sE7^;Uk66#TM&WNRIqF`vBT1Q`u6+q_tO zhjPU6#_TiQ(Yp0n{m3eL^;rD~3tGR;3fo@1(SypmS%(^UA^hGKy`4PK$DA!Zdkgs; zh!9SiBAz&khUyIwX1ytYVPF05L6iv488yOrk^4vut3P?M`>uD(E%FDz`vwpKc01QM z&#f=e-HtoNw;F`(u1EO?ibl!oMhiDHs+mCIRNwf~w)y>a(<}0-zX8`jTFf_Q!ha4s z5MNTG-0mOsVNQ4};py)F(MJ9IR3QmHbZgS;_&j9boiKQ06dVR8Srv(DVNA9jCbr(` zRA&e(%y{l|e+-846hb)XN`UCrPU|J{Xr?7x9PD!s)VX1(j%!IWLw*0c?hxeizGvkBhhtB$KvRc};FZg4z z*(h>6;*crw6EQ9^FBJ(CFlb)%+H`?3=9Ha(wLi zTUg(S6CIHRp3VCLS%sRh~LfN9u1}#0OiH{8Lv=Y?EDuqIi4QFa)QCF4a z_LR%^39Dn4YwqcR{7CbJO2MSzCSBx%xM9risoIVJg7s@6V*JVLEcY&CN;S}$N;4X4 zB^+x@PKvV3ijDB;!E>F_v|Ko{epb`*Az3CqJR3CO_7efU1Yp5tSY^LV+Vt0I9<&cC%WU(eci)kwOS8vZL%?ye%IxA4@SjmL?@tb%a#GWP|VKp$#P5k@KI+@~B_otHK}X0E6DS<0TB+M9f*cY2#bkftD6r zQWh)$dHja%f<%{}^}#gQyb#_e)fW_x5rsZ-esllvy$}A}#cQvJ-4*PFCY6vAqV`IO*ZMBy4>LK=tSwGyC;AtO5b&Fe@v?tUAKxzC;DA zI2=}lVaxm++6Yr6_UgfExB@F#oSo?+2K7(|m1!9d?T7*^X_&l=EZ!zVXeRAe2(|)h z`jpL_T@-thSD*cK({3+v#tZI8%@5@_I)^CeVKMM@rN_8fggAn@xN_L)lf-0dsbV@2 zKs?DGze->-6*ETk90;(fVvFF0q3%GM8?WRWvoMY~X8e>9%PN0VQXrLs{7O z-`2`e*3k_2gP$`}Rq2W*MJBuhFYsq~+8RGQf9gfCjmtWbJG?_*+R!KO^)xmf5a~*m zKe5Q&lkY3pn)sBA#p1)KPpigaX7T4s``T!%=2GVB6H{A26II z9b2W$;V1$!v5kfDLrrK^rFU?sGG;NW$X41sOsmfzI0>1>cd``~e-J#`3axYCxdOBE#&Umd{8FV^R{JgA- zdRatKpaNzYtt47sA$!PDTU$Az9SU8yzv&B_@Yj;LVJG%uly`L{;tN)C0_S5wq ztLP;hw_HplOL7^Em1G{!T3IvB=c|6!4MH37UW_)Jr>^46@^@B7)4b0ed#y~#PUA!+ z#L3QTwJ@Ap5#B5N)OSjOW!`QsG4gm)G`G)mv)CcJtI6cmD4psfHLlIU{rJ~7#vWpA zOP|VgZ`^y6O1kzMMtAcuMHj8w=iNBwRwbvR>nr8+*~N`bMA-^7AlzV@53_Z2?15i0 z5oTjIfAZ7lp)dq!(0n4R@9;JUm%FE9#z#JGJH|dK@L_MWWw7_tC>b0grVG{ZJ|6*m@UFoHchKkoCVzp+brB4cVa*s&%Mfq1F4vBM_J zjYeUva3E3%(XcO`rTy|-v712M^Scc=^J>SVl&;9b&?xm))ow+*wKg0=*H7%*aB$QU z*U)>{P;c`vlqg_CYmYM48`TJDFJK(oM2NlAlwBYB1Qi2BZKjNlVi2_}A%wE&1|5Xa zo%T#;O*(|lvX26#(HAnc;5mnKwDo^UODP+;YK+9D5EqTYQL&>ddZ657(tU@wMeyB= zU^it!nfntHSl$Ruq#n~5f1`Q}$)h_N&XihPy%>rL{k!0{Ec3iqO zBCUU0bjJcYPFBLdGg+VVX<(}HsId-|ygh0Te1k8SXETPg@@73k_Lt`{%6D3$t%)K- znygP@TUw>SqqU{v>aXX>f}AOeEE-AC{+OC6IV;c`&ilFZcG|S5G?#-cl?*6w=9*V9 zwhh!C8II~q+_1O-lqqI1yse~R#^pv%qAgIwB{R^r$hoW!dRlBxMehu1_-NtAPuDvW3I(C?k^6eMDPUdB0Oz?;;ZNr=5Kceh zA(~Ilx`RN|44wI-OL<}A=UMvk9Nizm18a8Pxm13HMjG2Wr$B}^_|;KGIL7n*$9qKd zCor@e@NoT&uN2#FZWMTDjrbOWM#@Ss0ujgZ0)=ZW6PeI0!LJU<>H^~w*4EozwIZ?-Zzs0)hmFTvvQQ)8#?eO zQqOnd5&lSxSTyE+=vyYyLsF=hF5ErN6yS;GFwe?-A05$~Y zU+KV^L#}PgYZ!d+(&MgSw*3g-_o7#bdzquZJ51{Kd|1v&iKmyNhWoa2tOzI^ znh+}Bm*PLON-&hRd=9Uy8^!c|dZ&d|l09#}RLGJ;*0#B(DQ)TFp}x~l4y982e0?z> zA$f&BH}PNy>3`x9jF~!@ko&900r8MLaa!CaG$X5dKgvn^ow3LT`=v?H4$hprI|cm( zqE*mhx`WZR{@^9oP)}%+e^ZI*1`oa&`@A!F=u2ygZ(b?~u zd8R!}_Msdt{hcQLa)N+@M(XH=T;6yiQlV)ElR60lc%?b=>tecVtE4ZmU+<@AM%N5< z&1!{-&+KcrZf9?7xT0DqKz;QAvx-k^?;klKlIJl7gNvzbD?(zed8MMn zhpnHiTmCbsAmU@f7g&!Ja^BE8DcD_%=J=LsnRq~zka9Ajrz#GOzhA`s?8Hy-*Pf5D2Uyfoyhf2JD*jS{ZGc+z9ar9xD|)owS-^ z;ZT;hqdO{Qq~#d$b9yuw!WQ!I+0X@{2RzT6!2b`B1V*r+8bX1A9m0cwvHuSs>Bc1m zH1*ujk8uNqbmrI2Ik}K{P)9ju#o1PZAAZnvQAqZylejgx?1zXh&4>4Rw^r9_{*+Ua zky|Tq@ReN)b6v$gQ&Myk}d>`Z6b3yI;qiKDXVnC9g?(&llm$F(_C8j$cOiqe_At+@Qau;|FD`!6 z7Z5)`!))sFhOTi(f6C!v=?^Jmtr9Kd5j*fAl7`ANUd4XI65y}vqKWH}=jBes4>AK9 z^D>A%pL+S>tD?(swa4+82kKf}DLTM#%(aYRTl4+ML?ENthEnh0WA&oODU~p@=Sc5w z)kE`WU3!l1R4?7Ay;NM9Y?CH{-==S;`38~IGVncU&K)EC(js-kxuqH!l+5VkSb6n6 zkJ#ih;gT}CD=)WLZdLb3wIF8fTw~qHID#5x>{j+3oS#f2-BZpj?Oe7Ti_aM2o=&9+ zlMuZbEi`JfYHsrC-!G{GgV*R7dqZ6zr+4uqT4}-F!?Ye*41faUu(}L%Xn1jybIvh! z8KR2_tto??EM%~%X%pHw)7Ll3{b%@FW;c2f2r(uIV>oS(?50 zC7N)c8*Valo_D;@UEl+%<4|GFgTDg_SXd&ygG<_3L-mF_!3*@hg%Z}TE9$&K_5QlX zdm%C;RSL_Q?+()X+mF)gv;*C4v%|^xfx@kG5zaOJ*Wi&LJ({ zbhB+GMb;rDcxJlbwt5R2GkUtsj#OgtGGfkenILK+3Cf@hVCpx3|I!$ikMlyi#i2aZ z9+r>$!W&3B%Zu9zwa{f>r5!kCWsfFqqnI{jmqy64HT!-i$6Cy72W1FDWo7%48q{_J z3NylgL3Ff!sSWaSRvl#g-ZzM-d1ms!Xo4JA=Gf2Ok`+bztBlMr7nU6>bR&7XCU6u_ zLQx0P&{4)}$u1L+IoxJM*YLf3)QzQrTpUF1K9M%Q?qoI#B`s3WJknW?_+kz-n5cu? z@LX$63!78;UChooQwe<+gE=FG{dct*r&CiWtTn7#m;&{#&#J4P6 z$T^U(IFFED;S;1^LA1c%6 zDBi9bXQ~eXs76H?n?vK$1@I^{3}iw*eL+KHNef;DE~{HVr0S;&gQode6cO*Zu$6}K zPdY=zY_}ee`2KDLz<)-cWICQbIhLW7`lY)QcQL!dCd#p;Fn%W{@m%1miPV>>+Gobm zunJhyYjg2N)hmbgd%xPb)hA1}-ig#_OSLD#gcAZ@{OEkJ*HvMWIOaU>nLvH-JLnds z4Syez-qe}!R#v;B&p+vn3EIOq!FdITzq4~ls~0MO`=5}M{-a&)=#@YI%$Xyz-mv=m z7pWw{?Y6?mRgG(bik|7*XEZh0QnhSXgDD*fAsY=V*mbx6Jj-ZZ#kKtJRMg1$LF34|%`i5z5_}Pb`+!%XP^5`uA z#z>QZ`Syd%iRoswd&9IBzuZ|@Vn&=6#p>Vx%NYKL)>%XZa7m{!RdP;aA z7+;c!21}2sH8iatTRjN9RozRCg`tV6NOc$zSS9mN*gB>za~=B+tsV+n+xbzU2Psa$ zK~?%czq-acro4LHjwUCsg@psbwFjz#RVK&0i*{)9WFlx`j149AUm{T?8%TCLf)l`c zP<&M+Rjm~UrLS2$T&$NL<`5_TU zd)N7*_qdBehPAv5YV^$oYB3!p^naUAnzCdqY4 zIB5@%4YQuAV`12eY!U=>`H{CDj&n8Yw>ZME`KIDa_!$vplY}{pV>6JvDFtBcbAOPK zX7Y$7C1cDIKV;Fnyduqnm=$12Be=bUwsDq zhW<#b_zzhBN>AVu8+d`z$yg306GtG^jPOK=mzgFB>o^}k!=j;jqFgxZt4pWV^Hom{Pw zr4&(;0df)`A>i)S{+B%q8G;BHvwdh{$V$j!BO;DDzP*{Lv?$VRg9JrxqGF7t2L+)_ zHprc7lU;^dMsGDXIbpixRPlL>e$y}hUl;#GllxEL6H<8deXDKx-7xQCW-$KS3JF)` zN?4bM{gB+jooYw7e7_AQC7OP4I=y!&as9)-hU+t6i&@_<7`SmhYKPgaD3Ll;HE@2o zi?7~|>ZE#OiOGrd!bk_&uT45$Ke+Sk=oT9eX#X#?hTA$F?V>o@q)Xcugxjh%#oQ`4 z#hs?{RvyM1I}{Qq+4&6g{~aPwAkKVXo=Kn^A6RO-%qXX^8VCIileZ16^yyg(+Yu!M;zDoGKqzHT@nLnfu8>Pkc z?u}|;`YWutw(0j?jS}8H;|Kby4Y%DmKyKgecP0I2uD$WZe-6Xx0z-(aUtj~PcC6^2 zZry^`2tO}#6YIp@p*r7?FE7qd`4-!(8oGdbEfN&oD&plq7eQV;q6XE*1MBP`65*Nh zdcN~`^ZU4BB#)~ZPcyMKc<=|nV z6iXS&8>y-3;zdR|aD8Zw+A+;O1jKD2>Q6T8E;Yig{HD?hn9EK`Cx0G=vLhHC**{~L|3t#u;Wh%9q2$RAvx0bi3*hP3JtiA!zij<6> z=F>Q*vRab~iL&eZwq+I=DSIx^{gu#FNo@0$RG6G=3q!0AA-{`q95=^qV-A1HR1r=O z?U8>7$f1v?~ybLvN*bHwavBDU>V zB1*Zicg=;u$qX{za2RNuC`tlh06f9`5dv%WXa)veV5AZ4OKaHnPTLSKm9i@XI{Qhp z!apfk+ZV#X`d#Fg;$UH$7xu4#k}TSaTwmWBDdOcDtq+C4?OS9-n~lm{jJ(SJ0RyO48KNV$$6c-#_prDUT86nd3vyhdYZ*p!fK*Twvb34A zbmD^T2eaBC+UQMx)x_(ahPZK|%k-!iBHyS)UP)rm!z0GT@ZL_3Dw=KFq_H4Kb z@E}L`o(lQqfq=zhHfG|5h0Dr0`2lHNY*oflt!W4;)8b4FcsmD`bw(~pLm$egMOH04 zfzew)NIzb2iyF_>NASncb`HS}T39FpX_wzmq^#)rQEvGG7g+X9U#QkvRWRg=-z&%|n21lRI4fldxdH zeT%WW9H?fTS}8SMs&pnr8m{9J4%>%*9d`|57MhJ}pUz3zPIAj_Oixs!;QoG0Wq#}$ z+oCW7ah1n7#^!U1WisyLBh%XqHL7|9`#R9vlRQ(h4!i!teX?PjXrwWQsJsvBFqvP= zT5s?hS;GN3-@bRK6bq8?tT&`Gq7#4hCYaxdbem(5R?j9{$f8^6kZ^|$r6PE-hl=7#w*y$&`mTmqS@cH zq@7D4uJc7PX|Y?;oO*GNNI*U*~O(lPcAGnps zXt?F{CYF@;iZnSZxD-uRog3`oCOV)rUvxhrzySzw$ zdoXTx$E7XJ86Z|jkf#;HW;YV?&jt6muW_#Y*tQQAIg!y1TW^5$D!^nO(99W0id#`#DAWyUunbmW#9j=BWj^4LtEp6qfPIXPw&A8^L^c|06ooQ2_ke?+&= zfY{VK56-&#!H!f}V)}!+YEB~Ez72@gy=WLvPzQfnR#~Uf=nXLlO7Ha1Q1n!o?qi2s z{hOO^a0oK`fz@`P-O}aV3L4OYSvt9MV|{B+M<^wZ+x zvTUS{p3>3J8n)=V7LzcC(+kU{avs(hw?=HSXnZteNc{afwO6Xq!8>sOHw93a^_zXV zbG$`q=Zqax1Nks@SVu)q^Bsr8K4Mm6(U@4xN;vrhPxJbl-XMx#=TD$*S7+ldi}%ck z+aDX=EQ1WA-gtxBd`lL&7k=^D?}R$;SC0PRm>^4oIKK7FA$~H2MH?%%0{&WF;!&b3!r^x=M~>MsFZ-kWw0Qf zJQ#rM`bpsh@71x<*1kbs&LUlgnZgnj5Q^)eMwBj_9&J1)Q-l2v`G5De#~0de$+=7- zz-@Om+;2g5ctE?NFFccE-(;@{zD{HeGwW9c8S_x_v2U|1)9C%!N+fkpI`oJugs zr&PyU(*=AegUkF70)3eEgw2l=VAle*(s%A z)T#~RH$^C6oOlB_X3QP3t4}G>?G!Oz?8QfF;Xj;}p*upb^d}tTJT6xr?5}?V-~V^H zxz#u?i~k`>E@Ut;vj0Jn)04)1Upf`L}s}-nZ*}p^WjrKnQv_?FhsvyJBoOZ1Q73f)ZJDFcf^@pGEewDnNJZq zale$ru33|R6FTuIHQ%j_zigNiwX?`jzRX_pi#}R@x^eQ0D7X^kO^bjK`5^Z2mz<2~ zS5uf3r$G5GJ)D6RLf0tZc^?)qKqqLuS~d{1LAxK5*t$_u<3;ZtrPxpRKA2YC(Aj%e zQyJydG2L!8gU?%*PYBqDNLGt8Uh`4cnhvvzD;07d`wrXBvBQBf1}gvj73}sp#54I zv!XN|)zahi!ls?K!5}_gV7##POL3g=?#$#)`+bN8T_jUvL%&z$3)NTAe!aKc==$E> zPy4`4RR-#6UAxMLz%f%a^Fi}G0XVM>9;E&w!3E&vdldbJ?YiU|*G~x}LkdPhW1qQ) zcpPwgiiDBJ3RW}FeB0dFP}^XAE%RnsIRo0?7m>k`>oBrVzxhjFT|2k4mhr*Ic}VA1 zpV3;^2scWXt8BV1&tdL(oG%V^vx9rfOapKFcjv3Lce`u{4p$E=e5=^rVwoA-X9li; zhXLSeW8If+7=A#uImk1+z`EJ~kkr zIl^3ZPDh-*a)rS(Q{5lyc2dK-Ub*=@vUAJ=QOrNXINzIY)OQwyj$gOZMOZyv0-+h+ zTFgdLd)`NId{x?>gk$?!bGUU6bZK7oJIcf3)!m=Uc$KDqC2F?yC7vZ}3V%eh_&53@ zwHY8Wb{YM|>ItznM_!)e-4N=$B2xsIajmBObAqhtbcfeqZ-v6W7S+4d)K9ASkmxjA ztX^Dm+m$5hO#&b)txqoNBKc}~Hw00-4_^gPI!SohSkdtTktC9VN$PgV z@OC5dzuzcdp(d?F(2G=tvTWYkh`R+# zvTs@P_g<)VID&tfBl)Q9tLp-Sky0<3CaQ!KSz-rTBA25ynM5WBZO$3;acm=WzSrfK zW}Jp$*CRN)G?Xf`BVA94g`=WjP)3Oj+-a3X*LP?SaeoUYyLNd{Px_EYf4xF%5qD<~ zL1)0-02;$H953(xACJm~@AyH14hHr_3kF8=ACF4r;-&yF4_k$)pO2=erWznc+iS?t z#)v#QXz-Fq8c9h^SV{IMypm!zD5rDf(9Si9HFYnGJlZh5DoQof(KzTp9?NUGwVVSHuQ@?sA+H1hl(a|wU7JhBUHdRRr;#6h!u7Mu;; zeMeL^rXk8fC>(PVZnPLje!BF(c@pO2V^`E?cU*vQI7P}AzAV!$G<19q6{;uN z5;h9$Q!KzOqlUXT)h)3LkKvdqqbJr9*A`ll$*k_}7oi*C#BPLfKzQfU$dL+nPHKC| z6$-42wZyIm6Tmj19v~&QvB!6N@QdV+@g5=PO?UO-Iz+3HKWZzm{W92 z7)gty4fUekKreF4%-%87nWc_0#M+y=KfLql=r$k~>d@b%&y0|9h#5t!W~!o2mQ146 z-)`viR9hG!e2itp=jNzeYiV>bxQ}~m#RryIFgXm0uIHHVKZ%ZZfd}znmMQx3M||8zg_TZb8MbC#Fr%Z}8(hy0voX<^l zT8w*jE;xB*-fYM#P48t}W6X!IF^aXt`m)=w=k8YaV?D!$duHU)%C$|2`%6@wyR)k> za-nLqSjDra=CROPV|??Qu$LQqrUK~4IkqT?xi8KX@bOqnS^Q+>(tE8>o6^){Y*f|N z>u2!PUu@mEW3McAUG?c_@SEXsFn9Hxba4yAJTwW9^rjqN!x)3mi}lhdF~7h!g8QG; zF^;CtydB4+=}Haf_RraTGgpR;$OTq+6J{bwDJ(k{k|q7d*)Ds}T7Q%M)CQb9@nrNg zJtG$2*#+?SH)-+R5e36k!f?SID&vrXe5XMg5g?F|w^5r%nE;CirEwOQWR=D=-E79H z%}VR-AIm{D{yFa4ZP&S1It)Ti3#@uxH?GNrN?A=(p>dtj53~rLvw??B};H1Hh?c_{;N=GBPXMU zJTfKsThXAe$3WC6=u-YhDsk{fc^Bd<&qT#}ZHW1@1}*!U>Oh`k@~C10su{%evPH0y z1Qf({)CiBZZlxoRu%GN71qeGd)>+)+z(q<^R>3$LwMH_N+9~h9@{X+gE9nKT^(F4~ z81fX^9y*!WtVLl5O2EXRYDNWx+H~MIpIc1(>tNHgZ`Mj%N4PJjq$$}^MwX(tym^f& zml68pj(P#OMm(M)^R@>iP`HT{kW%KZKOv@35=`79BT4zuv zh*{0sl$r2l{Uyvg%1QYO1?M)?Q-5Vt$qKk7x2ZEN88hbpq5_Q68L;OGqo?&6jK8TR zAE$A)^yRwKjh`BjOo`fHAvWvM)O34|l4|C?8PE*aows*b zL@c2dYWT3VfRLCC>7r+ZgNCWIwcLEMk==Bt^fhx@x|)kEA{OtGmo%~3cuqGYy0h}M zC>PXy^9+?wzXNZ6sc$wvO7j<()qS(_R8H~U_*3i3m@6J&1gqwWdZA*Jss>Ud3T`Dj z6-zhWYX`bCdeizU?mW7scA3XU(=~eYoRnYb{l9~b&FDFu3Q^vx#*Oyt#X{{7QKuN~ zZ0S`>X#9)Exo^2q{;r}onPQ5cK!!SLzGC%i9In53rUKVH{E52Vc2J?$s;}7}jH%a0 zJ#^cH&$4d$G+RH@M6ssr1+l{OgJn(hO*<@1v9NTcro2ZEr(r z3qM*s*y~Pjh2)zXR;ehmHEa-1bG4e$b)lQuEgwviVC4qVWDgYI^oG}6&vQw;=6BOb>rK=G}RE8IkY>KygQ_;<;iR51d&=fX#rhMcG!LGB60%sM9M5jkrsEZ?~(+b zr-!=XsS!4vmU@*XD-(jw8z zJ^>)GW#2!A!FUmf-F(%CoQ7FJ{JYh0Y0wA}g|dBDDyI<`_6(D1r6DvhL;Des3Xm8x z;NjcZMCnGGd^Ztm%~=xEpR^%_t(Htac`qE8sr~@JCXQNT)gYMuAbM}%FT0EWNRobR z;;4^lP%ax+d)3(bWh(MP+W4?G+miS3&8%Fnn7A@l5Bv;(`CGW1vs@RZwJ<`Sp`jF1~V3{R`87vJdNGk^s1j z@gR?CvstC(^^^X%32KMbnk&i)Qj(-E_fn5OH7b@Y&$tn8WYR& z*hoKQUmRC0PUGMjcl&}hgekBcLUp6~jHE0rc7iC2%}VhY$y43)salFF$6bg^?)dg9 zzYGgmJmjf}g!>pqKZkN;;fUo4905&MC4UvQd9QsL3d(2LTJxaMp;uSn*VX>KxO8Sm|^v&)Y8~WyCCyWuD!k@y#M;N=VSItXWr<}!| zfKr4{oK5f|N+fR5Q^pa;N)j{2Iw&XG(CxyQ<>2J-1aGl^yOb_mx$gb&js$WK-Fb^k z94|g5KV^p-Jsz`&TF}K@V~BOO#Aw-9QO-dwLJ00WHmkftF@?PjHGTq*v=2%0SCq*| z7A%FxO!#IoFAWxD2!*3;z0)j*`T9InY1VisV+Zw(UW`Tv&1 zvUZy<%;W!~gnGS2uJz^*SwLL4LHz!NOn{FoGx<-?Izib{{q}?GvA4otM6wf9p8Li9 zZLgqV@e}p(QFNCSR~4g51#h!!Ot>rWHf5HSQDaEHvu1-3X+&D z55iHHa>-*yhSSdN`zRXyB!}StHVs4bzQa^933hOq;3BG!hfN~@@7u>1-c(XUG>@PjyX1;+!QkseYfy8K!ghvI8H@WHe4kM)rG&K zo}2hyLR92paTqR5Tg@XL)HE)}@l`Om>)WzG`_q$2)0QJtF%&?!x8}m!zvSrrMsC5R z@CFx<`o{-7bHxX}(lv)WaUy)Her|b=e4d~Th1R={;kq2wZcg5w$uD?h-$FH3_~N=e zI7{D0z7Dnk(C1;1R@0pr2y%9m@Hqs-d|FsMt~i-faZM8)WmB&ES(D1~tBT|mTxyz= zsVydl{#3FP77aL@$9+KT#8nq<32_Vx2+-D(Y?~*XwTF7>PQDtb>EKB2jcfx8fUo}4 z_TgUI={`Ffj6H!jIamv}38(G$OekwhL!OPX+-Cb%68YfXK-D&z-<)o#8RqtGbHoM| z{<|s$>R^uF;0h0F0;GhiPRds~XI&M~!E89AoOBvkZ~!C$KVrrVajC(eft@{i%{0#HK8|HGM%NB2#z5 z(X3~d5_Z2b|B1WpFJ3ZtCft@Fz7x|kisa4{)lP_6cCeNVvjHR;Z&uB1c=E$JpO%!X z;&)maS-@)2mr)RmoYb`8OvD#PRldNWCH`r!1~R3@$t;wZqv(Mp0NT<3whsCR>$e+a zKzc7Uw*~$SsxJp(MmZ~)vK`Ye1GjAY{E`J6p)4-xdp&sBj~Qi~>iVKXmT#MfQY^Yb z>P_7LhpTf8&LrBpa6GZiiEZ1qZF6EL?*tRuwr$(CZQIV5`~ADOs=KQHcJ;14XYci_ zb$E3+Jy)JdR1X!f1-fW}G%ey|Rv|3hfcba8YU??lU}%Q17#umOcRzP*_Zn13}|3JqfSLiD!g z&cuiJl9o2cl9mKk=*t!k_{5~a;0HLR#&0W<)%1WC@?V`Wdeg>|J|!_R1E2@U18Q?5uy@OR|`-oQIe-3@N!|_txW%?I;wXB*1O+ahDir*pR+ky7iUiH^$)*sL~O$#%6le`9=Ufo zK*F%WP()huGWhf|tbMON<@t*RK$tZW@qs(47Mf@#?DIlARwwSN7MHme9pDL}SV7=I z=--Oo`*hHFLo_l*wC4f&-G#98315S}?FyQ~8#07(N~-Z`Wy4QDPrcI;Y&Ki|*9dWg z#xqhcIN92Q{4-?hZ!*Eq2En}ijN6EZ*Sz1;FxE6chTnyUKXt3`S~n^P@MmrvzQO5$ ziMrFlZZNsW*RwaO-}%X+izr8zyp&;r@w(a8krXy<%J8$hq?t4Dn@S7UtLDCPVw%*m3<`q zxEUV}C+vxDYVm_ZEGODxeeR>N*z&K3s$~c{WH5F(rQi#lm}pGr(_#*Vx24~tK(|**NXDSfr$Cw%Kz+WaQbC}g027!E<#&q=m=6^V z&bsT9$v)>~XNb&Z`C{gwv{QH_sUx{u%+h*hR zPg=EK0?oe+zA&5q0VZAK zV7UIwxIt;$|Gro!6>mq(3b(LnWScu;+pW<8^Ww!~Mt#NrBlI2%m$t>vK~jLx?U$AyzkCD#x7t+b!O*7m!yE#|$4XNG-v7)gpbmE@ z4^ve$4%y43pkzXest5gHBnvWuLLz`NBYUq*NUxH%%tej-4g>td9vmR}0R701IBtSN z)~Juk`pE_0%)X}re0{ya?I6m>OYE3|B->N_t(TA~2GEG;hHwjQB-;NLjGZwEi7tk4 z!&-srUt{iP+Bk--J(Iiyr0Cgt^CWQ07&sp0=SPWcD7H0KITa5gvf5_mDqLoj5~vKS(CVJHMj4*c#wuXnjfyOG z!cxPlWdGF!4H`1?ephkv^yWEC8q#k37mktDOd&_IL}nahq=738U`|*)4OH4kQFj@5 z;7aJFwk=+Re>B79wGA?v&f>!6XBt>h`m%u+k-^5oEs7P=5@L?@YN6j1#70n*$PW=0 z;z8eoI)ut(>L2?4`15A2lMm87MICBY6;@0C^2{fz!&d;~C8Y@7jnc2SH3zMu1*RUn zAn}!9ytXBwQ8f6oFKNg|+uI$S*AJzf67O{W2kr9-HIs9?in?C_5X*RgU6M2#rMhfRCjVWfF_qZuuG2c9p+>MH?tp2 z>MDO6SSz2-_+je(BtjtB&0ItKpE{HtvBYM;xbF&X7Q&9udf^sPR+G5ag0wqrNAeu( zoE6u-9uNlV|+_){?^y1pY3J*AP51xTS4`yJ`5$EveG{LEg6APvb2sDI(q12{k z#x&@!0g(|&#>Dw{4fuwpKVGN+8d-(5&U7iTxgsNeM9SkEOo_% zOIb|~Dk(<5fWe5IW&uWCjj5}V!tR)>Xiz$p-Jm_LHlxir;P|GNf)Br+{ALuB?;nQf zTSM6N$j2(C+`bc&x89&FntsS$I3vl85~@D(A^rFb8smd%Y!~|Avgwx?>UPAQHlsJr zHqtS@`0XY|7dg4U+N}>7zkI*A5cYpD*_E3bq*YL|k;#lzRe z(bLMs($LEHam>cCG5OJGP{H2B z&i5&_G<5K-lo(ul8Y7Z)xTZRRO|+lgIEg7bT{M7VCN zK_?=1&vG7)=UO_zU;`)|BhUe#Q?0KsV0#4|U33O;Xq)I&t_)JQ)eDB=zRd~Kb~NyK zyU*L>b{fc}f%iM0J-kXq)#}yCx^c&^zq3E~Td!BCn6EcFfI}Pe)DE95EVU`Zzl3sR z!H_Mnu}QN>meMZQ;6@+C9M!K%`A@uOJpLM??VqHXG)I8cJkhBOeE`hz+L_Mt^<W0iXqv=gy01J0rWQH)^r7VESmyv&A zB15Lgpyw9VCQ-_OwxUFKq=DJ89p-rl5<>Un_-OxV?~xVH*23aYC-%8iQ^lFtA!l_Q z@!r@NHKwn&8y5xTse+v)VM~B{I<_5*@c37zVe}m%1PnpH#<*HXcV+WkO>?icSE-`? z+xG)jH@QjYvrVSP6dvxYUnHQc{uK-=HfyK3TES5PTcbc}>YR=@Fn)Om+=PSxD1@Z` zYbx{yIjmEvd~-K8cj%|vj7y#k?+l3M+8@Qyu3sKB3AdLRB`mIU?Y7R6G1Goe#3w)v zx)lKz?Qw#&@|INRs%G{w8CLv4%$A%-gV@Ld08v=9{f& zMz`jEM%G9dlcZep0;Kz!6{X+~N(r$jv)?^VDKO1D1B7Wz=O2Irp&+EDGYo|_Bg0Z; zFM!~w_nL+D1{cL`t3T<^_6Oy6=RlQDyNfH?1wP=hvAu%;-@3lZ$iNNFZVUe9=)A(q zEb@o9oud<(C~phiA5Mx+C-_%Gutle!-ZEsYOM z*E8c6q$gu=^bJ7d*K6u9NNq-8M+hl>S}fNa5{=i%`T;3I44-eBoZg-ui#(I^d_xBDetp(=AfV|g7lz=Gr({8+SG9WN^%U`?u9Vi*b7WY! zl~A&`r;v{ecMa^#la$?Abw@PVGA%KCBJEnkY$K#W9DDvJiS>Ie*0R^hx5>Fm*Zcs=746J0)52hpq5BzMsfZ!P6I0!)qV={x6d$f*_EI z=frm0PXre)6c7-@|07+U>I497Wo=c|FE&XILfo-V776Ww#z1W)dI{17DpV3Q5K_oO zy)(iLvfv=6y-`WvKaX!(`ncw20e|49IHneo8Ac*}p2bVI<%kZJ#!Tv-)~tQJE*rZW zm*u|R-lO?}J3{{WoD3HDQxcppoc6oOP^aAX5=Z`d%`s4uBT{P5-LVCLJ3$}$So9TO zD00$C)KUz}5~$Efx>q2LyB8x-Yb5R^Z5$L~So3WRwJUXy556P62@+rJ2!p5jyZAM& zXxVspeogJo^$`s|$Q+ z*sg1FRj%i**bU7#quYj~49_svr%5(XqOPH|DW9To5Kt6YYvf`P7Y6@=p06-25W3e& zJMRb@H9lEO`r8eQ!#J;5yh9|zP&-8oFfsHbuPVmtXF7G-oOc6YNz|jP(r!ABacXI7 zFYLoBx(tc>Ma6v*N7-3Nk;d~U@{MWQ$qIX{fkcW9(A6i&sNwe~({VnYB0n?Y7@kt> zd$E5K9OoYblmGaZZCi)_!Z8vXTFEV$MVy*D`Rt{1r6e})d-{cB6|3-2B$u^>V1Q{w zKc_A-NnREHt}+6E>TD~9L-|z|t7wUJloRBF zwL`6@UjJY_SMUHfIUjd|&fs)Sd`c8?-=m}^4j6ToF%kz^d^XA%Qz2p1@uP2AufNe7 zrb=<=23Zs!g0MDGFw&G;TWY#U_x^gk_)57N*-zebPdn(>FpH^{tvy{0H@%4)&Z%6Z z${}f#QP~CrSIC zyFn1KYC4i0gwlN?yo;x@CU!#TOj-Z7FC4l<0Ux;Mm?mKQn;h;*FbtEi*gIVvg8+I+ zswIRAE`5$G+|goZx%rpN1}$3m`zXF=ez3wa{H(}BY10v|7G@r6a8s;L z13%VO?mkc^$ZRZV2D*FTxk?+@(59~&y!qM(r!#-w&Fn!duICW1FG6GX4Cle~fv))} zZx1H`L+$yC6VnqG@o_o1e{1N`JcZ6fU3s6K(_`LTIO z0~lwNA`UsNA#!~f(nJdkhIz|nA&d6v7bcPRzZmDa+&M2$m zp0*uf$u~@ti}nH3;-{pI4N(pcd*(X5)-?YZ3AUY<8KfB$Y2`J zY|LewFUeX=``pPK02-9LrE=esp|4@%8jU%pf3a_Y9G$ndz`R}?KpOM|WlzRvxIC9; z{aJ~6Z1P&D>v%RRN7=f(O3k#LuYc7fLFx6oWM06oT4r0zYTd4ln9lO>w(%H}J5bj> z&fmQ9Br_-6@Lq>rqU!*3C{&LAnSOUvuJx=9u1_jdoSmmC-lwZ7y(mu$F8h8e6BBQb zGZ$G!Ds493CHCD`ic%qv;}@U0<`qe_XekL1p1<_;T7pTx_El-z(fq=OLxY{w)3>9m zW<6`yo_ssHDvxA$feRR6hRyIL{Gz^PMU7cj*ejZd3<8D|-QoZ|X6q+bap~`gozJYn zFaUoA11W{vhN2F>T;|e4>Z5j%cym608f0W&Fl4$?OpipL^Y=U`U4S`sCU6KcMJO9) ze4&rg$K#2`*?>cZIVMmVB;+whD5!-)D4Ip^^rPvZC1W0V(3S$kkIq+X5)d=O-5AC_ym^?xX!Y6^LPX zB0zpKeC%mU#T39Y<%6|WLP(DNb}C4{2311*9WNDA0b7>?)|wA7I`%tS^1c8;Lh-K* zrJxeih!d%VDqLKgV3K70+^+%>0V}DP23RF^|C67o0W4*mT-vgaz!-=L*TTjp%zu~a z+E*~}FeDIA!Vft{@&D^A%`_SyQ9aZP<);I!jHU6=gdM@GVGIQV6^dhYYl%a}QQmk{ zpvAG8AJUYz_6n^)D@WU5u@v$wIMrQiAncU)9Bwd}ZGqy=mA!uF`;PI=^k}b8Gs=Cq zKYrfoNPSM@ds-i0{=`h!@!J^TD8)x|i2X(uqLXGH`p7iFfi>cfqy^_jIzIl{ zB{tR-9xf&U))|gG99)rSP>?vO$eUbQ@#0iQTK1WYc2QO~*Kb3~=2&_yt1YggBdN5& zC91zw!l(EMLafX#7B3~it!-hce4ZBG!&kifr?I|J54xCAAnQOGG~8MnsN6y;V`ry)do2iy@BKy8nPqST@^C&dyd(I>0Y9gm+g!G>Qb z!+cVC6{IDZh2cP#9HG9h9^?l^%-Bjm*}5rF7!C#^eAzGpQ6M%}a5INWkRR zVCJTBw36_p{ivGn`izIB-n{F}u{2doZk>-q_s)!NehS-ND) zKEu*H=x-S+c1(5hzjPejOA?wNuqG=Rpd^QjEWiOgT#`c~?0;zNqRilk?t{=4r<6`@ z3e!pr;NvJOTgT57Z?HSmdXYM`dKDk2+*2m;22=bAc7hAY&nXw_ z8cQf!qHv(0e}DT9q?M5JTyrCk!xeg-yvrclkAn(l<`oZFE}M) z(e4LKr@Y@&1;Ks5n0>@y$@2Nc`LEa}IBFtM*vZW@^D(kUVj;~};+E=DVw&s!q+fLR z{z3bQQt$pnx;6(<19nq7|5qc=qGm0GWsWIel_nx!e|Ror4H zWld_IY07)2ZB+4`tZweSi>M~@vUZCy;-8Cxpo4R&B?dK7NY3PQ;|Uc5$^W?kfA%urSN`H-rzqWrsBkW<~-rk9(GtaWx-DCO?jNiu-q zp$Gp;IXX|MbmIVS-1Ly`)<$^w^dYP-IV44k-zm;!TKiXBf!mO8L?ha+0Z#m2RQg7f zRqkt3&9Hbm*05R-gN?`*&-Y;jB2`RxOh@k<;pI1mh-PHwHN}<90E*FjSSam1;-4&p zO<#nKy;Obr2QQ7$sytF}yXVh{5(fam`N=te@pH^l00zG&K%_IQgv*BbEhbn`)Hyj= zPuRIRxGVUiAK#OAmKWcXm}n!%5^oou*6eJJg`RNJ=Y)I|IIm4no;t@Acgtw&(?@EJHugc#tUM_PlY zjUuZHx-f!tSwnL}UYr{Lo*3~iEoq0TdJRuj zhra}m1Gp>{(;mnad7OGJk#8!8Da*8Rp@x{+Elc#GX~RVR;+D4Ybz^8}iMZmp#@QJT?%Pj$^+coqedQX!lIAz}rBWboJ zvSU?%AGp~V+*QHZEpEZCZH#xIHM+`+UN71wrPnv3ADHVKh!A%Gd?Om;WKEgK5jaoX z2y9q{)+0dKe-5zAly)>L$gNFD zdo;_LT(Y4iBGR4i+l3DSK&T+cvl*1a8K1=)=;ZKYkGWz;M$366+>35+n>Rurs)-$G z^04U!=EPrB8;aV8S>dEgY9CwCO(**-a0ZessEZ>z$1OxtWddYzlx5{O+1#yxUtp1{ zU!Tj99M>j)J?jSY^Z=ZGZOY-9IuoZ!)}(q{x-K>;!Ts&Qrf7>SAQM8eDQb=MHblwm zt7x< zB`0L50;dzTQ8390W%KAOXWmu}RHuVo+fEiY>q(?xa&hzMBxyu12`~=;j=!+T0E-!U zNQ#fJlmJjSh1gbZ&mbJ;JFoMl3Q0iDS_qXP%?%8jVlCSmB*O~l1Nye}=vih52zH{G zwlrfDk{TWikGD9zu3Mdswl?_zfG-I95MofMi<1>r#NH@iyy?VeArDSG4?J)@(IEH1 zM;%SmkfXa|Sgp-#`=G_3^V@jC>PgXYD>o=fc7XT#S(a9x>N`})==9a!+=ZKe#$0`8 z&aIYiy7f!=5{9dFSI?eF%Zes0+uX-Oj*4b0EfbH8-`UqEWFm=bvQ9 zE_SK9<4zOJsCgRW%q#6rSk0!K1VIwvq_}f+O(FhOl5^H{i@Y==kETI1kmZ-uBrs}>W{Yii;{gkwm}mv%ZVaT?~*AT5_|aYSYhlbh(QY- zWI>mtBLBuGL>2>?FC66M(uLr1kvZCm1?(I2v9(;(pgN3yZpa zk*QGOHZD9^YravF2nd~QBN0kZ=CVq*V{3}BskH#T3(BFoSw#Laz@}~>8wS=84ZGNs zmPn(?pQ=>M%6k`6BW{0%iH*hx$wHMPw)m&y5V*1hd3!w%O>H9R0aY>YOI)IYc`PGG zMfoRltEQ)2UOq5S+5K;DLitoI6bBpmG3GFT+7N%y3;BU{AkSdEfM-xvZStU+LG%CW za=RWw3^G4ss^fTE8ySG1L5gE&0a`9v%tBzn08C+{6n;Vv>C z`fX@nxWxToRG{icVoi`&#z#W-mGsC@Cz(ym9pPdFYmB1p{R%Kb)R(#;WDoh(pVFhA zv2rvX6;j>$RTUCUgwF_Dz@l5Pj`7&SrMLodGI%tXkDbx%hL6}rg@xhVEJUva1Y*sS*M0ZiGAxz6C|e%LY>ZMo5i$gNF@xe zXcO>@m@S~zQ)FLs`1j(P>Ji2s4jq0WFEheZ6SnaCtWK{pkqbkQ{RgYELtpg5u2!t* zmeV_EG68giN(R-=yZ&T=Vqw*%Al*hrxW!`JRz{Hjj^z9*_?s7k49E*xiB}xzf|MNs z&I&6TDG^FG%n@_@Xo|QNkcGlRx50A3y*u;@g3-Ml3o-JmT4_B({`ZNU`XuEhQ8?+? zS$5|7KE^cBVU;n%6UlR@M!q&}!rA{a>Yn(Fb`AhTXT9()pzkLxSY%aBCjLedk>_V9 zc&ojfwf1UR%O;V9_q1)2|MF0>G;`X+_iLOB#j*T~}0|)?ahuAZz1&6Xovf_6=eOb`lwSg~@PTbW=2-hx6FKnAiU;7a&hi z`ooENeyTVixkd+p{gdW215?e0aZMj2E`o4IwJbau|0^b3)-rl*M3{LKxHFJU^&liV zjCAi02=Pj0iYmp#8TkiYpA!0T#%X{+nVcuRGeje4VNu$xgV>Pl{$2!2CdHTfx@6Co zqnRi9ScnEz*fpWjQ`LIA{x-f+3Y?%AXe{(E=RR5;>B4uidt7A9xT!@1#^V#l>`-6= z3{N-n(K&0ref~;DQ19Y1EQ++8&E*hY?Xi?mq~m4loE=lAka`-&8KZP zlI(Sx314goUqFC_7TH{|cp;3WOpp5OpF@R?pU-i6c&G53@iLGzNR(l!;BvQNifwv7 z5-4}OF{@TN4h5mS@Z>#=K&qjUxT41c0obXftpUvbIFh+3qpai`i%;3|N~hh7yvOm1Z0U@wP=#x)_2+m6Y+Y5EL&2z`>*{Dmp-L7f zJznGw5Dwi|JQbNr35Q&}@=T3y7^JPUo=`g5RT_cgms9E0fmZ;SzDd`;0QA=g>iSic zbO8$kmo!=-6x#I5)e6&FHe4Lm8hw?GG^t@VczcZjYoY_l;X$vv{|Qqg@*LHQ0|x?1 zh5HYSFl&z&&$$8%$oz@+{Lf)l37F%xqR6lVyZFEV8l!BWgCrCvKsF0V{4J0a&JnFu z6<3MX(L9=a@BD=6c&I!qL}*a%E0}P88xS^sBYzWK^A%S!S*?>UiVvO2TFlz~rEhahSg=|uOaFV2%G{fpE-dTP` z6=io&#A2=?G@PKvqFQWZjr_FKjaO<1w@7nTgsaUS-cRl!qN-{ zFb;l7EFxATbQ%SgyqroBI*XT92TD(6ht)($6Wvo5fXT|kvhnoO7g$H3N_n^ri*=9Y z?4mCV!?>b4H(FXq-p9VF2Xuxe8Xa$r|Fou>HCG0C$+ExZ3cR+d_!kGh_NTf!qy#;dRu4%Rm!XzmpBa1e%>dfC9j>LFmt$nZ$)oP;N|J;cU=^x znkp#~Iy5dJyi-hNev%<#d`m4IdcMLyu`Ep%%wcHarX$os0`8?|jF9RH{;T_Y&; zkf)GQt3xClmNqDM=Q{P1C-o>@-9F_d1X`Q{tYuCPm-Nm!0j)}P(t*EPoiZ~AkZh&K zc@;Rx1$0i;S!ywDjGpA~Pd0}ja0ME`oQkcMd-`0P<=EdbRSfAAfo(V+yBacJmD41UBg~TpV{ml?BFVA2F#2XD2jBjZ z#>p9=I6a!J*SE&(Tz9IwU!nI5aH7zxr8DlQSw3(os2^W>!P{HC>N2%Sn zabzFcEkqAKR_y^bXdmu+tOJrU3{Kyu+^gprT*ln1C)adRm_oICLG-ExaB=OUc4c>p zaFpu`#`Or4c#nn?Nrzr&U$1^HJn~&>CeV6GwLLp_yg!J`Jvd6;m_#4NyNQ#2m%+Xh z7)>2PDG-{nn@`2=A%1rs^1%l$reEdo(TYC>tzyB`ZN078a&#u&@G_Zm%JV>zgd*=~ zX}YsQHZlWWr&y{gk*n(AK&B^%U#taDm3m!K+8WfxG0z=++xJeKV^wea`X388lfG$m z(GMMX@^e$f11FKh-^~63{EHBT?qfg{{kCY-f^j`;3$YfSRzL+4?L`@IEupmpk$kkL z!&v(yHV77Js=?*f27z`;a5_n=`*R*3>nNl+%yDp`BWj#r5iMQ@Td>!{g z1qxxQpP%(?E+!C~1s2vOmQ>K7g$ph29eX4q352+5)lVimN8u3)oyH1D>W1zY?&c8} zd`n24ae{iApMY5wJzUF&`=SAdQYd$zn_;rnoI){z%W2}xs^E_QX=qF~&_0K=Y}-|c4hhieY*dSV<( zN6-hr`e{uuhG6-Ok5I)^Ri{KTr2_-9@S%J&;&+70|B54%F_Ic+^}*sZJ9C-_7*5Q_ zny0`V3n6$M1Hnl8h-D4!;^vrZ?o^xu@Ol_jNI z;GnG6<>E$`#u?ol_y`?d>4>U^_ps%0#n)>VDnLevl}ZI$Sw!%)GdIhy5~Kp+*XHA=_LvN8s(A@f8qc{hyNX%H6g<4M{AoD`}_;x9zy1yz4Hbl|3ZDH^WyjR z`jvfiEvNw>TSPeGI@?nJ%OQ3BvbP-ppy@KXC$MldCNRRo*{k90^~-RzyPa3MDot$6WtqeEm#zM z471Q8p1uE3tMw81=JJ2kYRX^#33UI@U!!^s8Bm#MD?k53wtqL{oS#78P=rjw_zS=! z-q_|m z-f~*c?(X^oQWy$@K%2*|OF1Q%RA($FjP!sbLsv;qrC~Za^Q|A{E#Grz>vtc-fU742 zv>dLPZMsa5>onL{yKI~PBXkoK_PIDe?uzX2Jz&#GnyV8c?0?+(T357W%E4yy zit!qdqRR`Xecq0~n||GDyZ$UCdsc^V^IA;8rN@^XRE1!uz}roUA3dx=OUML&rDB%B zHH8Xj+WiyW_Sq9#$+&jNOdmc$9NgipJDyLF7DgI$DDuX9-FV!6@0qc+VyMgn7;-^l zztb3!4Qe%h{M~=y(3EX>`?4v>&f%esRn#W;P;*a3`pW2ivUvWTYq(eWQhn+=Sco%0 zd;TT&+dZ*;{fUskk4oxca6+i*yXJy2?k#78k9Zu&5WAmO(g$8Nr`~F0``D(BaATtUYKyaG=?## z9%&z`GUtTCTl}px7a%5Ow(@)mRD%T~ZMzI#g9u#c97Xc);JyVj_gakAI?l>lRBXH0 z$}tGaM3XwHY>Cj?)=?3i5C8TmHaa4EHj=ky6BMf}#5Cp9o~Z@ZET zlpOrTS&j-8Hg?$(d%EBdVWtNhTbd;=r4W$g$7#|1KMkJnEc(&e&$8hiALS*ty>x*+7ZL;Z@ zS)~h(rL|-CV;QbkEl8!gh|{W$B9En-Hnvr2wWY8}#^sD7>djuICp8rTv|ZX~C|G@% zC~$&uHCM~Ds;$A&?@6#O&qCy|DF_7GL*_lug9PU(R?19|30B)I?Y*ZM4x%_3`5){OLp+Pj8dK>oR{4#Qlca2M%}MU zNWn1}$TWP$5&5}#>QpTkSo-yRl$O%6zl7#Z3sTWXsdkC%rbNjAL+2>-u)P5v6Uq1% zrs@0K;8apXMyb1H$FXeD95eRM;Je9Z{fX@5zd8{eh3WX!XfGLZ^U=-ZeS1&!Jc`e2 zL-+kL*}SNPI`ybDX_Fg0dAZnylh1z=U_Hhn7i%7^&XV3FP^X0 zz1C%a!5)DJ-0EQhF6?8n{DjZGR%8>K6+G9hovYIz^v#9PGL-6z;%M%5-M6^)O4SkI}jRL8iQpK^L<(I}n`liup+31sLEQl25<{&lA_C&cfE zX%TH)b*AHgPY}M8^d8m62urrvP_F~}uB#w1VhZ6xl zIDmLHk;&_1Hg$3FOrYNdVzD#uH!99Af=kIgFRGXeUbUqnzbwBfe=5dKQa#8vNTk(q zD(1pnxVL6DvC195o%!$&F!T;D3a$I6C)4Rl+9f(*^rjOQ^NEWGu?Yu~JtnsUxpOA@ zs%2l6u60^?+ML058O?WiSMWE*uFLZM@Y2e$u($@OxK?1My<16H*S_!Xk2&lyvH-pi zyM-acRCjAwZDX#LMyG&c5uS>1$zzB_Zn3TB6c4(l(FDb8CShgk7n# zZ%?U}IMJl9hZaS=Mh3UvXk@#9sWy}rk7X?R`<)43oDE~NqE#5OYw+2C$+o}9aeq;J z{wlO+{&T)&EtwT%9SlqV7nf&RkQ@P;>8yEDV9}8Xd%PXG<{D5zS$3`uk+^WwR0Fh? zWPy#HTeSM#`lO$}>z)I&7iZj*RTGSHxZKz?e)X)z{F2*BYp4TO0GK>|235e;jG zZZZ(;%Ac@+LxPpGi+*JwUBF`ep1Ln9SeFjd5=qZ^50J$Z#+J4+u4=8=4P%x z`I9xF{O`xNjseUVM(LU;<9#Vh=qB}Jmt-I($%@0EFBOm&{{3Hb2eV{#v`JOlrD~L zaH}#!)d2B)W#n*Gvb)fr`*qX>F4H;B0M{(NBjn?7XDXK4avaX> z=`m%4i3<6s;c&NvCNr0f&0Vnmt3ep5Pvos4B?)uECg2&EX@Nrl z#X@JN%oXoS5PB;3(hN2{3B{%UsaCnP$mL?SF^9L_VvO!v;MV6!H2-v1- z0cWRnM;C^kGLq;^=xNHM+qUL_H%-jubUw@hsGtqhPtKDxZomBwqDtxIgsO+0GM4Lg zau@5nht}%}hxQ%rJy}baYJpg^5^DqG%bhfJnY_~PJ|lnS>_lG#@b3ymU4Pyw-Y~U) zPP>ctigr-d_!16QCB%lDvpueW=C_sYuzvhYUIu3?Sj39LUXE|VC(a&NG$Zl^t(I_C z?$xIGu>DZ&+Xh=ON-Wz!-a=}50)4Xbi~7H=&H^Zordz{6aCcqY-QC>+gy0(7-F<@- zEWi@no!|s_cXxM!TOhdH{c>;J@6X>=Th%rD%S?>**x`8jd`E0|WD?3vr#vqy zE=x8{ULy#b(iQ6XELag{m2VNDXUB`@BCIaY6HaaT7rALZMX;rDlp zb=vPeGtzD+cx;x16&Jn{Kp=X#Y#07ABql@}wmvGp0X9IzvcA^s$G}7hcd2*8I^!L! zwQ<36#G3HnN7uzahCSzcn9`ISiG0IZtHy{%zp>h9vy8!w_hT~eFZp@7CJCFqVQKF# z8Sv40Xy@u-yC~AGS;R`*>?O`3-O%7ChouSV%bjNI-7PgTbs&DP&AJN58hQG~U~ps7 z`(v8I0HAAkw^O`Diz(zw9uB_>sYiv|4;{5GY#03mPXv;gAp+hNO$L55r~b-`tGCY+ z8+xh1WVm{yKL$8!s@gMTXG(eDzXc?C%&2U*NO={I(e-W7KDJk8i``&*yqJY4t$ zDOR`ECWPaB{Sjh`C-6XZMj}OT{JF?#7h56E1Qu90%^wnG`GF_&m(?tGfz>s3&-C#W zwx(1W0%P_S)rQo`w>YalAxIseNQNtXo=8xWLWV?z7a(Q_;RZvAc0^m`)`WCR+|Kh0 zT+MzVNnb;_P(%eQ<;m@?8UYdY*A7vEn8@&IYt#`ZGxJ05azd7vtWYi|RIRa`U;P8t z<;p-(*l*4;GA5IcF}cIc6n^%6}<>GKqVIbKeTDtzI=1l79E>dcM{P0&CtAT<6fYq9tch z^YRtI^Qg=F`@xPhZ`z%Z^!D>)sB(d0fVlva7EWvz(>j)Ky2TZSrk-@ye2_(z?ITfo zv8wCw3-H0jhBYTRC&6_|C|p2c@KLUpE04*ZFeaXc)IuVtNW6{twZk3PUI&q<0H+Rk zcdZmrxjXmtM)e^Jt;Zc3HUdeT(y*w!U9JX*(-^hLMh;dLzk;RdPxsPp3kB`i>tiNg zW_iKAVO)G-sWrnX&s-;3TsjDe%7Em|fdFKqY5Kke!syn|r;?ODGLZ5HE%+dZ}jJP8nN{8e85;d{_702umFN`T+#kWAn3bVdTV=B<@XcB< zY*6CY-|_y=xw?&lw9v{^x!2WZa5q`ot_Fx+$PDH3D6e|Zw!%5GikV%B z6bF)M&iGX(A2Ju~{zwGmgy7(??`ia@Mu51e=gSx{Q<5yD>BptmiTb%=`&Zmi6s6b) z*>09V*+8e&=`)n%R)OGKG5B>u&gR`i)L#AN+Oq!dcP})t zVoCHPk>tpCxV;h$q+$*r@>&ZGx@>XlrVg#ug|lM6)dJ9VZGG>!k^|94G*rrSq~IB1 zZ!^AF^6w(Q8Vpdr>z-yw%c?L?YwS8Uy3g1x0)euswfo7>IIL;<>QO|MN)DsV%wcSe zWgMN9tON;3R^oQId;1t7WV$mgY7+uDPKUoX_#x1&MUiLtHeu)B3JHs;#!kLAs(d>?MKU=p3!n9qBu$1=GN$;vMm}1-S^cj1rNKVXYBA>xgEqY11GilO0*%K3EsjfwyKhIyEf4{(e_AaN5 z>q9{isQqOvk)NGTNW)%84ky`P2aChWV8KnrR%~J@9Z#;~nyv&qpHoaS<%mw!pQhu2 zu3ELowa^P4)c3ORzecn8M*dT@vF%i!)D5rFfR4&%+=c0lrMI`#$*ak)vM0x`#odO7 z`ki2f#XPo(*L=~?hv5q6)}B!67qXHe4+4&_hL1@nc5Oc?J}^G56}NSq@f9uAH6Nsg z=%Jo*tY&|CwQU&&ZfYw13IRqKXY5QSclUt4x`3_03UJ#j+c2`Juyo!BtFK{JwFg6t7!(RT z_U|u(2~HU)fHDE3(<jEQwxh%=HEN=07q-7_b5wrF+GZGl$`VGdQ z{1Hy>6t?oVCcji3wjqK~?|6pU!!0V%0eW$jd$&n!fs1UP%Ky^@BZj~zppyOr&R^;ck{-OBGfdE5+Q>= zNIA*;FoOgUV2UP9k0n%6+si%^yrOs9!$ebsO}^6Y_%6WGrelLD4%&q-_S=$g7w5lb zLnwGLQpWHT1V6KX_C`SBkYXAs&`YRhGM1K!yrwT#(B8dF#$2Gblba%V>Q}E<(p!y77~4>Zykr zRYFmiBv*5&c(q6odG-o+@shg5c6JA-xqKY&Y+}{0%~hwnrK2o~<_?`o??!ir8aw8b zxL&TrS14F-tf&b5RQgos-a zha=ex;?IqVb^d0_#S5#MxNlJ64?V}^dR4!bs88;=>pHTzGuzP4v1)ChwtJU&g>gQr?{IN_d2-=6yr|%)SQqNzz8u?D<;;^^tz2 zpyUqJ4y{M*p688$#7gD`rhNpSoO4GfA9#Z)GAF0niH4M2K-?wydG zWo@Lal=En@SG0U>W^R-c2DOC;T`R4shiw6Qr`+U(fr(MptK|n}$7U0C0pz6ZPQN{~ zOl9)#XeS2QSLjZe)|iu*88BUnC`W6MX*MF9L=BE16F z{>!j4?DQv#!9p-aAe2-w;70>RB5`ctA38rgl$7eyc08l$W#}ESKT%OwiD3sJ#tYr^ z$cLQ`S!i0&QorSQ7N~@Z+`oIwkM!h$#6Yt4AeqQ)J;)jr_I!PQc|qPn|Kw*@ebn1u z6rK=nA9h-*K9j+2_TASWwh`kFTC#)wTNA;Jpv302vQ%on#oE4VI?&eoVzrZk z`*wIpd)+w`ew|}&s%owNqXaKS7F7`4;J9|Uc!@VFrZ{f`I2ih_`FL3*zhy))UZB^n zoNn%?4Dxt0s&FE~&yAZ>jXN5}PMqJS(z?MNAHuftR{Zuhz0sXE3%h}am!r;ADrkyOp=UAX@6Q{@V{$cTk_wPGjP76V?2n@CkrRs1 ztVv3$D1Jtm)mEahd>boif1AR{yNHF@TH~U_s#A zDnK=G4-3AbvuF*8^#&u!bZJ2vE6nEfJG2o4gjkf2Sjez%C}-M9cAn|${fGJY(68U5 z$foM}HUwSIg%2kFB(a1n*;yv5%t@p%Es7V{33H`apGgTXG$uzu zF}MCc!&Ky?m1QvF^r`n<7~fqwa&tXU_%&V+)iZs;;&9c$HPyJwHoV~=;fG19N7D$K z-l_0nI=wwRm5re!o7tkBR{2GvN61^!1)qe+uj94loet(*x>R`2v(Mt2A5a9%>HPLv zdaC$^;GxfSZL>D((nK8jdcF%bXk6W%SdAo zhf8yF) zcy}}6-O9WMJALMvLyKcwRQhvrH%#cI3HfCKUny}w-cVUNb)jG+W-6Vi&?-HTuR`Y< z-}#4MLjhQaMoVRUZezH zo4u1VUXjFj3#lkTVpT*&AuKr2vJOE0f&+Tv37fF<6Wrz^v#l}im%NyZ@nU{i^Tv%V4kk>ji{_(#;M)6XRy9(|xsQF+bAH`IT=}AEBLFIP_5ro}N5`hDM2yy*uvEo({xe)~e{y z*aw7B-Vj=f!jt1-&w9y^c5J};md8!vvGMg`!*{dI`^TqF|7YZ!5O*YmF}>;=6ADL@ z?w~;Fb|Idcf=2ljnk1z}eAtG>vL5>|?T+e6BXwY#TI_?HXS^x~_1gCWFNyjCWPXK|L zO2H8tJGG|pmL;TZxS7fIjM6cq1u<8c^$wEy-79h{|=nLdjJRq84oDaA2Zry@6 zDP;nTioF6;u5gR4P-X0Vue5B}hqAiU*d{Oz&Q6;(C((IVvW?nIk|XdCgr_QzQ9+wM|vPw+heL>3=^-B$6z&Um^mdMn#cDF&qU9m zVoZ8`ahy_)-kXxRLK>5PWMNx+9X$#c=>NEqbwsmFHQZWZ%G=1Y4yn%`vkPfqEgBY% zArdYdpV5myj0_e)G)k$0`U8Y^SZ@^&gHuv-8BcB*3mU zbuKJk02|M-A1A&NRa#1%J%Rv~uOQN3ZD)Xc0!%h^9$UWS>EWxDxvRN>+^-LD& zUB-pqLB{GPFyIx+31U!&aFvH+X(G>Nm%xK*j+qNSRV(WzZ!tu@-C2T7uzQNS+(YKj>MnLr1;WHOrGx>!s` zJf`HPJ8Z{+ThNvEP;O=1o<{M^YDkZ^-%+bw>EWT;9!x^D;YC)I;`Y5nY=&Wh5Wfwt zGJ+0k8=^b{_Y9Ne4m@LEhQyC*MLL>Cl#OE7$T9lXrc466pK(Zg#$;7P9l*@R$r)F~ zJ@p|${<xD;(yoM};S^_wWvF>Cwvu6Pb!b8V zoA~FV9$Py5m<(n@SKkWZc_6+MF`CI%IGyCDSlPwXw3_h?dkaCeF}gq|tD$lE9mS-9 zf{N6=U)b3<+}T`NIhe)5F}saU$>s#~BBjRtb^Q5ZPfVmgq{yG}J}45rgT{t{fCops zLx@A{natB}gDu1XyoMwC$3mzVNkCuoVSwO1Sa_Jqm_rO6{VFtqZWtYYJC`DYqOec{ zTQ{`UXpu^7_s}dX?S0~>%TM}pIA0m?0zSWO4J`J=5b?HDw+W{rVn|RsZ~xplU%kud z@IN2ld%0-5fY@`^6Gz@f6kwYcFUw#9_3ZU`Ck9!MNjY;z+9JCw?`Dzo10~TT z`3y6uv&Q3Mn`OHKOWcnXt{t|WKPIA56PryI4>QZtE)6T}pHagv?%Qv`QV6bD|SwO4g?N z;fBAVp-YT{e53G3-pu8kYE!kalZxZjjf6N~j!f%Dl}J<4nGNe4ht+6y!(B?3k&Rk6 zb`xIwuFGRlu4-?O4aeF#Vfy!mDfGrg1Twmiv!e?ck$O`Xsd+D@ruwF1ND&eHH_?1q&=vQ+F^0nD7hLR>xS;;sDEL|<@s~fywovH zv%K&4WC|BoS0fx!cS_Q;a!&%>BNm8@qeu^W&-seM@dO>}FT?v3Byx*WeE7Z$(+I*6 z0!Ynb_Nt&y|G=pFJ|fT3nWafU(a=nOd$Hb^f}*G6q0Qzu*Ooco*KUYZq`2CKuS4mS zTNmy|&9EI}BkJ9*dxfAbQ9q~eX&-#!qoZDc}ev#)> zXfLVsEmyT9U}|U{%dxV+=@S)@9Yv+^BoT{3tnz=Hha(kL8!y$s>HQ{lfG_hd(^T$B zAWCE4B07XUn>kAPS1uxSE>#OF&6DfL8XHu$h` zFFECez~T2PsSv6qd2{IyCeG^OPuRLd(aOzg}2*G23u&VFy`t@=9puKL*O z9d+KOrjETwrMcbRJr(w?YF}76xw@$L+<3lk#**TS0;>F+L45>1JWGp^4j)tzzAz`h zDwCe<<*Ess8O@Y_(b)xLx>*|t5S%V7_Zp@yaNSyGqz?)l9I1DODL)8F?8Xb=T$Kn- z90Uom>~ZdK>h4lxqF&;Yex0`#`Z~K*HA_WuvCFxC2mSCKJ5pZ2pcmw?&qQfCY#AB- z{`VD^6>xag->+eS(+!a=Y7a9;(fzy0t#fMK{x5`~NW=$q#9)SPK{b&dQY!Th9wgbl zu6t@ZSNfT_uJr@gZn`7a+=e;MG0LM}xytozauK+;ZBoV8{fLV0Egvj!ubT-MkKJn6 z^aou1^l3HuRVs++)D^=El-*wj~YyZ6y`S}Kl@YBy-n&m~U?3 zD^&tHO&21a#t;XCuC|mJ(Iq_WrU#k`8o5x#clwJ{ zPmCCx@KHH&=HX1)BN{Ai^X+XAeM{|Q^i+Mhikw5N>j?Qp@nialS{o#)P&wD2U`NXr zW zIdFQWX&77?l`z=U>$H|>raN2V7!GkUm@u}M3l5vA6S}uABc!bj80jrVdrH^z?x+5U zI?X2MT+=k?)NbOyv@f%XN@FH3`_7MDu5fB^&Stkf!lDnA1!F;ny{tjHW?xS=*53yt zs(#0vwaq$)-JqSRTDGj-@fn*AjcBDdIG%yKPK2A$d=?Dp&Qi>B6HM}S8w=pAIZ<; z0@aGoG_}3kUpNzc5PaIUS!vEU?CVR^%ia zMzK-(yRC<+>mt;j#1~lKqgXm%IWMNRv%@tC*c)mgVR;+!-z;$Q3e#|YZxI9mxC1IDch}Ca3OlONNVCM ziM#UH`MEq&=TPG?K@kRP_i;>DOh=k_^QTfqg@u3eDK?rYWdDJpd{gXO)bIlf!@&{< zb2Gw>?B-Ceu0N6nG(>_^58+q=9Kjimf99&!gf-18wQZe8Sz59VYayF64c%3JXkT-P z;)wT7BTUkWuWW+O$qyS6%a9~K<{L6|4sgNeqU1~rbIC$44_ufxKDS6qvlkz{%uK4! zosDC5TM3(G-)-_apBe=13fwiwp<^_ZP}%z%P+`}meH+Y_Sb;1T;-l#U4j;)i*qrdN zVXUcvY7!Z1&AV{e88wEy8a?V8B2@-ZFzeorM>J{DlX6VnL54+*n3*bOdQyRU*XD{R z4%HcsEnI)#L@A{Uy3)+vB;<*AgqF)F_d=W#Ah zBYTUImW`vP_#CFtnnrVa+ptJ|K8V~wpu_QaO9EJ-$WC_0bDG6cE%{t4W7~W^?{iP;y{F}MlC>NiSX8LIUwA|cQ zZ{E#Pw+<0tKv z(GCuEgeRDEw~FQ+1y(0@Ku6~GR=w(IP;jaR%8$=PP0C-~J87~9Q38TGv(_x3=6Y8v zlGL3hh-j`)6*Ib3eyI`YmBz@7JMeRG1rP>e(D585-!KM$jRZnwZ(E{>45(PInNF!R zFK+GXjH(q!bo*mEpC8p6ZAW+gU|-ofxQ7jXdhmu%t!%r2f}+>dXF@_2A-ComAc)}( zP(ADnq@b>ldAHJ#N{-!XK+IRJ7RqPC>U+}9Su|*Y4QJYK0S`YlW3U!PL1tI49*N8P zCAHJ~vXP3&<`_8U1djdS(bJ8jw6;|$N1>HXvrp4riynybgxNjo2d|E(6S^U-f4>@q z?d++tc}_Fy`~99Me7J@W6V)A$SGLEQ6F)33^UMN&^$TozRQl?6jYWty@)dop`VVuN zeOlP^GryQmy!R-oW)j0ig}yxQh8jdro9#qu?UBssqY6Nj01!r-T39r-KI#NeyR=Zm*)oxCy zq7?61y2OAu&s$uT1vhSm`EOjIWrth~4+CFQ&wA-X!a5XC{qeF23vf^upnj{fIh(`P zS=h0jF?{0l!fsk~7$7WPcmACd+%iq9DJqkkD1fv1Kx}cR+1xI>JG^S9`bAQwReXci z!!0#e8&`bBfJ~cx$mXq0i7R7z{A_1!O?z=cn<5VwrJ%`|v_naCINMO&ZH)9bcD8zz zX<;Ujg0IuQI0hJS6z%hTlA~`MlPvZ#Zeo=X8S+{bU^3<4KANsN^XfoA8JfF78r(homvn?36{BvXU2HIq_ROUM~d|<<}m$Gq364 zn2MAYZY|ZM0{1~^^~#4stSz{sJX6Wi@Zg|mKqIMgOrd6o%eILk$G1@@n?ajnEt3rVv$?M;VvX_{t{%!lOtAo ztPBocvL&I4+)Dg9!ZR}^KtXTxdf!u5kV>Jg#RtJWO8v^RV-AaOP2H>PE6-5uN@=j? z(q~m-h5?zaG8~5n4m&f}szy9!w=MnbJzj>G^yhK-lo^x$Ak{KzcQh>M!=8|ZXeSL| zmhl> zk}TS6GP|X1W`2pw=*ZXEArivh>X8*%M~w_I8V->fEW#3BOQSf-u(=`u^Q)0gaA8^EUiu)bs51*FPPYgn ze60TM+eY6qEL(cYh7;HVmz@riKYr^8z|#Vmd-U{gw-zJha>tihU`pf}~&j{V$t zrW^G|Vc9*z*|w}Tt7z=zAov|jRAuq1tS)=Ai0J-ZOFU48_nP!b^ObRZfOPhZKj9Vg zNm#Dh!K_#7$o76i9(3gF1Jy1YXU9<_&e@w4%EFTgYpr&8azk|lkem<=#c7_hDNFk; z?%qzNz7=`gLi&0#f4K}79!IUQ}eJ5u$l8fWysu$xIG6mY#Q z7QNmLuYDix_Q3l&^$2_+>APTs@~?|1&Fz83BGilTO6U(_3nbBX*_(SZf5*-bvp_?Z ziq?<{-8v?vhr(JONV?EFNwmc~?TPiwjo@tLD=kX}#46M8`i1$3Yqh2>HoKioTiwf@ zSy3QtlpZL3dQ9QJyAlu8aN)(RrFhqbYT&Fss zuA7Y%3R0OLw6?E@+a_0JA#N++tT)Tl&i%Sjwy$uX8rU1-MgHkR)%YIo_mV*V%M1 zZ%E%(9j*BNbJ}X4SD-&QyfD5H-Tssng?!{=0LHi%_+0chK;exZigr=Lt%Xiz9_3UW z0)rQTVUK7KwX#Qz2yyn|=4OkuBPZ`@lbXJ-OP_y1o30d#_w|U8B`>2@jk!*elA$PFZ90P8##)TZdP)gvbv@I^9B1sG^Ak~ezjNJsIi?91lK9PNU{ zDLlc)z9c=lTOu#Rh$_`@8X7>!ZMSI_tvcn zbeXuMdLwMm%|a>}scxheRz*}Vn# z%pE^F#6xstGOwNFTG~o9{1x5f340b*q`_?4!PH`sxA>W0|yT=c&}g!Q}}QWC*4 zw}TjyHzNHtFtOMqFPICrVCSSGzPy$LtUiB(z#~g^nJO7+J5&yCPXl5Tp@Fs zqHZ^wg0+^NB5`Bny!4K1em$zCLKXaBJ5d9Wr#m(V9If_M{%~x8KBJmjQTWr0%;SVy z#QPK&C9la#i|WX-_jz9YMGiX92KuQSY{#7Bj0v7<&2J5Tz zj~@4K@4m&YGptAV-b|b>d(p@Vny?;zgIK5WJ07u!xxFO|qjKY#sn)Mlb>j+I{+fnW(5SyD$*bQ&5VO7YrhG~8+TQ5N8%r5qo>(u|OH*K-HjHS7M>9K|hzv4kKpgn5R)!7QoGlF;mri(aT zGPXIQ(eZ`6!#vk{6L8Qut|im%mc3LZ(86g8_B8j?>b|@5hY{^Uqc%JL$;;Wmt%>~% zFPe0P#nUHqZvL#^J$dPh0N3u!mvL0O#~Fhc-@53f_oW@SOYMNQHV$WOD=`S?IY?`g zMlIM=S|oof`5q@+rYeXYER_!de0bs3G=G-qre!bQYP=_w^PD7=;{e;)d-2w;L!>9^0hXbrqAdYUoVM^{>}Qg)@jV`PTJj7W_jrda)r{U2 z&-xm5!nC6E5k=k$7j2ivu{b@r%nD-m4C!4q&o5L`L769$Sq-p~=abR)KolYrTTSX1 zzS$?I27)$k+Nt0rm|0!*FYMS;4JrX_LyzNmNuz11+N6Fl!FJbW)FO_A4d0^cVZ;FMOB$}MU?zyuo*Myqv zz$U!dexQ1Puvo3sqmfY%mvE_Nm)#trd0dAg3+q799_CAhEZdrwWhf&TYx3m8bSFY# zP*XWX=n>-Vu|LtNdEVxkIT6C%a{x)_=fSy0{9LgqUJ4Q&2M-tbeA&M|u|*kF*ATJp z1hj=98`8+<3pr;%F3W8jN0WyYPe;jKvcC?dQ19@?qB|zsV|gWdkn{{@R*{QC#_r&6 z$$iC;?m-wq+&rHsP}z}J$N=~yNcaV5hA0DDC~s-qCPH1K6+1#Rf4E|h(-5;W^fjTs zeXg`cjfS5QUfBimou?h&t-*hB{%L{mwk-(G(6M?*(|D&WJD~p>P8lQ|zAP#6D<$UK z2kwjCV@LXuJ5Tn;SzAkFpXyLL2`AaIg4k@Tm&!lG z-6I2A*~K!An??v1+Oh`jt6F63j0}K-vPUg@Y_>oxL{B5d?Cx|64 zwo@u&l?|9%d7=INW2Fw5cmB|;YK!FS^{Cd_!!?{FZIM=H?%?~-TNZ#QAeUEC?C`NB zoa(f&s{ID2oic$@myU`MoM+A%uzve+ezUDse)rgdBba?_{enzw=wbMblNS;iYb7br z$}4rCTYL@domf3gRlJDen_pu^yv6l-OZZf^Qio+6s8podAx4V_pygyS$^?g|(l?Y_ zwWIG}r`)9wx~06!%aiA6kSeXD&Lhbok`7dfky!`Qk^HVLfk&5#H~+1_8P~<3`1H{L z3TIOsMtV@34PsrAs}1_xVar@5(O@Ih%Abq9`rL5@LHB(IL-9;PHZ#S&hlk*-oJ#@1 z-~qNSu|nT+)P1?-6UyZ62cW&j<8M$!8EFegb(ns{D!po>{!Mrv^vi+Ip#!^cFZl!0 zx_7KlW7l=Y6YKr?*$$om5`7W0Z>Q#a$&OLU*nA2}YB<*`;NPqBpv*R$Ht^aj8Q6=2 z3AEir`A2LjMosUJ7fC}S^BF~%IcmU`I(oov?o#pqMd$ zh?6uO_c>n_;-vh+@tKpSEQQ$eeyN%KPnj2foBW)eZ+>r(Mi`9rW*Hk(@;m8>(i1tx?cTB;+hr)E^yk$ z&h-Fim4`l|Vu{6|{YHW$3UlU?MlLf>#Q(;@G$f(MKw{$soghTmBkx@~ z%^^I^<^984_HDpB%-qyBd9frsc_KFKJc4lprd_Z2TNtBk z%?AG?**7;?Ya@eT#JAy&vK|9lu6F;l`)VhlznaZE0XCy$|M3k z#EQJ&3%#O;7Rd8HVLYAYS4}Z&lG9Pj@Ke^!LJ3ux%pH*S*2k3*xG>{%k_UX=Fr{j! zJtj{OjrxQxD1k=cPa391yM2f;gYg`6TWa8(!WCpws-B;HKUgY3f<-)^yfQyXPjXdu z;+F8Xlkhu{X&bIsFk#o&WhxHPPs6T;qw6JuVXXzCx|ThR*>d+}{252XPucN-l<3#+ zI+ukXft#h|z(+bpqMNORpm!xTXlBo*$pa6lG0P3z{G{LB{>xP=N}SscgwUr8MTqso zyw4318U!Jj3R>^igOvKgKkx*JJIy)62@M|C!ogSJ%zeZGFTH|8Ng+VpLjaK35F^4B z7xs=gc!MS=Wk?M&19UpX4%r1F8RmmL0c-W+fgFa>02e;G$roT3mwB*K4l|hjw~#=B z;#qM(mBWlgL1q^D;4kLEAIg6La7`Qb590@i0Ky!>1B^HdxT1ldN-^*Ox&8u$;QkLJ z2l)7#sA-(3CKW^U=X_0XI&qBI;!B)`TN=;NkJH6M4;(WD!@s(4X6ka ze7^V)5a9Le|0wJfU`Po1;8fccq&kKRMam35IxYw}Mgqtz!@r*cp8^jW1O&x@>;eIS zzygM-KY{C`SNaAQXL7YLO5b3u0F z0Kk9eaQv6URQv;xPN0AW#_0fmwK|AEeDnE?MCq4qC_6a5F$nPUX}73cFW0j}YF z1?9}C5dD7~1^7|n{R=<@WzR8y7?aRJ*|X??|949VYp?v1pY-2?{|)3cuK@VV<@w)l z!8M#@@GsH-p9eH72x$Qw@Rx__U+{hE|G_kXzuf!&f@U)QTp)2S2555*9q^Zz*Iyug z_8*XXkp=MI9x(rMmU&=^0z^5A3;1`&;eXv^{)I5>K^I*Vp#4Q$z+VHKe+h8yxD5=k z{CU>?=@|ULR*VGxzXbk*E;~WaOYi@;qv&43h3e}DEiaivCHI2Fme~RS*-L<%yZ { protected boolean queryOnlyStartedJobs - protected final Duration maxAgeOfJobsForQueries + protected final Duration maxTrackingTimeForFinishedJobs protected String userIDForQueries @@ -85,7 +85,7 @@ abstract class BatchEuphoriaJobManager { this.isTrackingOfUserJobsEnabled = parms.userIdForJobQueries as boolean this.queryOnlyStartedJobs = parms.trackOnlyStartedJobs - this.maxAgeOfJobsForQueries = parms.maxAgeOfJobsForJobQueries + this.maxTrackingTimeForFinishedJobs = parms.maxTrackingTimeForFinishedJobs this.userIDForQueries = parms.userIdForJobQueries this.userEmail = parms.userEmail diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy index 2a550e89..2c629835 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy @@ -23,7 +23,7 @@ class GenericJobInfo { ResourceSet askedResources ResourceSet usedResources String jobName - File tool + String command BEJobID jobID /** The date-time the job entered the queue. */ @@ -81,9 +81,9 @@ class GenericJobInfo { ZonedDateTime timeOfCalculation - GenericJobInfo(String jobName, File tool, BEJobID jobID, Map parameters, List parentJobIDs) { + GenericJobInfo(String jobName, String command, BEJobID jobID, Map parameters, List parentJobIDs) { this.jobName = jobName - this.tool = tool + this.command = command this.jobID = jobID this.parameters = parameters this.parentJobIDs = parentJobIDs diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy index 47d657e0..72de7890 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy @@ -23,7 +23,7 @@ class JobManagerOptions { String userIdForJobQueries - Duration maxAgeOfJobsForJobQueries + Duration maxTrackingTimeForFinishedJobs boolean trackOnlyStartedJobs @@ -91,7 +91,7 @@ class JobManagerOptionsBuilder { JobManagerOptionsBuilder() { trackOnlyStartedJobs = false updateInterval = Duration.ofMinutes(5) - maxAgeOfJobsForJobQueries = Duration.ofDays(14) + maxTrackingTimeForFinishedJobs = Duration.ofDays(14) createDaemon = false requestMemoryIsEnabled = true requestWalltimeIsEnabled = true diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFCommandParser.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFCommandParser.groovy index fbaec626..5c865b1e 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFCommandParser.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFCommandParser.groovy @@ -9,9 +9,8 @@ import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.ComplexLine import de.dkfz.roddy.tools.TimeUnit import groovy.transform.CompileStatic -import static de.dkfz.roddy.StringConstants.SPLIT_COLON -import static de.dkfz.roddy.StringConstants.SPLIT_COMMA -import static de.dkfz.roddy.StringConstants.SPLIT_EQUALS + +import static de.dkfz.roddy.StringConstants.* /** * Used to convert commands from cli to e.g. GenericJobInfo @@ -53,11 +52,11 @@ class LSFCommandParser { if (!commandString.startsWith("bsub")) return // It is obviously not a PBS call - String[] splitted = line.splitBy(" ").findAll { it } + Collection splitted = line.splitBy(" ").findAll { it } script = splitted[-1] jobName = "not readable" - for (int i = 0; i < splitted.length - 1; i++) { + for (int i = 0; i < splitted.size() - 1; i++) { String option = splitted[i] if (!option.startsWith("-")) continue // It is not an option but a parameter or a text (e.g. bsub, script) @@ -124,7 +123,7 @@ class LSFCommandParser { } GenericJobInfo toGenericJobInfo() { - GenericJobInfo jInfo = new GenericJobInfo(jobName, new File(script), jobID, parameters, dependencies) + GenericJobInfo jInfo = new GenericJobInfo(jobName, script, jobID, parameters, dependencies) ResourceSet askedResources = new ResourceSet(null, memory ? new BufferValue(memory as Integer, bufferUnit) : null, cores ? cores as Integer : null, nodes ? nodes as Integer : null, walltime ? new TimeUnit(walltime) : null, null, null, null) diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy index b968610c..55558d05 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy @@ -56,12 +56,24 @@ class LSFJobManager extends AbstractLSFJobManager { return date } + /** + * Important here is, that LSF puts " L" or other status codes at the end of some dates, e.g. FINISH_DATE + * Thus said, " L" does not apply for all dates reported by LSF! This method just removes the last two characters + * of the time string. + */ + static String stripAwayStatusInfo(String time) { + if (time) + return time[0..-3] + return null + } + @Override Map queryExtendedJobStateById(List jobIds) { Map queriedExtendedStates = [:] for (BEJobID id : jobIds) { - Map jobDetails = runBjobs([id], true).get(id) - queriedExtendedStates.put(id, convertJobDetailsMapToGenericJobInfoobject(jobDetails)) + Map jobDetails = runBjobs([id], true)[id] + if (jobDetails) // Ignore filtered / nonexistent ids + queriedExtendedStates.put(id, convertJobDetailsMapToGenericJobInfoObject(jobDetails)) } return queriedExtendedStates } @@ -102,7 +114,7 @@ class LSFJobManager extends AbstractLSFJobManager { } - Map> runBjobs(List jobIDs, boolean extended) { + Map> runBjobs(List jobIDs, boolean extended) { StringBuilder queryCommand = new StringBuilder(extended ? LSF_COMMAND_QUERY_EXTENDED_STATES : LSF_COMMAND_QUERY_STATES) // user argument must be passed before the job IDs @@ -121,12 +133,12 @@ class LSFJobManager extends AbstractLSFJobManager { throw new BEException(error) } - Map> result = convertBJobsJsonOutputToResultMap(resultLines.join("\n")) - return filterJobMapByAge(result, maxAgeOfJobsForQueries) + Map> result = convertBJobsJsonOutputToResultMap(resultLines.join("\n")) + return filterJobMapByAge(result, maxTrackingTimeForFinishedJobs) } - static Map> convertBJobsJsonOutputToResultMap(String rawJson) { - Map> result = [:] + static Map> convertBJobsJsonOutputToResultMap(String rawJson) { + Map> result = [:] if (!rawJson) return result @@ -135,7 +147,7 @@ class LSFJobManager extends AbstractLSFJobManager { List records = (List) parsedJson["RECORDS"] for (record in records) { BEJobID jobID = new BEJobID(record["JOBID"] as String) - result[jobID] = record as Map + result[jobID] = record as Map } result @@ -150,17 +162,16 @@ class LSFJobManager extends AbstractLSFJobManager { * @return The map of records where too old entries are filtered out. */ @CompileStatic - static Map> filterJobMapByAge( - Map> records, + static Map> filterJobMapByAge( + Map> records, Duration maxJobKeepDuration ) { records.findAll { def k, def record -> String finishTime = record["FINISH_TIME"] boolean youngEnough = true if (finishTime) { - String timeString = "${finishTime[0..-3]}" withCaughtAndLoggedException { - ZonedDateTime _finishTime = parseTime(timeString) + ZonedDateTime _finishTime = parseTime(stripAwayStatusInfo(finishTime)) Duration timeSpan = Duration.between(_finishTime.toLocalDateTime(), LocalDateTime.now()) if (dateTimeHelper.durationExceeds(timeSpan, maxJobKeepDuration)) youngEnough = false @@ -173,82 +184,95 @@ class LSFJobManager extends AbstractLSFJobManager { /** * Used by @getJobDetails to set JobInfo */ - GenericJobInfo convertJobDetailsMapToGenericJobInfoobject(Map jobResult) { + GenericJobInfo convertJobDetailsMapToGenericJobInfoObject(Map _jobResult) { + // Remove empty entries first to keep the output clean (use null, where the value is null or empty.) + Map jobResult = _jobResult.findAll { String k, String v -> v } GenericJobInfo jobInfo BEJobID jobID + String JOBID = jobResult["JOBID"] try { - jobID = new BEJobID(jobResult["JOBID"] as String) + jobID = new BEJobID(JOBID) } catch (Exception exp) { - throw new BEException("Job ID '${jobResult["JOBID"]}' could not be transformed to BEJobID ") + throw new BEException("Job ID '${JOBID}' could not be transformed to BEJobID ") } - List dependIDs = ((String) jobResult["DEPENDENCY"]) ? ((String) jobResult["DEPENDENCY"]).tokenize(/&/).collect { it.find(/\d+/) } : null - jobInfo = new GenericJobInfo(jobResult["JOB_NAME"] as String ?: null, jobResult["COMMAND"] as String ? new File(jobResult["COMMAND"] as String) : null, jobID, null, dependIDs) - - String queue = jobResult["QUEUE"] ?: null - Duration runTime = withCaughtAndLoggedException { - jobResult["RUN_TIME"] ? parseColonSeparatedHHMMSSDuration(jobResult["RUN_TIME"] as String) : null - } + List dependIDs = jobResult["DEPENDENCY"]?.tokenize(/&/)?.collect { it.find(/\d+/) } + jobInfo = new GenericJobInfo(jobResult["JOB_NAME"], jobResult["COMMAND"], jobID, null, dependIDs) + + /** Common */ + jobInfo.user = jobResult["USER"] + jobInfo.userGroup = jobResult["USER_GROUP"] + jobInfo.description = jobResult["JOB_DESCRIPTION"] + jobInfo.projectName = jobResult["PROJ_NAME"] + jobInfo.jobGroup = jobResult["JOB_GROUP"] + jobInfo.priority = jobResult["JOB_PRIORITY"] + jobInfo.pidStr = jobResult["PIDS"]?.split(",")?.toList() + jobInfo.submissionHost = jobResult["FROM_HOST"] + jobInfo.executionHosts = jobResult["EXEC_HOST"]?.split(":")?.toList() + + /** Resources */ + String queue = jobResult["QUEUE"] + Duration runLimit = safelyParseColonSeparatedDuration(jobResult["RUNTIMELIMIT"]) + Duration runTime = safelyParseColonSeparatedDuration(jobResult["RUN_TIME"]) + BufferValue memory = safelyCastToBufferValue(jobResult["MAX_MEM"]) BufferValue swap = withCaughtAndLoggedException { - jobResult["SWAP"] ? new BufferValue((jobResult["SWAP"] as String).find("\\d+"), BufferUnit.m) : null - } - BufferValue memory = withCaughtAndLoggedException { - String unit = (jobResult["MAX_MEM"] as String).find("[a-zA-Z]+") - BufferUnit bufferUnit - if (unit == "Gbytes") - bufferUnit = BufferUnit.g - else - bufferUnit = BufferUnit.m - jobResult["MAX_MEM"] ? new BufferValue((jobResult["MAX_MEM"] as String).find("([0-9]*[.])?[0-9]+"), bufferUnit) : null - } - Duration runLimit = withCaughtAndLoggedException { - jobResult["RUNTIMELIMIT"] ? parseColonSeparatedHHMMSSDuration(jobResult["RUNTIMELIMIT"] as String) : null + String SWAP = jobResult["SWAP"] + SWAP ? new BufferValue(SWAP.find("\\d+"), BufferUnit.m) : null } - Integer nodes = withCaughtAndLoggedException { jobResult["SLOTS"] ? jobResult["SLOTS"] as Integer : null } - - ResourceSet usedResources = new ResourceSet(memory, null, nodes, runTime, null, queue, null) - jobInfo.setUsedResources(usedResources) - - ResourceSet askedResources = new ResourceSet(null, null, null, runLimit, null, queue, null) - jobInfo.setAskedResources(askedResources) - - jobInfo.setUser(jobResult["USER"] as String ?: null) - jobInfo.setDescription(jobResult["JOB_DESCRIPTION"] as String ?: null) - jobInfo.setProjectName(jobResult["PROJ_NAME"] as String ?: null) - jobInfo.setJobGroup(jobResult["JOB_GROUP"] as String ?: null) - jobInfo.setPriority(jobResult["JOB_PRIORITY"] as String ?: null) - jobInfo.setPidStr(jobResult["PIDS"] as String ? (jobResult["PIDS"] as String).split(",").toList() : null) - jobInfo.setJobState(parseJobState(jobResult["STAT"] as String)) - jobInfo.setExitCode(jobInfo.jobState == JobState.COMPLETED_SUCCESSFUL ? 0 : (jobResult["EXIT_CODE"] ? Integer.valueOf(jobResult["EXIT_CODE"] as String) : null)) - jobInfo.setSubmissionHost(jobResult["FROM_HOST"] as String ?: null) - jobInfo.setExecutionHosts(jobResult["EXEC_HOST"] as String ? (jobResult["EXEC_HOST"] as String).split(":").toList() : null) + Integer nodes = withCaughtAndLoggedException { jobResult["SLOTS"] as Integer } + + jobInfo.usedResources = new ResourceSet(memory, null, nodes, runTime, null, queue, null) + jobInfo.askedResources = new ResourceSet(null, null, null, runLimit, null, queue, null) + jobInfo.resourceReq = jobResult["EFFECTIVE_RESREQ"] + jobInfo.runTime = runTime + jobInfo.cpuTime = safelyParseColonSeparatedDuration(jobResult["CPU_USED"]) + + /** Status info */ + jobInfo.jobState = parseJobState(jobResult["STAT"]) + jobInfo.exitCode = jobInfo.jobState == JobState.COMPLETED_SUCCESSFUL ? 0 : (jobResult["EXIT_CODE"] as Integer) + jobInfo.pendReason = jobResult["PEND_REASON"] + + /** Directories and files */ + jobInfo.cwd = jobResult["SUB_CWD"] + jobInfo.execCwd = jobResult["EXEC_CWD"] + jobInfo.logFile = getBjobsFile(jobResult["OUTPUT_FILE"], jobID, "out") + jobInfo.errorLogFile = getBjobsFile(jobResult["ERROR_FILE"], jobID, "err") + jobInfo.inputFile = jobResult["INPUT_FILE"] ? new File(jobResult["INPUT_FILE"]) : null + jobInfo.execHome = jobResult["EXEC_HOME"] + + /** Timestamps */ + jobInfo.submitTime = safelyParseTime(jobResult["SUBMIT_TIME"]) + jobInfo.startTime = safelyParseTime(jobResult["START_TIME"]) + jobInfo.endTime = safelyParseTime(stripAwayStatusInfo(jobResult["FINISH_TIME"])) + + return jobInfo + } + + Duration safelyParseColonSeparatedDuration(String value) { withCaughtAndLoggedException { - jobInfo.setCpuTime(jobResult["CPU_USED"] ? parseColonSeparatedHHMMSSDuration(jobResult["CPU_USED"] as String) : null) + value ? parseColonSeparatedHHMMSSDuration(value) : null } - jobInfo.setRunTime(runTime) - jobInfo.setUserGroup(jobResult["USER_GROUP"] as String ?: null) - jobInfo.setCwd(jobResult["SUB_CWD"] as String ?: null) - jobInfo.setPendReason(jobResult["PEND_REASON"] as String ?: null) - jobInfo.setExecCwd(jobResult["EXEC_CWD"] as String ?: null) - jobInfo.setLogFile(getBjobsFile(jobResult["OUTPUT_FILE"] as String, jobID, "out")) - jobInfo.setErrorLogFile(getBjobsFile(jobResult["ERROR_FILE"] as String, jobID, "err")) - jobInfo.setInputFile(jobResult["INPUT_FILE"] ? new File(jobResult["INPUT_FILE"] as String) : null) - jobInfo.setResourceReq(jobResult["EFFECTIVE_RESREQ"] as String ?: null) - jobInfo.setExecHome(jobResult["EXEC_HOME"] as String ?: null) - - String submissionTime = jobResult["SUBMIT_TIME"] - String startTime = jobResult["START_TIME"] - String finishTime = jobResult["FINISH_TIME"] - - if (submissionTime) - withCaughtAndLoggedException { jobInfo.setSubmitTime(parseTime(submissionTime)) } - if (startTime) - withCaughtAndLoggedException { jobInfo.setStartTime(parseTime(startTime as String)) } - if (finishTime) - withCaughtAndLoggedException { jobInfo.setEndTime(parseTime(finishTime[0..-3])) } + } - return jobInfo + ZonedDateTime safelyParseTime(String time) { + if (time) + return withCaughtAndLoggedException { + return parseTime(time) + } + return null + } + + BufferValue safelyCastToBufferValue(String MAX_MEM) { + withCaughtAndLoggedException { + if (MAX_MEM) { + String bufferSize = MAX_MEM.find("([0-9]*[.])?[0-9]+") + String unit = MAX_MEM.find("[a-zA-Z]+") + BufferUnit bufferUnit = unit == "Gbytes" ? BufferUnit.g : BufferUnit.m + return new BufferValue(bufferSize, bufferUnit) + } + return null + } } private File getBjobsFile(String s, BEJobID jobID, String type) { diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy index e8635fe8..0d348035 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy @@ -331,7 +331,7 @@ class LSFRestJobManager extends AbstractLSFJobManager { */ private GenericJobInfo setJobInfoForJobDetails(NodeChild jobDetails) { - GenericJobInfo jobInfo = new GenericJobInfo(jobDetails.getProperty("jobName").toString(), new File(jobDetails.getProperty("command").toString()), new BEJobID(jobDetails.getProperty("jobId").toString()), null, null) + GenericJobInfo jobInfo = new GenericJobInfo(jobDetails.getProperty("jobName").toString(), jobDetails.getProperty("command").toString(), new BEJobID(jobDetails.getProperty("jobId").toString()), null, null) String queue = jobDetails.getProperty("queue").toString() BufferValue swap = jobDetails.getProperty("swap") ? withCaughtAndLoggedException { new BufferValue(jobDetails.getProperty("swap").toString(), BufferUnit.m) } : null diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParser.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParser.groovy index f0119075..88bfa9b0 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParser.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParser.groovy @@ -16,9 +16,7 @@ import de.dkfz.roddy.tools.ComplexLine import de.dkfz.roddy.tools.TimeUnit import groovy.transform.CompileStatic -import static de.dkfz.roddy.StringConstants.SPLIT_COLON -import static de.dkfz.roddy.StringConstants.SPLIT_COMMA -import static de.dkfz.roddy.StringConstants.SPLIT_EQUALS +import static de.dkfz.roddy.StringConstants.* /** * Used to convert commands from cli to e.g. GenericJobInfo @@ -60,11 +58,11 @@ class PBSCommandParser { if (!commandString.startsWith("qsub")) return // It is obviously not a PBS call - String[] splitted = line.splitBy(" ").findAll { it } + Collection splitted = line.splitBy(" ").findAll { it } script = splitted[-1] jobName = "not readable" - for (int i = 0; i < splitted.length - 1; i++) { + for (int i = 0; i < splitted.size() - 1; i++) { String option = splitted[i] if (!option.startsWith("-")) continue // It is not an option but a parameter or a text (e.g. qsub, script) @@ -131,7 +129,7 @@ class PBSCommandParser { } GenericJobInfo toGenericJobInfo() { - GenericJobInfo jInfo = new GenericJobInfo(jobName, new File(script), jobID, parameters, dependencies) + GenericJobInfo jInfo = new GenericJobInfo(jobName, script, jobID, parameters, dependencies) ResourceSet askedResources = new ResourceSet(null, memory ? new BufferValue(memory as Integer, bufferUnit) : null, cores ? cores as Integer : null, nodes ? nodes as Integer : null, walltime ? new TimeUnit(walltime) : null, null, null, null) diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy index 9f63825f..a5c1c07c 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy @@ -33,7 +33,7 @@ class LSFJobManagerSpec extends Specification { new File("src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/", file) } - void "queryJobInfo, bjobs JSON output with lists "() { + void "Test convertJobDetailsMapToGenericJobInfoObject"() { given: def parms = JobManagerOptions.create().build() @@ -44,17 +44,17 @@ class LSFJobManagerSpec extends Specification { List records = (List) parsedJson.getAt("RECORDS") when: - GenericJobInfo jobInfo = jm.convertJobDetailsMapToGenericJobInfoobject(records.get(0)) + GenericJobInfo jobInfo = jm.convertJobDetailsMapToGenericJobInfoObject(records.get(0)) then: jobInfo != null jobInfo.jobID.toString() == "22005" - jobInfo.tool == expectedTool + jobInfo.command == expectedCommand where: - resourceFile | expectedJobId | expectedTool - "queryExtendedJobStateByIdTest.json" | "22005" | new File("ls -l") - "queryExtendedJobStateByIdWithoutListsTest.json" | "22005" | new File("ls -l") + resourceFile | expectedJobId | expectedCommand + "queryExtendedJobStateByIdTest.json" | "22005" | "ls -l" + "queryExtendedJobStateByIdWithoutListsTest.json" | "22005" | "ls -l" "queryExtendedJobStateByIdEmptyTest.json" | "22005" | null } @@ -65,6 +65,8 @@ class LSFJobManagerSpec extends Specification { @CompileStatic String localDateTimeToLSFString(LocalDateTime date) { + // Important here is, that LSF puts " L" or other status codes at the end of some dates, e.g. FINISH_DATE + // Thus said, " L" does not apply for all dates reported by LSF! DateTimeFormatter.ofPattern('MMM ppd HH:mm').withLocale(Locale.ENGLISH).format(date) + " L" } @@ -109,7 +111,7 @@ class LSFJobManagerSpec extends Specification { manager.parseTime(zonedDateTimeToString(laterTime)).truncatedTo(ChronoUnit.MINUTES).equals(laterLastYear.truncatedTo(ChronoUnit.MINUTES)) } - void "test queryExtendedJobStateById"() { + void "test queryExtendedJobStateById with overdue date"() { given: JobManagerOptions parms = JobManagerOptions.create().build() def jsonFile = getResourceFile("queryExtendedJobStateByIdTest.json") @@ -121,6 +123,22 @@ class LSFJobManagerSpec extends Specification { when: Map result = manager.queryExtendedJobStateById([new BEJobID("22005")]) + then: + result.size() == 0 + } + + void "test queryExtendedJobStateById"() { + given: + JobManagerOptions parms = JobManagerOptions.create().setMaxTrackingTimeForFinishedJobs(Duration.ofDays(360000)).build() + def jsonFile = getResourceFile("queryExtendedJobStateByIdTest.json") + BEExecutionService testExecutionService = [ + execute: { String s -> new ExecutionResult(true, 0, jsonFile.readLines(), null) } + ] as BEExecutionService + LSFJobManager manager = new LSFJobManager(testExecutionService, parms) + + when: + Map result = manager.queryExtendedJobStateById([new BEJobID("22005")]) + then: result.size() == 1 GenericJobInfo jobInfo = result.get(new BEJobID("22005")) @@ -146,14 +164,15 @@ class LSFJobManagerSpec extends Specification { jobInfo.usedResources.swap == null jobInfo.jobName == "ls -l" - jobInfo.tool == new File("ls -l") + jobInfo.command == "ls -l" jobInfo.jobID == new BEJobID("22005") // The year-parsing/inferrence is checked in another test. Here just take the parsed value. - jobInfo.submitTime == ZonedDateTime.of(jobInfo.submitTime.year, 12, 28, 19, 56, 0, 0, ZoneId.systemDefault()) + ZonedDateTime testTime = ZonedDateTime.of(jobInfo.submitTime.year, 12, 28, 19, 56, 0, 0, ZoneId.systemDefault()) + jobInfo.submitTime == testTime jobInfo.eligibleTime == null - jobInfo.startTime == ZonedDateTime.of(jobInfo.submitTime.year, 12, 28, 19, 56, 0, 0, ZoneId.systemDefault()) - jobInfo.endTime == ZonedDateTime.of(jobInfo.submitTime.year, 12, 28, 19, 56, 0, 0, ZoneId.systemDefault()) + jobInfo.startTime == testTime + jobInfo.endTime == testTime jobInfo.executionHosts == ["exec-host", "exec-host"] jobInfo.submissionHost == "from-host" jobInfo.priority == null diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManagerSpec.groovy index e4ae5e8c..b8744b60 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManagerSpec.groovy @@ -126,7 +126,7 @@ Output retained on that host in: /var/spool/torque/undelivered/4564045.pbsserver jobInfo.usedResources.swap == null jobInfo.jobName == "r180328_183957634_pid_4_starAlignment" - jobInfo.tool == null + jobInfo.command == null jobInfo.jobID == new BEJobID("4499334") jobInfo.submitTime.isEqual ZonedDateTime.of(2018, 03, 28, 16, 39, 22, 0, ZoneOffset.UTC) jobInfo.eligibleTime.isEqual ZonedDateTime.of(2018, 03, 28, 16, 39, 22, 0, ZoneOffset.UTC) @@ -202,7 +202,7 @@ Output retained on that host in: /var/spool/torque/undelivered/4564045.pbsserver jobInfo.usedResources.swap == null jobInfo.jobName == "r180405_163953553_stds_snvJoinVcfFiles" - jobInfo.tool == null + jobInfo.command == null jobInfo.jobID == new BEJobID("4564045") jobInfo.submitTime.isEqual ZonedDateTime.of(2018, 4, 5, 14, 39, 18, 0, ZoneOffset.UTC) jobInfo.eligibleTime.isEqual ZonedDateTime.of(2018, 4, 5, 14, 39, 42, 0, ZoneOffset.UTC) From c5f19f7ecd688eaf586dd828785cf893dc629e75 Mon Sep 17 00:00:00 2001 From: heinold Date: Tue, 5 Feb 2019 17:03:06 +0100 Subject: [PATCH 3/3] Big rework of test structure, remove update daemon Update the dependency to RoddyToolLib to 2.1.1 // Library classes Renamed GenericJobInfo to ExtendedJobInfo Make both JobInfo classes use the @EqualsAndHashCode annotation to improve testability of code with (Extended)JobInfo objects. In ExtendedJobInfo: - Reordered the variable in attempt to group them better. - Renamed some of the variables to better match their function. - Make the class extend JobInfo and move jobID, endTime and jobState to the new class. Removed tracking and daemon fields from all classes. The daemon is now a conceptual idea which can be implemented by software using BE. In BatchEuphoriaJobManager: - Unify get(Extended)JobInfo method layout. - Implemented most methods in the class and made them final, add tests for them. In ClusterBasedJobManager: - Made withCaughtAndLoggedException public. - Added the toJobID(), safelyParseColonSeparatedDuration(), safelyParseTime() and the abstract parseTime() methods. In GridEngineBasedJobManager: - Add parseTime() method - Rename queryJobStates() to queryJobInfo() - Extract the assembleJobInfoQueryCommand() method from queryJobInfo() The method will assemble "qstat [-u] [id] [id] [...]" by default and is overridden in SGEJobManager. - Removed processQstatOutputFromPlainText(), we only parse from xml, so this was redundant and actually never used. - Add safe cast and convert methods for BufferValue, Integer, Duration TimeUnit and others. They will safely convert/case a value from a String. - Moved the processQstatOutputFromXML() method to PBSJobManager, SGE has a completely different implementation! In SGEJobManager: - Override assembleJobInfoQueryCommand() - Add convertJobInfoMapToExtendedMap() - Override queryExtendedJobInfo(), this works in a multi step process, first it will call query(All)JobInfo() to get basic information about jobs. Why? Because SGE provides you with different output depending whether active jobs can be found or not. Detailed explanations are in the method. After the first step, extended information are queried and processed. - Add parseGPathJobInfoElement(), which will parse a job/element entry in extended qstat xml output - Add extractAndParseTime() and extractStatisticsValue() In PBSJobManager: - Override queryExtendedJobInfo() - Add processQstatOutputFromXMLNodeChild(), which will parse a job entry in the extended qstat xml output. Actually it was moved here from GridEngineBasedJobManager. Also the contents are reordered to match the corresponding methods in LSFJobManager and SGEJobManager In LSFJobManager: - Reduce the returned JSON result in queryJobInfo() to the most essential fields: jobid, job_name, stat and finish_time. - Improve the parseTime method to cover several versions of possible LSF datetime reports. Detailed explanations are in the method comments - Improve stripAwayStatusInfo() - Remove queryExtendedJobStateById(), its in BEJobManager - Override queryJobInfo() and queryExtendedJobInfo(), remove queryJobStates - Remove filterJobMapByAge(). This is a method for a daemon and not covered by BatchEuphoria anymore. - Rework convertJobDetailsMapToGenericJobInfoObject() further - Remove safelyParseColonSeparatedDuration() and safelyParseTime(), they are in a super class. In LSFRestJobManager: - Override queryExtendedJobInfo() - Just saw, that a lot of work is still necessary in this class. But please review the rest first. // Test classes and files Add a lot of resource files, they are mostly extracted from tests and this is done to make the test layout more structured and the tests more readable. To load a resource file, utilize the new BETestBaseSpec class, which has the getResourceFile() method and a test for it. Created the JobManagerImplementationBaseSpec class, which is now a base specification for all job manager specific Spock specs: - The method createJobManagerWithModifiedExecutionService() can be used to create a job manager, which will use the passed resource for queryJobInfo() or queryExtendedJobInfo() test cases. It will create a job manager of type T! T needs to be set, when the Spec is extended - The method createJobManager() will create a very basic job manager of type T - Also there is a list of possible methods / test features, which are meant to bring some basic identical structure to all job manager tests - The class is tested in JobManagerImplementationBaseSpecSpec Create the BatchEuphoriaJobManagerSpec: - Make it implement JobManagerImplementationBaseSpec - Add tests for the different final query methods in BatchEuphoriaJobManager In GridEngineBasedJobManagerSpec, extend test for getExecutionHosts() In ClusterJobManagerSpec: - Add tests for parseColonSeparatedHHMMSSDuration() - Fix prepareClassLogger() - Rename tests Add the SGEJobManagerSpec: - Has stubs for most of the tests in JobManagerImplementationBaseSpec - Has several tests for query(Extended)JobInfo() which try to tackle the slightly special behaviour of queryExtendedJobInfo() In PBSJobManagerSpec: - Remove all the XML variables and move content to resource files. - Add stubs for tests in JobManagerImplementationBaseSpec - Make tests use the new equals method of (Extended)JobInfo. - Improved layout and readability of some tests. - Move the parseColonSeparatedHHMMSSDuration() tests to ClusterJobManagerSpec In PBSCommandParserTest: - Reduce code by utilizing the TestExecutionService for the PBSJobManager. In LSFJobManagerSpec: - Remove the getResourceFile() method. Its in a super class. - Make the class extend JobManagerImplementationBaseSpec and create test stubs - Spockify the parseTime() test and cover all cases and also malformatted strings - Make the query(E)JI() tests use the new equals feature of JobInfo - Remove the filterJobMapByAge test Create a test class LSFRestJobManagerSpec. It is not implemented yet! --- build.gradle | 2 +- .../de/dkfz/roddy/config/ResourceSet.groovy | 2 + .../jobs/BatchEuphoriaJobManager.groovy | 293 ++++---- .../execution/jobs/ExtendedJobInfo.groovy | 235 +++++++ .../execution/jobs/GenericJobInfo.groovy | 91 --- .../dkfz/roddy/execution/jobs/JobInfo.groovy | 45 ++ .../execution/jobs/JobManagerOptions.groovy | 10 +- .../jobs/cluster/ClusterJobManager.groovy | 36 +- .../cluster/GridEngineBasedJobManager.groovy | 277 +++----- .../jobs/cluster/lsf/LSFCommandParser.groovy | 10 +- .../jobs/cluster/lsf/LSFJobManager.groovy | 238 ++++--- .../cluster/lsf/rest/LSFRestJobManager.groovy | 48 +- .../jobs/cluster/pbs/PBSCommandParser.groovy | 10 +- .../jobs/cluster/pbs/PBSJobManager.groovy | 110 ++- .../jobs/cluster/sge/SGEJobManager.groovy | 172 ++++- .../jobs/cluster/slurm/SlurmJobManager.groovy | 20 +- ...irectSynchronousExecutionJobManager.groovy | 20 +- .../roddy/batcheuphoria/BETestBaseSpec.groovy | 32 + .../jobs/JobManagerOptionsTest.groovy | 1 - .../execution/jobs/BEIntegrationTest.groovy | 10 +- .../jobs/BatchEuphoriaJobManagerSpec.groovy | 174 +++++ .../execution/jobs/ExtendedJobInfoTest.groovy | 14 + .../jobs/SubmissionCommandTest.groovy | 15 +- .../jobs/cluster/ClusterJobManagerSpec.groovy | 47 +- .../GridEngineBaseJobManagerTest.groovy | 200 ------ .../GridEngineBasedJobManagerSpec.groovy | 23 +- .../cluster/GridEngineQstatParserTest.groovy | 429 ------------ .../JobManagerImplementationBaseSpec.groovy | 123 ++++ ...obManagerImplementationBaseSpecSpec.groovy | 150 +++++ .../jobs/cluster/lsf/LSFJobManagerSpec.groovy | 394 ++++++----- .../lsf/rest/LSFRestJobManagerSpec.groovy | 121 ++++ .../cluster/pbs/PBSCommandParserTest.groovy | 55 +- .../jobs/cluster/pbs/PBSJobManagerSpec.groovy | 475 ++++++------- .../jobs/cluster/sge/SGEJobManagerSpec.groovy | 212 ++++++ .../jobs/cluster/sge/SGEJobManagerTest.java | 36 - .../batcheuphoria/getResourceFileTest.txt | 1 + ...anagerWithModifiedExecutionServiceTest.txt | 3 + .../lsf/queryExtendedJobStateByIdTest.json | 2 +- ...yExtendedJobStateByIdWithoutListsTest.json | 12 +- .../cluster/lsf/queryJobStateByIdTest.json | 12 + ...anagerWithModifiedExecutionServiceTest.txt | 3 + .../processExtendedQstatXMLOutputTests.xml | 52 ++ ...sExtendedQstatXMLOutputWithFinishedJob.xml | 65 ++ ...essExtendedQstatXMLOutputWithQueuedJob.xml | 40 ++ .../pbs/queryExtendedJobInfoWithEmptyXML.xml | 5 + ...endedJobInfoWithPlaceholderReplacement.xml | 7 + ...anagerWithModifiedExecutionServiceTest.txt | 3 + .../queryExtendedJobStatesWithFinishedJob.xml | 317 +++++++++ ...xtendedJobStatesWithJobWithPipedScript.xml | 111 +++ .../queryExtendedJobStatesWithQueuedJobs.xml | 632 ++++++++++++++++++ .../queryExtendedJobStatesWithUnknownJobs.xml | 9 + .../jobs/cluster/sge/simpleQStatOutput.txt | 6 + 52 files changed, 3583 insertions(+), 1827 deletions(-) create mode 100644 src/main/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfo.groovy delete mode 100644 src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy create mode 100644 src/main/groovy/de/dkfz/roddy/execution/jobs/JobInfo.groovy create mode 100644 src/test/groovy/de/dkfz/roddy/batcheuphoria/BETestBaseSpec.groovy create mode 100644 src/test/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManagerSpec.groovy create mode 100644 src/test/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfoTest.groovy delete mode 100644 src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBaseJobManagerTest.groovy delete mode 100644 src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineQstatParserTest.groovy create mode 100644 src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpec.groovy create mode 100644 src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpecSpec.groovy create mode 100644 src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManagerSpec.groovy create mode 100644 src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerSpec.groovy delete mode 100644 src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerTest.java create mode 100644 src/test/resources/de/dkfz/roddy/batcheuphoria/getResourceFileTest.txt create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/createJobManagerWithModifiedExecutionServiceTest.txt create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryJobStateByIdTest.json create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/createJobManagerWithModifiedExecutionServiceTest.txt create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputTests.xml create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithFinishedJob.xml create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithQueuedJob.xml create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithEmptyXML.xml create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithPlaceholderReplacement.xml create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/createJobManagerWithModifiedExecutionServiceTest.txt create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithFinishedJob.xml create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithJobWithPipedScript.xml create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithQueuedJobs.xml create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithUnknownJobs.xml create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/simpleQStatOutput.txt diff --git a/build.gradle b/build.gradle index 15a643fc..e237802e 100644 --- a/build.gradle +++ b/build.gradle @@ -63,7 +63,7 @@ dependencies { compile group: 'commons-cli', name: 'commons-cli', version: '1.2' compile group: 'org.apache.commons', name: 'commons-text', version: '1.1' compile 'com.google.guava:guava:23.0' - compile 'com.github.theroddywms:RoddyToolLib:2.1.0' + compile 'com.github.theroddywms:RoddyToolLib:2.1.1' } task writePom { diff --git a/src/main/groovy/de/dkfz/roddy/config/ResourceSet.groovy b/src/main/groovy/de/dkfz/roddy/config/ResourceSet.groovy index d23b18c3..53d3c354 100644 --- a/src/main/groovy/de/dkfz/roddy/config/ResourceSet.groovy +++ b/src/main/groovy/de/dkfz/roddy/config/ResourceSet.groovy @@ -4,10 +4,12 @@ import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.RoddyConversionHelperMethods import de.dkfz.roddy.tools.TimeUnit import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode import java.time.Duration @CompileStatic +@EqualsAndHashCode class ResourceSet { private final String queue private ResourceSetSize size diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy index 3265e5b3..ed197c65 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy @@ -14,8 +14,6 @@ import groovy.transform.CompileStatic import org.slf4j.Logger import org.slf4j.LoggerFactory -import java.time.Duration -import java.time.LocalDateTime import java.util.concurrent.TimeoutException /** @@ -29,14 +27,12 @@ abstract class BatchEuphoriaJobManager { final static Logger log = LoggerFactory.getLogger(BatchEuphoriaJobManager) - protected final BEExecutionService executionService + final BEExecutionService executionService protected boolean isTrackingOfUserJobsEnabled protected boolean queryOnlyStartedJobs - protected final Duration maxTrackingTimeForFinishedJobs - protected String userIDForQueries private String userEmail @@ -47,30 +43,12 @@ abstract class BatchEuphoriaJobManager { private String userAccount - private final Map activeJobs = [:] - - private Thread updateDaemonThread - - /** - * Set this to true to tell the job manager, that an existing update daemon shall be closed, e.g. because - * the application is in the process to exit. - */ - protected boolean updateDaemonShallBeClosed /** * Set this to true, if you do not want to allow any further job submission. */ protected boolean forbidFurtherJobSubmission - private Map cachedStates = [:] - private final Object cacheStatesLock = new Object() - private LocalDateTime lastCacheUpdate - private Duration cacheUpdateInterval - - public boolean surveilledJobsHadErrors = false - - private final List updateDaemonListeners = [] - boolean requestMemoryIsEnabled boolean requestWalltimeIsEnabled boolean requestQueueIsEnabled @@ -85,14 +63,12 @@ abstract class BatchEuphoriaJobManager { this.isTrackingOfUserJobsEnabled = parms.userIdForJobQueries as boolean this.queryOnlyStartedJobs = parms.trackOnlyStartedJobs - this.maxTrackingTimeForFinishedJobs = parms.maxTrackingTimeForFinishedJobs this.userIDForQueries = parms.userIdForJobQueries this.userEmail = parms.userEmail this.userGroup = parms.userGroup this.userAccount = parms.userAccount this.userMask = parms.userMask - this.cacheUpdateInterval = parms.updateInterval this.requestMemoryIsEnabled = parms.requestMemoryIsEnabled this.requestWalltimeIsEnabled = parms.requestWalltimeIsEnabled @@ -102,13 +78,6 @@ abstract class BatchEuphoriaJobManager { this.passEnvironment = parms.passEnvironment this.holdJobsIsEnabled = Optional.ofNullable(parms.holdJobIsEnabled).orElse(getDefaultForHoldJobsEnabled()) - if (parms.createDaemon) { - createUpdateDaemonThread() - } - } - - void setQueryJobStatesFilter() { - } /** @@ -129,6 +98,10 @@ abstract class BatchEuphoriaJobManager { return job.runResult } + @Deprecated + void addToListOfStartedJobs(BEJob job) { + } + /** * Resume given job * @param job @@ -146,8 +119,13 @@ abstract class BatchEuphoriaJobManager { } } + static final assertListIsValid(List list) { + assert list && list.every { it != null } + } + /** * Try to abort a list of jobs + * * @param jobs */ void killJobs(List jobs) { @@ -163,6 +141,16 @@ abstract class BatchEuphoriaJobManager { } } + final JobInfo queryJobInfoByJob(BEJob job) { + assert job + queryJobInfoByID(job.jobID) + } + + final JobInfo queryJobInfoByID(BEJobID jobID) { + assert jobID + return queryJobInfoByID([jobID])[jobID] + } + /** * Queries the status of all jobs in the list. * @@ -173,36 +161,66 @@ abstract class BatchEuphoriaJobManager { * @param jobs * @return */ - Map queryJobStatus(List jobs, boolean forceUpdate = false) { - Map result = queryJobStatesUsingCache(jobs*.jobID, forceUpdate) - return jobs.collectEntries { BEJob job -> - [job, job.jobState == JobState.ABORTED ? JobState.ABORTED : - result[job.jobID] ?: JobState.UNKNOWN] + final Map queryJobInfoByJob(List jobs) { + assertListIsValid(jobs) + + Map jobsByID = jobs.collectEntries { + def job = it + def id = it.jobID + [id, job] } + + return queryJobInfoByID(jobs*.jobID) + .collectEntries { + BEJobID id, JobInfo info -> [jobsByID[id], info] + } as Map } /** - * Queries the status of all jobs in the list. + * Query a list of job states by their id. + * If the status of a job could not be determined, the query will return UNKNOWN as the state for this job. * - * Every job ID in the list is supposed to have an entry in the result map. If - * the manager cannot retrieve info about the job, the result will be UNKNOWN - * for this particular job. + * @param jobIDs + * @return A map of all job states accessible by their id. + */ + final Map queryJobInfoByID(List jobIDs) { + assertListIsValid(jobIDs) + + Map result = queryJobInfo(jobIDs) ?: [:] as Map + + // Collect the result and fill in empty entries with UNKNOWN + return jobIDs.collectEntries { + BEJobID jobID -> + [jobID, result[jobID] ?: new JobInfo(jobID)] + } + } + + /** + * Query the states of all jobs available limited by the settings provided in the constructor. * - * @param jobIds * @return */ - Map queryJobStatusById(List jobIds, boolean forceUpdate = false) { - Map result = queryJobStatesUsingCache(jobIds, forceUpdate) - return jobIds.collectEntries { BEJobID jobId -> [jobId, result[jobId] ?: JobState.UNKNOWN] } + final Map queryAllJobInfo() { + queryJobInfo(null) } /** - * Queries the status of all jobs. + * Needs to be overriden by JobManager implementations. + * jobIDs may be null or [] which means, that the query is not for specific jobs but for all available (except for + * the filter settings of the JobManager) jobs. * - * @return + * @param jobIDs A list of IDs OR null/[] */ - Map queryJobStatusAll(boolean forceUpdate = false) { - return queryJobStatesUsingCache(null, forceUpdate) + abstract Map queryJobInfo(List jobIDs) + + final ExtendedJobInfo queryExtendedJobInfoByJob(BEJob job) { + assert job + return queryExtendedJobInfoByID(job.jobID) + } + + final ExtendedJobInfo queryExtendedJobInfoByID(BEJobID id) { + assert id + return queryExtendedJobInfoByID([id])[id] } /** @@ -211,15 +229,17 @@ abstract class BatchEuphoriaJobManager { * - The used cores * - The used walltime * - * @param jobs - * @return + * If the jobid cannot be found, an empty extend job info object will be returned with the jobstate set to UNKNOWN */ - Map queryExtendedJobState(List jobs) { + final Map queryExtendedJobInfoByJob(List jobs) { + assertListIsValid(jobs) - Map queriedExtendedStates = queryExtendedJobStateById(jobs.collect { it.getJobID() }) - return (Map) queriedExtendedStates.collectEntries { - Map.Entry it -> [jobs.find { BEJob temp -> temp.getJobID() == it.key }, (GenericJobInfo) it.value] - } + Map jobsByID = jobs.collectEntries { [it.jobID, it] } + + return queryExtendedJobInfoByID(jobs*.jobID) + .collectEntries { + BEJobID id, JobInfo info -> [jobsByID[id], info] + } as Map } /** @@ -228,19 +248,34 @@ abstract class BatchEuphoriaJobManager { * - The used cores * - The used walltime * - * @param jobIds - * @return + * If the jobid cannot be found, an empty extend job info object will be returned with the jobstate set to UNKNOWN */ - abstract Map queryExtendedJobStateById(List jobIds) - - void addToListOfStartedJobs(BEJob job) { - if (updateDaemonThread) { - synchronized (activeJobs) { - activeJobs.put(job.getJobID(), job) - } - } + final Map queryExtendedJobInfoByID(List jobIDs) { + assertListIsValid(jobIDs) + + Map queriedExtendedStates = queryExtendedJobInfo(jobIDs) ?: [:] as Map + + jobIDs.collectEntries { + BEJobID jobID -> + [ + jobID, + queriedExtendedStates[jobID] ?: new ExtendedJobInfo(jobID) + ] + } as Map } + final Map queryAllExtendedJobInfo() { return queryExtendedJobInfo(null) } + + /** + * Needs to be overriden by JobManager implementations. + * jobIDs may be null or [] which means, that the query is not for specific jobs but for all available (except for + * the filter settings of the JobManager) jobs. + * + * @param jobIDs A list of IDs OR null/[] + * @return a list of jobs + */ + abstract Map queryExtendedJobInfo(List jobIDs) + abstract String getJobIdVariable() abstract String getJobNameVariable() @@ -285,9 +320,9 @@ abstract class BatchEuphoriaJobManager { abstract String getSubmissionCommand() - abstract String getQueryJobStatesCommand() + abstract String getQueryCommandForJobInfo() - abstract String getExtendedQueryJobStatesCommand() + abstract String getQueryCommandForExtendedJobInfo() ProcessingParameters convertResourceSet(BEJob job) { return convertResourceSet(job, job.resourceSet) @@ -302,7 +337,7 @@ abstract class BatchEuphoriaJobManager { * @param commandString * @return */ - abstract GenericJobInfo parseGenericJobInfo(String command) + abstract ExtendedJobInfo parseGenericJobInfo(String command) protected List collectJobIDsFromJobs(List jobs) { BEJob.jobsWithUniqueValidJobId(jobs).collect { it.runResult.getJobID() } @@ -322,9 +357,6 @@ abstract class BatchEuphoriaJobManager { job.resetJobID(jobID) jobResult = new BEJobResult(command, job, res, job.tool, job.parameters, job.parentJobs as List) job.setRunResult(jobResult) - synchronized (cacheStatesLock) { - cachedStates.put(jobID, isHoldJobsEnabled() ? JobState.HOLD : JobState.QUEUED) - } } else { def job = command.getJob() jobResult = new BEJobResult(command, job, res, job.tool, job.parameters, job.parentJobs as List) @@ -340,100 +372,59 @@ abstract class BatchEuphoriaJobManager { abstract protected JobState parseJobState(String stateString) - abstract protected Map queryJobStates(List jobIDs) - abstract protected ExecutionResult executeStartHeldJobs(List jobIDs) abstract protected ExecutionResult executeKillJobs(List jobIDs) - protected void createUpdateDaemonThread() { - updateDaemonThread = Thread.startDaemon("Job state update daemon.", { - while (!updateDaemonShallBeClosed) { - updateActiveJobList() - - waitForUpdateIntervalDuration() - } - }) - } - final void addUpdateDaemonListener(UpdateDaemonListener listener) { - synchronized (updateDaemonListeners) { - updateDaemonListeners << listener - } + @Deprecated + Map queryJobStatus(List jobs) { + queryJobInfoByJob(jobs).collectEntries { BEJob job, JobInfo ji -> [job, ji.jobState] } as Map } - boolean isDaemonAlive() { - return updateDaemonThread != null && updateDaemonThread.isAlive() - } - - void waitForUpdateIntervalDuration() { - long duration = Math.max(cacheUpdateInterval.toMillis(), 10 * 1000) - // Sleep one second until the duration is reached. This allows the daemon to finish faster, when it shall stop - // (updateDaemonShallBeClosed == true) - for (long timer = duration; timer > 0 && !updateDaemonShallBeClosed; timer -= 1000) - Thread.sleep(1000) - } - - void stopUpdateDaemon() { - updateDaemonShallBeClosed = true - updateDaemonThread?.join() + /** + * Queries the status of all jobs in the list. + * + * Every job ID in the list is supposed to have an entry in the result map. If + * the manager cannot retrieve info about the job, the result will be UNKNOWN + * for this particular job. + * + * @param jobIds + * @return + */ + @Deprecated + Map queryJobStatusById(List jobIds) { + queryJobInfoByID(jobIds).collectEntries { BEJobID id, JobInfo ji -> [id, ji.jobState] } as Map } - private void updateActiveJobList() { - List listOfRemovableJobs = [] - synchronized (activeJobs) { - Map states = queryJobStatesUsingCache(activeJobs.keySet() as List, true) - - for (BEJobID id : activeJobs.keySet()) { - JobState jobState = states.get(id) - BEJob job = activeJobs.get(id) - - job.setJobState(jobState) - if (!jobState.isPlannedOrRunning()) { - synchronized (updateDaemonListeners) { - updateDaemonListeners.each { it.jobEnded(job, jobState) } - } - if (!jobState.successful) { - surveilledJobsHadErrors = true - } - listOfRemovableJobs << id - } - } - listOfRemovableJobs.each { activeJobs.remove(it) } - } + /** + * Queries the status of all jobs. + * + * @return + */ + @Deprecated + Map queryJobStatusAll(boolean forceUpdate = false) { + return queryAllJobInfo().collectEntries { BEJobID id, JobInfo ji -> [id, ji.jobState] } as Map } - private Map queryJobStatesUsingCache(List jobIDs, boolean forceUpdate) { - if (forceUpdate || lastCacheUpdate == null || cacheUpdateInterval == Duration.ZERO || - Duration.between(lastCacheUpdate, LocalDateTime.now()) > cacheUpdateInterval) { - synchronized (cacheStatesLock) { - cachedStates = queryJobStates(jobIDs) - } - lastCacheUpdate = LocalDateTime.now() - } - return new HashMap(cachedStates) + @Deprecated + Map queryExtendedJobState(List jobs) { + queryExtendedJobInfoByJob(jobs) } /** - * The method will wait until all started jobs are finished (with or without errors). - * - * Note, that the method does not allow further job submission! As soon, as you call it, you cannot submit jobs! + * Will be used to gather extended information about a job like: + * - The used memory + * - The used cores + * - The used walltime * - * @return true, if there were NO errors, false, if there were any. + * @param jobIds + * @return */ - boolean waitForJobsToFinish() { - if (!updateDaemonThread) { - throw new BEException("The job manager must be created with JobManagerOption.createDaemon set to true to make waitForJobsToFinish() work.") - } - forbidFurtherJobSubmission = true - while (!updateDaemonShallBeClosed) { - synchronized (activeJobs) { - if (activeJobs.isEmpty()) { - break - } - } - waitForUpdateIntervalDuration() - } - return !surveilledJobsHadErrors + @Deprecated + Map queryExtendedJobStateById(List jobIds) { + return queryExtendedJobInfo(jobIds) } + + } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfo.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfo.groovy new file mode 100644 index 00000000..0762d523 --- /dev/null +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfo.groovy @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2017 eilslabs. + * + * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). + */ + +package de.dkfz.roddy.execution.jobs + +import de.dkfz.roddy.config.ResourceSet +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString + +import java.time.Duration +import java.time.ZonedDateTime + +/** + * Stores extended information like e.g. statistical data about a job + * + * Note, that the amount of gathered information depends on the target system and can vary with different job states. + * E.g. the fields + * + * This class is a value class. Normally, we'd use final fields for all values or some kind of + * builder interface to create instances of it. But due to the sheer amount of its fields, + * we will refrain from both. Please make sure, that you do net mess things up. + */ +@CompileStatic +@ToString(includeNames = true) +@EqualsAndHashCode(callSuper = true) +class ExtendedJobInfo extends JobInfo { + + // Common information + + String jobName + + String user + + String userGroup + + /** + * Umask with which files and directories are created + */ + String umask + + String execUserName + + String description + + String projectName + + String jobGroup + + /** + * The executed command + */ + String command + + /** + * Job parameters during submission + */ + Map parameters + + /** + * IDs of the jobs parent jobs + */ + List parentJobIDs + + String priority + + /** + * List of process ids within the job + */ + List processesInJob + + /** + * Currently active process group ID in a job. + */ + String processGroupID + + String submissionHost + + String account + + /** + * Submission server + */ + String server + + // Resources + + String rawHostQueryString + + List executionHosts + + /** + * Requested resources like e.g. walltime, max memory + */ + ResourceSet requestedResources + + /** + * Used resources like e.g. walltime, max memory + */ + ResourceSet usedResources + + /** + * resource requirements + */ + String rawResourceRequest + + /** + * Time in seconds that the job has been in the run state + */ + Duration runTime + + /** + * Cumulative total CPU time in seconds of all processes in a job + */ + Duration cpuTime + + // Status + + /** + * (UNIX) exit status of the job + */ + Integer exitCode + + /** + * Reason, why a job is on hold or suspended + */ + String pendReason + + /** + * How often was the job started + */ + Integer startCount + + // Directories + + /** + * Current working directory + */ + String cwd + + /** + * Executed current working directory + */ + String execCwd + + String execHome + + File logFile + + File errorLogFile + + File inputFile + + // Timestamps and timing info + + /** + * user time used + */ + String userTime + + /** + * system time used + */ + String systemTime + + /** + * The date-time the job entered the queue. + */ + ZonedDateTime submitTime + + /** + * The date-time the job became eligible to run when all conditions like job dependencies are met, i.e. in a queued state while residing in an execution queue. + */ + ZonedDateTime eligibleTime + + /** + * The date-time the job was started. + */ + ZonedDateTime startTime + + /** + * Suspended by its owner or the LSF administrator after being dispatched + */ + Duration timeUserSuspState + + /** + * Waiting in a queue for scheduling and dispatch + */ + Duration timePendState + + /** + * Suspended by its owner or the LSF administrator while in PEND state + */ + Duration timePendSuspState + + /** + * Suspended by the job system after being dispatched + */ + Duration timeSystemSuspState + + Duration timeUnknownState + + ZonedDateTime timeOfCalculation + + // Whatever... + + /** + * Guess what... don't know + */ + String otherSettings + + + ExtendedJobInfo(BEJobID jobID) { + super(jobID) + } + + ExtendedJobInfo(BEJobID jobID, JobState jobState) { + super(jobID) + this.jobState = jobState + } + + @Deprecated + ExtendedJobInfo(String jobName, String command, BEJobID jobID, Map parameters, List parentJobIDs) { + super(jobID) + this.jobName = jobName + this.command = command + this.jobID = jobID + this.parameters = parameters + this.parentJobIDs = parentJobIDs + } + +} diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy deleted file mode 100644 index 2c629835..00000000 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2017 eilslabs. - * - * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). - */ - -package de.dkfz.roddy.execution.jobs - -import de.dkfz.roddy.config.ResourceSet -import groovy.transform.CompileStatic -import groovy.transform.ToString - -import java.time.Duration -import java.time.ZonedDateTime - -/** - * Created by michael on 06.02.15. - */ -@CompileStatic -@ToString(includeNames = true) -class GenericJobInfo { - - ResourceSet askedResources - ResourceSet usedResources - String jobName - String command - BEJobID jobID - - /** The date-time the job entered the queue. */ - ZonedDateTime submitTime - /** The date-time the job became eligible to run when all conditions like job dependencies are met, i.e. in a queued state while residing in an execution queue. */ - ZonedDateTime eligibleTime - /** The date-time the job was started. */ - ZonedDateTime startTime - /** The date-time the job was completed. */ - ZonedDateTime endTime - - List executionHosts - String submissionHost - String priority - - File logFile - File errorLogFile - File inputFile - - String user - String userGroup - String resourceReq // resource requirements - Integer startCount - - String account - String server - String umask - - Map parameters - List parentJobIDs - String otherSettings - JobState jobState - String userTime //user time used - String systemTime //system time used - String pendReason - String execHome - String execUserName - List pidStr - String pgidStr // Currently active process group ID in a job. - Integer exitCode // UNIX exit status of the job - String jobGroup - String description - String execCwd //Executed current working directory - String askedHostsStr - String cwd //Current working directory - String projectName - - Duration cpuTime //Cumulative total CPU time in seconds of all processes in a job - Duration runTime //Time in seconds that the job has been in the run state - Duration timeUserSuspState //Suspended by its owner or the LSF administrator after being dispatched - Duration timePendState //Waiting in a queue for scheduling and dispatch - Duration timePendSuspState // Suspended by its owner or the LSF administrator while in PEND state - Duration timeSystemSuspState //Suspended by the LSF system after being dispatched - Duration timeUnknownState - ZonedDateTime timeOfCalculation - - - GenericJobInfo(String jobName, String command, BEJobID jobID, Map parameters, List parentJobIDs) { - this.jobName = jobName - this.command = command - this.jobID = jobID - this.parameters = parameters - this.parentJobIDs = parentJobIDs - } -} diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/JobInfo.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/JobInfo.groovy new file mode 100644 index 00000000..4c43ebf3 --- /dev/null +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/JobInfo.groovy @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). + * + * Distributed under the MIT License (license terms are at https://www.github.com/TheRoddyWMS/BatchEuphoria/LICENSE.txt). + */ + +package de.dkfz.roddy.execution.jobs + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode + +import java.time.ZonedDateTime + +/** + * The class contains the bare minimum of information about a job gathered by + * BatchEuphoriaJobManager.queryJobInfo() + * Note that all fields except jobID can be undefined / null! + * + * There are three different settings for the content of this class: + * - A job was not found by the manager, the state will be set to UNKNOWN, with no endTime + * - A job was found but is still running, the state will be set accordingly, with no endTime + * - A job was found and is finished, the state will be set accordingly with the endTime set + * + * This class is a value class. Normally, we'd use final fields for all values or some kind of + * builder interface to create instances of it. But due to the sheer size of ExtendedJobInfo, + * we will refrain from both. Please make sure, that you do net mess things up. + */ +@CompileStatic +@EqualsAndHashCode +class JobInfo { + + BEJobID jobID + + /** + * The date-time the job was completed. + * Can be null, e.g. if the batch system cannot track the job anymore. + * */ + ZonedDateTime endTime + + JobState jobState = JobState.UNKNOWN + + JobInfo(BEJobID jobID) { + this.jobID = jobID + } +} diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy index 72de7890..9b6ddec5 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy @@ -17,14 +17,10 @@ import java.time.ZoneId @CompileStatic class JobManagerOptions { - boolean createDaemon - Duration updateInterval String userIdForJobQueries - Duration maxTrackingTimeForFinishedJobs - boolean trackOnlyStartedJobs String userGroup @@ -91,14 +87,12 @@ class JobManagerOptionsBuilder { JobManagerOptionsBuilder() { trackOnlyStartedJobs = false updateInterval = Duration.ofMinutes(5) - maxTrackingTimeForFinishedJobs = Duration.ofDays(14) - createDaemon = false requestMemoryIsEnabled = true requestWalltimeIsEnabled = true requestQueueIsEnabled = true requestCoresIsEnabled = true - requestStorageIsEnabled = false // Defaults to false, not supported now. - passEnvironment = false // Setting this to true should be a conscious decision. Therefore the default 'false'. + requestStorageIsEnabled = false // Defaults to false, not supported now. + passEnvironment = false // Setting this to true should be a conscious decision. Therefore the default 'false'. additionalOptions = [:] timeZoneId = ZoneId.systemDefault() } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManager.groovy index 5a26c61a..87b22b47 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManager.groovy @@ -16,6 +16,7 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import java.time.Duration +import java.time.ZonedDateTime /** * A class for processing backends running on a cluster. @@ -29,7 +30,7 @@ abstract class ClusterJobManager extends BatchEuphoriaJobMana super(executionService, parms) } - protected static T withCaughtAndLoggedException(final Closure closure) { + static T withCaughtAndLoggedException(final Closure closure) { try { return closure.call() } catch (Exception e) { @@ -46,7 +47,36 @@ abstract class ClusterJobManager extends BatchEuphoriaJobMana return null } - protected static Duration parseColonSeparatedHHMMSSDuration(String str) { + static BEJobID toJobID(String jobIdRaw) { + BEJobID jobID + try { + jobID = new BEJobID(jobIdRaw) + } catch (Exception exp) { + throw new BEException("Job ID '${jobIdRaw}' could not be transformed to BEJobID ") + } + jobID + } + + static Duration safelyParseColonSeparatedDuration(Object value) { + String _value = value as String + withCaughtAndLoggedException { + return _value ? parseColonSeparatedHHMMSSDuration(_value) : null + } + } + + ZonedDateTime safelyParseTime(Object time) { + String _time = time as String + if (time) + return withCaughtAndLoggedException { + return parseTime(_time) + } + return null + } + + abstract ZonedDateTime parseTime(String time) + + + static Duration parseColonSeparatedHHMMSSDuration(String str) { String[] hhmmss = str.split(":") if (hhmmss.size() != 3) { throw new BEException("Duration string is not of the format HH+:MM:SS: '${str}'") @@ -99,4 +129,6 @@ abstract class ClusterJobManager extends BatchEuphoriaJobMana abstract void createMemoryParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) abstract void createStorageParameters(LinkedHashMultimap parameters, ResourceSet resourceSet) + + } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManager.groovy index 0f2914fb..70d8e4e0 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManager.groovy @@ -1,8 +1,13 @@ +/* + * Copyright (c) 2019 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). + * + * Distributed under the MIT License (license terms are at https://www.github.com/TheRoddyWMS/BatchEuphoria/LICENSE.txt). + */ + package de.dkfz.roddy.execution.jobs.cluster import com.google.common.collect.LinkedHashMultimap import de.dkfz.roddy.BEException -import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.execution.io.ExecutionResult import de.dkfz.roddy.execution.jobs.* @@ -11,13 +16,11 @@ import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.RoddyConversionHelperMethods import de.dkfz.roddy.tools.TimeUnit import groovy.transform.CompileStatic -import groovy.util.slurpersupport.GPathResult import java.time.Duration import java.time.Instant import java.time.ZoneId import java.time.ZonedDateTime -import java.util.regex.Matcher @CompileStatic abstract class GridEngineBasedJobManager extends ClusterJobManager { @@ -25,6 +28,10 @@ abstract class GridEngineBasedJobManager extends ClusterJobMa public static final String WITH_DELIMITER = '(?=(%1$s))' private final ZoneId TIME_ZONE_ID + @Override + ZonedDateTime parseTime(String str) { + return ZonedDateTime.ofInstant(Instant.ofEpochSecond(str as long), TIME_ZONE_ID) + } GridEngineBasedJobManager(BEExecutionService executionService, JobManagerOptions parms) { super(executionService, parms) @@ -48,65 +55,54 @@ abstract class GridEngineBasedJobManager extends ClusterJobMa } @Override - protected Map queryJobStates(List jobIDs) { - StringBuilder queryCommand = new StringBuilder(getQueryJobStatesCommand()) - - if (jobIDs && jobIDs.size() < 10) { - queryCommand << " " << jobIDs*.id.join(" ") - } - - if (isTrackingOfUserJobsEnabled) - queryCommand << " -u $userIDForQueries " + Map queryJobInfo(List jobIDs) { + String queryCommand = assembleJobInfoQueryCommand(jobIDs) - ExecutionResult er = executionService.execute(queryCommand.toString()) + ExecutionResult er = executionService.execute(queryCommand) List resultLines = er.resultLines - Map result = [:] - - if (!er.successful) { + if (!er.successful) throw new BEException("The execution of ${queryCommand} failed.\n\t" + er.resultLines?.join("\n\t")?.toString()) - } else { - if (resultLines.size() > 2) { - for (String line : resultLines) { - line = line.trim() - if (line.length() == 0) continue - if (!RoddyConversionHelperMethods.isInteger(line.substring(0, 1))) - continue //Filter out lines which have been missed which do not start with a number. + if (resultLines.size() < 2) + return [:] - String[] split = line.split("\\s+") - final int ID = getColumnOfJobID() - final int JOBSTATE = getColumnOfJobState() + Map result = [:] + for (String line : resultLines) { + line = line.trim() + if (line.length() == 0) continue + if (!RoddyConversionHelperMethods.isInteger(line.substring(0, 1))) + continue //Filter out lines which have been missed which do not start with a number. - BEJobID jobID = new BEJobID(split[ID]) + String[] split = line.split("\\s+") + final int ID = getColumnOfJobID() + final int JOBSTATE = getColumnOfJobState() - JobState js = parseJobState(split[JOBSTATE]) - result.put(jobID, js) - } - } + BEJobID jobID = new BEJobID(split[ID]) + if (jobIDs && !jobIDs.contains(jobID)) + continue //Ignore ids which are not queried but keep them, if we don't use a filter (jobIDs). + + JobState jobState = parseJobState(split[JOBSTATE]) + def info = new JobInfo(jobID) + info.jobState = jobState + + result[jobID] = info } return result } - @Override - Map queryExtendedJobStateById(List jobIds) { - Map queriedExtendedStates - String qStatCommand = getExtendedQueryJobStatesCommand() - qStatCommand += " " + jobIds.collect { it }.join(" ") - - if (isTrackingOfUserJobsEnabled) - qStatCommand += " -u $userIDForQueries " + String assembleJobInfoQueryCommand(List jobIDs) { + StringBuilder queryCommand = new StringBuilder(getQueryCommandForJobInfo()) - ExecutionResult er = executionService.execute(qStatCommand.toString()) + if (jobIDs && jobIDs.size() < 10) + queryCommand << " " << jobIDs*.id.join(" ") - if (er != null && er.successful) { - queriedExtendedStates = this.processQstatOutputFromXML(er.resultLines.join("\n")) - } else { - throw new BEException("Extended job states couldn't be retrieved. \n Returned status code:${er.exitCode} \n ${qStatCommand.toString()} \n\t result:${er.resultLines.join("\n\t")}") - } - return queriedExtendedStates + if (isTrackingOfUserJobsEnabled) + queryCommand << " -u $userIDForQueries " + queryCommand.toString() } + @Override protected ExecutionResult executeStartHeldJobs(List jobIDs) { String command = "qrls ${jobIDs*.id.join(" ")}" @@ -119,162 +115,75 @@ abstract class GridEngineBasedJobManager extends ClusterJobMa return executionService.execute(command, false) } - /** - * Reads qstat output - * @param qstatOutput - * @return output of qstat in a map with jobid as key - */ - private static Map> processQstatOutputFromPlainText(String qstatOutput) { - return qstatOutput.split(String.format(WITH_DELIMITER, "\n\nJob Id: ")).collectEntries { - Matcher matcher = it =~ /^\s*Job Id: (?\d+)\..*\n/ - def result = new HashMap() - if (matcher) { - result[matcher.group("jobId")] = it - } - result - }.collectEntries { jobId, value -> - // join multi-line values - value = ((String) value).replaceAll("\n\t", "") - [(jobId): value] - }.collectEntries { jobId, value -> - Map p = ((String) value).readLines(). - findAll { it.startsWith(" ") && it.contains(" = ") }. - collectEntries { - String[] parts = it.split(" = ") - new MapEntry(parts.head().replaceAll(/^ {4}/, ""), parts.tail().join(' ')) - } - [(jobId): p] - } as Map> + BufferValue safelyCastToBufferValue(Object value) { + String _value = value as String + if (_value) + return withCaughtAndLoggedException { new BufferValue(Integer.valueOf(_value.find(/(\d+)/)), BufferUnit.valueOf(_value[-2])) } + return null } - private ZonedDateTime parseTime(String str) { - return withCaughtAndLoggedException { ZonedDateTime.ofInstant(Instant.ofEpochSecond(str as long), TIME_ZONE_ID) } + Integer safelyCastToInteger(Object value) { + if (value) + return withCaughtAndLoggedException { return Integer.valueOf(value as String) } + return null } - /** - * Reads the qstat output and creates GenericJobInfo objects - * @param resultLines - Input of ExecutionResult object - * @return map with jobid as key - */ - protected Map processQstatOutputFromXML(String result) { - Map queriedExtendedStates = [:] - if (result.isEmpty()) { - return [:] - } + Duration safelyCastToDuration(Object value) { + String _value = value as String + if (_value) + return withCaughtAndLoggedException { Duration.ofSeconds(Math.round(Double.parseDouble(_value)), 0) } + return null + } - GPathResult parsedJobs = new XmlSlurper().parseText(result) - for (job in parsedJobs.children()) { - String jobIdRaw = job["Job_Id"] as String - BEJobID jobID - try { - jobID = new BEJobID(jobIdRaw) - } catch (Exception exp) { - throw new BEException("Job ID '${jobIdRaw}' could not be transformed to BEJobID ") - } - - List jobDependencies = withCaughtAndLoggedException { getJobDependencies(job["depend"] as String) } - String jobName = job["Job_Name"] as String ?: null - GenericJobInfo gj = new GenericJobInfo(jobName, null, jobID, null, jobDependencies) - - BufferValue mem = null - Integer cores - Integer nodes - TimeUnit walltime = null - String additionalNodeFlag - - Object resourceList = job["Resource_List"] - String resourcesListMem = resourceList["mem"] as String - String resourcesListNoDect = resourceList["nodect"] as String - String resourcesListNodes = resourceList["nodes"] as String - String resourcesListWalltime = resourceList["walltime"] as String - if (resourcesListMem) - mem = withCaughtAndLoggedException { new BufferValue(Integer.valueOf(resourcesListMem.find(/(\d+)/)), BufferUnit.valueOf(resourcesListMem[-2])) } - if (resourcesListNoDect) - nodes = withCaughtAndLoggedException { Integer.valueOf(resourcesListNoDect) } - if (resourcesListNodes) - cores = withCaughtAndLoggedException { Integer.valueOf(resourcesListNodes.find("ppn=.*").find(/(\d+)/)) } - if (resourcesListNodes) - additionalNodeFlag = withCaughtAndLoggedException { resourcesListNodes.find(/(\d+):(\.*)/) { fullMatch, nCores, feature -> return feature } } - if (resourcesListWalltime) - walltime = withCaughtAndLoggedException { new TimeUnit(resourcesListWalltime) } - - BufferValue usedMem = null - TimeUnit usedWalltime = null - Object resourcesUsed = job["resources_used"] - String resourcedUsedMem = resourcesUsed["mem"] as String - String resourcesUsedWalltime = resourcesUsed["walltime"] as String - if (resourcedUsedMem) - withCaughtAndLoggedException { usedMem = new BufferValue(Integer.valueOf(resourcedUsedMem.find(/(\d+)/)), BufferUnit.valueOf(resourcedUsedMem[-2])) } - if (resourcesUsedWalltime) - withCaughtAndLoggedException { usedWalltime = new TimeUnit(resourcesUsedWalltime) } - - gj.setAskedResources(new ResourceSet(null, mem, cores, nodes, walltime, null, job["queue"] as String ?: null, additionalNodeFlag)) - gj.setUsedResources(new ResourceSet(null, usedMem, null, null, usedWalltime, null, job["queue"] as String ?: null, null)) - - gj.setLogFile(withCaughtAndLoggedException { getQstatFile(job["Output_Path"] as String, jobIdRaw) }) - gj.setErrorLogFile(withCaughtAndLoggedException { getQstatFile(job["Error_Path"] as String, jobIdRaw) }) - gj.setUser(job["euser"] as String ?: null) - gj.setExecutionHosts(withCaughtAndLoggedException { getExecutionHosts(job["exec_host"] as String) }) - gj.setSubmissionHost(job["submit_host"] as String ?: null) - gj.setPriority(job["Priority"] as String ?: null) - gj.setUserGroup(job["egroup"] as String ?: null) - gj.setResourceReq(job["submit_args"] as String ?: null) - gj.setRunTime(job["total_runtime"] ? withCaughtAndLoggedException { Duration.ofSeconds(Math.round(Double.parseDouble(job["total_runtime"] as String)), 0) } : null) - gj.setCpuTime(resourcesUsed["cput"] ? withCaughtAndLoggedException { parseColonSeparatedHHMMSSDuration(job["resources_used"]["cput"] as String) } : null) - gj.setServer(job["server"] as String ?: null) - gj.setUmask(job["umask"] as String ?: null) - gj.setJobState(parseJobState(job["job_state"] as String)) - gj.setExitCode(job["exit_status"] ? withCaughtAndLoggedException { Integer.valueOf(job["exit_status"] as String) }: null ) - gj.setAccount(job["Account_Name"] as String ?: null) - gj.setStartCount(job["start_count"] ? withCaughtAndLoggedException { Integer.valueOf(job["start_count"] as String) } : null) - - if (job["qtime"]) // The time that the job entered the current queue. - gj.setSubmitTime(parseTime(job["qtime"] as String)) - if (job["start_time"]) // The timepoint the job was started. - gj.setStartTime(parseTime(job["start_time"] as String)) - if (job["comp_time"]) // The timepoint the job was completed. - gj.setEndTime(parseTime(job["comp_time"] as String)) - if (job["etime"]) // The time that the job became eligible to run, i.e. in a queued state while residing in an execution queue. - gj.setEligibleTime(parseTime(job["etime"] as String)) - - queriedExtendedStates.put(jobID, gj) - } - return queriedExtendedStates + TimeUnit safelyCastToTimeUnit(Object value) { + String _value = value as String + if (_value) + return withCaughtAndLoggedException { new TimeUnit(_value) } + return null } - private static List getExecutionHosts(String s) { - if (!s) { - return null + static List getExecutionHosts(Object hosts) { + String _hosts = hosts as String + if (!_hosts) { + return [] + } + withCaughtAndLoggedException { + _hosts.split(/\+/) + .collect { String str -> str.split("/") } + .collect { it.first() } + .unique() } - s.split(/\+/) - .collect { String it -> it.split("/") } - .collect { it.first() } - .unique() } - private static List getJobDependencies(String s) { - if (!s) { + static List getJobDependencies(Object deps) { + String _deps = deps as String + if (!_deps) { return [] } - s.split(",") - .find { it.startsWith("afterok") } - ?.findAll(/(\d+)(\.\w+)?/) { fullMatch, String beforeDot, afterDot -> return beforeDot } ?: []as List + withCaughtAndLoggedException { + return _deps.split(",") + .find { it.startsWith("afterok") } + ?.findAll(/(\d+)(\.\w+)?/) { fullMatch, String beforeDot, afterDot -> return beforeDot } ?: [] as List + } } - private File getQstatFile(String s, String jobId) { - if (!s) { + File safelyGetQstatFile(Object s, String jobId) { + String _s = s as String + if (!_s) { return null } - String fileName - if (s.startsWith("/")) { - fileName = s - } else if (s =~ /^[\w-]+:\//) { - fileName = s.replaceAll(/^[\w-]+:/, "") - } else { - return null + withCaughtAndLoggedException { + String fileName + if (_s.startsWith("/")) { + fileName = _s + } else if (_s =~ /^[\w-]+:\//) { + fileName = _s.replaceAll(/^[\w-]+:/, "") + } else { + return null + } + new File(fileName.replace("\$${getJobIdVariable()}", jobId)) } - new File(fileName.replace("\$${getJobIdVariable()}", jobId)) } @Override diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFCommandParser.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFCommandParser.groovy index 5c865b1e..0f0db4ad 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFCommandParser.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFCommandParser.groovy @@ -3,7 +3,7 @@ package de.dkfz.roddy.execution.jobs.cluster.lsf import de.dkfz.roddy.BEException import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.jobs.BEJobID -import de.dkfz.roddy.execution.jobs.GenericJobInfo +import de.dkfz.roddy.execution.jobs.ExtendedJobInfo import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.ComplexLine @@ -13,7 +13,7 @@ import groovy.transform.CompileStatic import static de.dkfz.roddy.StringConstants.* /** - * Used to convert commands from cli to e.g. GenericJobInfo + * Used to convert commands from cli to e.g. ExtendedJobInfo * Created by kaercher on 15.05.17. */ @CompileStatic @@ -122,12 +122,12 @@ class LSFCommandParser { } } - GenericJobInfo toGenericJobInfo() { - GenericJobInfo jInfo = new GenericJobInfo(jobName, script, jobID, parameters, dependencies) + ExtendedJobInfo toGenericJobInfo() { + ExtendedJobInfo jInfo = new ExtendedJobInfo(jobName, script, jobID, parameters, dependencies) ResourceSet askedResources = new ResourceSet(null, memory ? new BufferValue(memory as Integer, bufferUnit) : null, cores ? cores as Integer : null, nodes ? nodes as Integer : null, walltime ? new TimeUnit(walltime) : null, null, null, null) - jInfo.setAskedResources(askedResources) + jInfo.setRequestedResources(askedResources) return jInfo } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy index 55558d05..3a05431c 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy @@ -1,7 +1,7 @@ /* - * Copyright (c) 2017 eilslabs. + * Copyright (c) 2019 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). * - * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). + * Distributed under the MIT License (license terms are at https://www.github.com/TheRoddyWMS/BatchEuphoria/LICENSE.txt). */ package de.dkfz.roddy.execution.jobs.cluster.lsf @@ -16,7 +16,6 @@ import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.DateTimeHelper import groovy.json.JsonSlurper -import groovy.transform.CompileStatic import java.time.Duration import java.time.LocalDateTime @@ -33,23 +32,60 @@ class LSFJobManager extends AbstractLSFJobManager { private static final String LSF_COMMAND_QUERY_STATES = "bjobs -a -o -hms -json \"jobid job_name stat finish_time\"" private static final String LSF_COMMAND_QUERY_EXTENDED_STATES = "bjobs -a -o -hms -json \"jobid job_name stat user queue " + "job_description proj_name job_group job_priority pids exit_code from_host exec_host submit_time start_time " + - "finish_time cpu_used run_time user_group swap max_mem runtimelimit sub_cwd " + + "finish_time cpu_used run_time user_group swap max_mem max_req_proc" + + "memlimit runtimelimit swaplimit sub_cwd " + "pend_reason exec_cwd output_file input_file effective_resreq exec_home slots error_file command dependency \"" private static final String LSF_COMMAND_DELETE_JOBS = "bkill" - static final DateTimeHelper dateTimeHelper = new DateTimeHelper("MMM ppd HH:mm yyyy", Locale.ENGLISH) + public static final String longReportedDateFormatWithoutStatus = "MMM ppd HH:mm yyyy" + + static final DateTimeHelper dateTimeHelper = new DateTimeHelper(longReportedDateFormatWithoutStatus, Locale.ENGLISH) LSFJobManager(BEExecutionService executionService, JobManagerOptions parms) { super(executionService, parms) } - static ZonedDateTime parseTime(String str) { - ZonedDateTime date = dateTimeHelper.parseToZonedDateTime("${str} ${LocalDateTime.now().year}") - /** - * parseToZonedDateTime() parses a zoned time as provided by LSF. Unfortunately, LFS does not return the submission year! - * The (kind of reasonable) assumption made here is that if the job's submission time (assuming the current - * year) is later than the current time, then the job was submitted last year. - */ + /** + * Time formats in LSF can be configured. + * E.g. to our knowledge, by default, reported values look like: + * "Jan 1 10:21 L" with status information or "Jan 1 10:21" without them. + * LSF can be configured to also report the year, so values would be reported as: + * "Jan 1 10:21 2010 L" with status information or "Jan 1 10:21 2010" without them. + * + * There might be different configurations, but we stick to those 4 versions. + * + * Furthermore, we do not know, if LSF will report in other languages than english. + * We assume, that english is used for month names. + * + * We also assume, that the method will not be misused. If its misused, it will throw + * an exception. + */ + @Override + ZonedDateTime parseTime(String str) { + // Prevent NullPointerException, will throw a DateTimeParserException later + if (str == null) str = "" + String dateForParser = str + if (str.size() == "Jan 01 01:00".size()) { + // Lets start and see, if the date is reported in its short version, if so, add the year. + dateForParser = "${str} ${LocalDateTime.now().year}" + } else if (str.size() == "Jan 01 01:00 L".size()) { + // Here we need to strip away the status first, then append the current year. + dateForParser = "${stripAwayStatusInfo(str)} ${LocalDateTime.now().year}" + } else if (str.size() == "Jan 01 01:00 1000".size()) { + // Easy enough, just keep it like it is. + dateForParser = str + } else if (str.size() == "Jan 01 01:00 1000 L".size()) { + // Again, strip away the status info. + dateForParser = stripAwayStatusInfo(str) + } + + // Finally we try to parse the date. Lets see if it works. If not, an exception is thrown. + ZonedDateTime date = dateTimeHelper.parseToZonedDateTime(dateForParser) + + // If LSF is not configured to report the date, the (kind of reasonable) assumption made here is that if + // the job's submission time (assuming the current year) is later than the current time, then the job was + // submitted last year. + if (date > ZonedDateTime.now()) { return date.minusYears(1) } @@ -62,20 +98,12 @@ class LSFJobManager extends AbstractLSFJobManager { * of the time string. */ static String stripAwayStatusInfo(String time) { - if (time) - return time[0..-3] - return null - } - - @Override - Map queryExtendedJobStateById(List jobIds) { - Map queriedExtendedStates = [:] - for (BEJobID id : jobIds) { - Map jobDetails = runBjobs([id], true)[id] - if (jobDetails) // Ignore filtered / nonexistent ids - queriedExtendedStates.put(id, convertJobDetailsMapToGenericJobInfoObject(jobDetails)) + String result = time + if (time && time.size() > 2) { + if (time[-2..-1] ==~ ~/[ ][a-zA-Z0-9]/) + result = time[0..-3] } - return queriedExtendedStates + return result } @Override @@ -101,19 +129,34 @@ class LSFJobManager extends AbstractLSFJobManager { } @Override - GenericJobInfo parseGenericJobInfo(String commandString) { + ExtendedJobInfo parseGenericJobInfo(String commandString) { return new LSFCommandParser(commandString).toGenericJobInfo(); } - protected Map queryJobStates(List jobIDs) { + @Override + Map queryJobInfo(List jobIDs) { def bjobs = runBjobs(jobIDs, false) - bjobs.collectEntries { BEJobID jobID, Object value -> + bjobs = bjobs.findAll { BEJobID k, Map v -> v } + + return bjobs.collectEntries { BEJobID jobID, Object value -> JobState js = parseJobState(value["STAT"] as String) - [(jobID): js] - } as Map + JobInfo jobInfo = new JobInfo(jobID) + jobInfo.jobState = js + [(jobID): jobInfo] + } as Map } + @Override + Map queryExtendedJobInfo(List jobIds) { + Map queriedExtendedStates = [:] + for (BEJobID id : jobIds) { + Map jobDetails = runBjobs([id], true)[id] + queriedExtendedStates.put(id, convertJobDetailsMapToGenericJobInfoObject(jobDetails)) + } + return queriedExtendedStates + } + Map> runBjobs(List jobIDs, boolean extended) { StringBuilder queryCommand = new StringBuilder(extended ? LSF_COMMAND_QUERY_EXTENDED_STATES : LSF_COMMAND_QUERY_STATES) @@ -133,8 +176,7 @@ class LSFJobManager extends AbstractLSFJobManager { throw new BEException(error) } - Map> result = convertBJobsJsonOutputToResultMap(resultLines.join("\n")) - return filterJobMapByAge(result, maxTrackingTimeForFinishedJobs) + return convertBJobsJsonOutputToResultMap(resultLines.join("\n")) } static Map> convertBJobsJsonOutputToResultMap(String rawJson) { @@ -153,116 +195,70 @@ class LSFJobManager extends AbstractLSFJobManager { result } - /** - * For all entries in records, check if they are finished and if so, check if they are younger (or older) than - * the maximum age. - * @param records A map of informational entries for one or more job ids - * @param reference Time A timestamp which can be set. It is compared against the timestamp of finished entries. - * @param maxJobKeepDuration Defines the maximum duration - * @return The map of records where too old entries are filtered out. - */ - @CompileStatic - static Map> filterJobMapByAge( - Map> records, - Duration maxJobKeepDuration - ) { - records.findAll { def k, def record -> - String finishTime = record["FINISH_TIME"] - boolean youngEnough = true - if (finishTime) { - withCaughtAndLoggedException { - ZonedDateTime _finishTime = parseTime(stripAwayStatusInfo(finishTime)) - Duration timeSpan = Duration.between(_finishTime.toLocalDateTime(), LocalDateTime.now()) - if (dateTimeHelper.durationExceeds(timeSpan, maxJobKeepDuration)) - youngEnough = false - } - } - youngEnough - } - } - /** * Used by @getJobDetails to set JobInfo */ - GenericJobInfo convertJobDetailsMapToGenericJobInfoObject(Map _jobResult) { + ExtendedJobInfo convertJobDetailsMapToGenericJobInfoObject(Map _rawExtendedStates) { // Remove empty entries first to keep the output clean (use null, where the value is null or empty.) - Map jobResult = _jobResult.findAll { String k, String v -> v } - - GenericJobInfo jobInfo - BEJobID jobID - String JOBID = jobResult["JOBID"] - try { - jobID = new BEJobID(JOBID) - } catch (Exception exp) { - throw new BEException("Job ID '${JOBID}' could not be transformed to BEJobID ") - } + Map extendedStates = _rawExtendedStates.findAll { String k, String v -> v } + + BEJobID jobID = toJobID(extendedStates["JOBID"]) - List dependIDs = jobResult["DEPENDENCY"]?.tokenize(/&/)?.collect { it.find(/\d+/) } - jobInfo = new GenericJobInfo(jobResult["JOB_NAME"], jobResult["COMMAND"], jobID, null, dependIDs) + List dependIDs = extendedStates["DEPENDENCY"]?.tokenize(/&/)?.collect { it.find(/\d+/) } + ExtendedJobInfo jobInfo = new ExtendedJobInfo(extendedStates["JOB_NAME"], extendedStates["COMMAND"], jobID, null, dependIDs) /** Common */ - jobInfo.user = jobResult["USER"] - jobInfo.userGroup = jobResult["USER_GROUP"] - jobInfo.description = jobResult["JOB_DESCRIPTION"] - jobInfo.projectName = jobResult["PROJ_NAME"] - jobInfo.jobGroup = jobResult["JOB_GROUP"] - jobInfo.priority = jobResult["JOB_PRIORITY"] - jobInfo.pidStr = jobResult["PIDS"]?.split(",")?.toList() - jobInfo.submissionHost = jobResult["FROM_HOST"] - jobInfo.executionHosts = jobResult["EXEC_HOST"]?.split(":")?.toList() + jobInfo.user = extendedStates["USER"] + jobInfo.userGroup = extendedStates["USER_GROUP"] + jobInfo.description = extendedStates["JOB_DESCRIPTION"] + jobInfo.projectName = extendedStates["PROJ_NAME"] + jobInfo.jobGroup = extendedStates["JOB_GROUP"] + jobInfo.priority = extendedStates["JOB_PRIORITY"] + jobInfo.processesInJob = extendedStates["PIDS"]?.split(",")?.toList() + jobInfo.submissionHost = extendedStates["FROM_HOST"] /** Resources */ - String queue = jobResult["QUEUE"] - Duration runLimit = safelyParseColonSeparatedDuration(jobResult["RUNTIMELIMIT"]) - Duration runTime = safelyParseColonSeparatedDuration(jobResult["RUN_TIME"]) - BufferValue memory = safelyCastToBufferValue(jobResult["MAX_MEM"]) + jobInfo.executionHosts = extendedStates["EXEC_HOST"]?.split(":")?.toList() + // Count hosts! The node count has no custom entry. However, we can calculate it from the host list. + Integer noOfExecutionHosts = jobInfo.executionHosts?.sort()?.unique()?.size() + String queue = extendedStates["QUEUE"] + Duration requestedWalltime = safelyParseColonSeparatedDuration(extendedStates["RUNTIMELIMIT"]) + Duration usedWalltime = safelyParseColonSeparatedDuration(extendedStates["RUN_TIME"]) + BufferValue usedMemory = safelyCastToBufferValue(extendedStates["MAX_MEM"]) + BufferValue requestedMemory = safelyCastToBufferValue(extendedStates["MEMLIMIT"]) + Integer requestedCores = extendedStates["MAX_REQ_PROC"] as Integer BufferValue swap = withCaughtAndLoggedException { - String SWAP = jobResult["SWAP"] + String SWAP = extendedStates["SWAP"] SWAP ? new BufferValue(SWAP.find("\\d+"), BufferUnit.m) : null } - Integer nodes = withCaughtAndLoggedException { jobResult["SLOTS"] as Integer } - jobInfo.usedResources = new ResourceSet(memory, null, nodes, runTime, null, queue, null) - jobInfo.askedResources = new ResourceSet(null, null, null, runLimit, null, queue, null) - jobInfo.resourceReq = jobResult["EFFECTIVE_RESREQ"] - jobInfo.runTime = runTime - jobInfo.cpuTime = safelyParseColonSeparatedDuration(jobResult["CPU_USED"]) + jobInfo.usedResources = new ResourceSet(usedMemory, null, noOfExecutionHosts, usedWalltime, null, queue, null) + jobInfo.requestedResources = new ResourceSet(requestedMemory, requestedCores, null, requestedWalltime, null, queue, null) + jobInfo.rawResourceRequest = extendedStates["EFFECTIVE_RESREQ"] + jobInfo.runTime = usedWalltime + jobInfo.cpuTime = safelyParseColonSeparatedDuration(extendedStates["CPU_USED"]) /** Status info */ - jobInfo.jobState = parseJobState(jobResult["STAT"]) - jobInfo.exitCode = jobInfo.jobState == JobState.COMPLETED_SUCCESSFUL ? 0 : (jobResult["EXIT_CODE"] as Integer) - jobInfo.pendReason = jobResult["PEND_REASON"] + jobInfo.jobState = parseJobState(extendedStates["STAT"]) + jobInfo.exitCode = jobInfo.jobState == JobState.COMPLETED_SUCCESSFUL ? 0 : (extendedStates["EXIT_CODE"] as Integer) + jobInfo.pendReason = extendedStates["PEND_REASON"] /** Directories and files */ - jobInfo.cwd = jobResult["SUB_CWD"] - jobInfo.execCwd = jobResult["EXEC_CWD"] - jobInfo.logFile = getBjobsFile(jobResult["OUTPUT_FILE"], jobID, "out") - jobInfo.errorLogFile = getBjobsFile(jobResult["ERROR_FILE"], jobID, "err") - jobInfo.inputFile = jobResult["INPUT_FILE"] ? new File(jobResult["INPUT_FILE"]) : null - jobInfo.execHome = jobResult["EXEC_HOME"] + jobInfo.cwd = extendedStates["SUB_CWD"] + jobInfo.execCwd = extendedStates["EXEC_CWD"] + jobInfo.logFile = getBjobsFile(extendedStates["OUTPUT_FILE"], jobID, "out") + jobInfo.errorLogFile = getBjobsFile(extendedStates["ERROR_FILE"], jobID, "err") + jobInfo.inputFile = extendedStates["INPUT_FILE"] ? new File(extendedStates["INPUT_FILE"]) : (File)null + jobInfo.execHome = extendedStates["EXEC_HOME"] /** Timestamps */ - jobInfo.submitTime = safelyParseTime(jobResult["SUBMIT_TIME"]) - jobInfo.startTime = safelyParseTime(jobResult["START_TIME"]) - jobInfo.endTime = safelyParseTime(stripAwayStatusInfo(jobResult["FINISH_TIME"])) + jobInfo.submitTime = safelyParseTime(extendedStates["SUBMIT_TIME"]) + jobInfo.startTime = safelyParseTime(extendedStates["START_TIME"]) + jobInfo.endTime = safelyParseTime(extendedStates["FINISH_TIME"]) return jobInfo } - Duration safelyParseColonSeparatedDuration(String value) { - withCaughtAndLoggedException { - value ? parseColonSeparatedHHMMSSDuration(value) : null - } - } - - ZonedDateTime safelyParseTime(String time) { - if (time) - return withCaughtAndLoggedException { - return parseTime(time) - } - return null - } - BufferValue safelyCastToBufferValue(String MAX_MEM) { withCaughtAndLoggedException { if (MAX_MEM) { @@ -313,12 +309,12 @@ class LSFJobManager extends AbstractLSFJobManager { } @Override - String getQueryJobStatesCommand() { + String getQueryCommandForJobInfo() { return null } @Override - String getExtendedQueryJobStatesCommand() { + String getQueryCommandForExtendedJobInfo() { return null } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy index 0d348035..865436d6 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy @@ -134,7 +134,7 @@ class LSFRestJobManager extends AbstractLSFJobManager { @Override - GenericJobInfo parseGenericJobInfo(String command) { + ExtendedJobInfo parseGenericJobInfo(String command) { return null } @@ -271,9 +271,9 @@ class LSFRestJobManager extends AbstractLSFJobManager { * Updates job information for given jobs * @param jobList */ - private Map getJobDetails(List jobList) { + private Map getJobDetails(List jobList) { List
headers = [] - Map jobDetailsResult = [:] + Map jobDetailsResult = [:] headers.add(new BasicHeader("Accept", "text/xml,application/xml;")) RestResult result = restExecutionService.execute(new RestCommand(URI_JOB_DETAILS + prepareURLWithParam(jobList), null, headers, RestCommand.HttpMethod.HTTPGET)) as RestResult @@ -296,14 +296,14 @@ class LSFRestJobManager extends AbstractLSFJobManager { * @param list of job ids */ @Override - Map queryJobStates(List jobIds) { + Map queryJobInfo(List jobIds) { List
headers = [] headers.add(new BasicHeader("Accept", "text/xml,application/xml;")) RestResult result = restExecutionService.execute(new RestCommand(URI_JOB_BASICS, null, headers, RestCommand.HttpMethod.HTTPGET)) as RestResult if (result.isSuccessful()) { GPathResult res = new XmlSlurper().parseText(result.body) - Map resultStates = [:] + Map resultStates = [:] res.getProperty("pseudoJob").each { NodeChild element -> String jobId = null if (jobIds) { @@ -315,7 +315,11 @@ class LSFRestJobManager extends AbstractLSFJobManager { } if (jobId) { - resultStates.put(new BEJobID(jobId), parseJobState(element.getProperty("jobStatus").toString())) + def jobID = new BEJobID(jobId) + def jobState = parseJobState(element.getProperty("jobStatus").toString()) + JobInfo jobInfo = new JobInfo(jobID) + jobInfo.jobState = jobState + resultStates[jobID] = jobInfo } } return resultStates @@ -324,14 +328,18 @@ class LSFRestJobManager extends AbstractLSFJobManager { } } - /** + @Override + Map queryExtendedJobInfo(List jobIDs) { + return null + } +/** * Used by @getJobDetails to set JobInfo * @param job * @param jobDetails - XML job details */ - private GenericJobInfo setJobInfoForJobDetails(NodeChild jobDetails) { + private ExtendedJobInfo setJobInfoForJobDetails(NodeChild jobDetails) { - GenericJobInfo jobInfo = new GenericJobInfo(jobDetails.getProperty("jobName").toString(), jobDetails.getProperty("command").toString(), new BEJobID(jobDetails.getProperty("jobId").toString()), null, null) + ExtendedJobInfo jobInfo = new ExtendedJobInfo(jobDetails.getProperty("jobName").toString(), jobDetails.getProperty("command").toString(), new BEJobID(jobDetails.getProperty("jobId").toString()), null, null) String queue = jobDetails.getProperty("queue").toString() BufferValue swap = jobDetails.getProperty("swap") ? withCaughtAndLoggedException { new BufferValue(jobDetails.getProperty("swap").toString(), BufferUnit.m) } : null @@ -364,23 +372,23 @@ class LSFRestJobManager extends AbstractLSFJobManager { jobInfo.setRunTime(jobDetails.getProperty("runTime") ? withCaughtAndLoggedException { Duration.ofSeconds(Math.round(Double.parseDouble(jobDetails.getProperty("runTime").toString()))) } : null) jobInfo.setProjectName(jobDetails.getProperty("projectName").toString()) jobInfo.setExitCode(jobDetails.getProperty("exitStatus").toString() ? withCaughtAndLoggedException { Integer.valueOf(jobDetails.getProperty("exitStatus").toString()) } : null) - jobInfo.setPidStr(jobDetails.getProperty("pidStr") as String ? (jobDetails.getProperty("pidStr") as String).split(",").toList() : null) - jobInfo.setPgidStr(jobDetails.getProperty("pgidStr").toString()) + jobInfo.setProcessesInJob(jobDetails.getProperty("pidStr") as String ? (jobDetails.getProperty("pidStr") as String).split(",").toList() : null) + jobInfo.setProcessGroupID(jobDetails.getProperty("pgidStr").toString()) jobInfo.setCwd(jobDetails.getProperty("cwd").toString()) jobInfo.setPendReason(jobDetails.getProperty("pendReason").toString()) jobInfo.setExecCwd(jobDetails.getProperty("execCwd").toString()) jobInfo.setPriority(jobDetails.getProperty("priority").toString()) jobInfo.setLogFile(new File(jobDetails.getProperty("outFile").toString())) jobInfo.setInputFile(new File(jobDetails.getProperty("inFile").toString())) - jobInfo.setResourceReq(jobDetails.getProperty("resReq").toString()) + jobInfo.setRawResourceRequest(jobDetails.getProperty("resReq").toString()) jobInfo.setExecHome(jobDetails.getProperty("execHome").toString()) jobInfo.setExecUserName(jobDetails.getProperty("execUserName").toString()) - jobInfo.setAskedHostsStr(jobDetails.getProperty("askedHostsStr").toString()) + jobInfo.setRawHostQueryString(jobDetails.getProperty("askedHostsStr").toString()) return jobInfo } - private ZonedDateTime parseTime(String str) { + ZonedDateTime parseTime(String str) { if (!str) { return null } @@ -391,7 +399,7 @@ class LSFRestJobManager extends AbstractLSFJobManager { * Get the time history for each given job * @param jobList */ - private void updateJobStatistics(Map jobList) { + private void updateJobStatistics(Map jobList) { List
headers = [] headers.add(new BasicHeader("Accept", "text/xml,application/xml;")) @@ -415,7 +423,7 @@ class LSFRestJobManager extends AbstractLSFJobManager { * @param job * @param jobHistory - xml job history */ - private void setJobInfoFromJobHistory(GenericJobInfo jobInfo, NodeChild jobHistory) { + private void setJobInfoFromJobHistory(ExtendedJobInfo jobInfo, NodeChild jobHistory) { GPathResult timeSummary = jobHistory.getProperty("timeSummary") as GPathResult DateTimeFormatter lsfDatePattern = DateTimeFormatter.ofPattern("EEE MMM ppd HH:mm:ss yyyy").withLocale(Locale.ENGLISH) @@ -430,8 +438,8 @@ class LSFRestJobManager extends AbstractLSFJobManager { } @Override - Map queryExtendedJobStateById(List jobIds) { - Map jobDetailsResult = getJobDetails(jobIds) + Map queryExtendedJobStateById(List jobIds) { + Map jobDetailsResult = getJobDetails(jobIds) updateJobStatistics(jobDetailsResult) return jobDetailsResult } @@ -442,12 +450,12 @@ class LSFRestJobManager extends AbstractLSFJobManager { } @Override - String getQueryJobStatesCommand() { + String getQueryCommandForJobInfo() { return null } @Override - String getExtendedQueryJobStatesCommand() { + String getQueryCommandForExtendedJobInfo() { return null } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParser.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParser.groovy index 88bfa9b0..48e2ae9a 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParser.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParser.groovy @@ -9,7 +9,7 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs import de.dkfz.roddy.BEException import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.jobs.BEJobID -import de.dkfz.roddy.execution.jobs.GenericJobInfo +import de.dkfz.roddy.execution.jobs.ExtendedJobInfo import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.ComplexLine @@ -19,7 +19,7 @@ import groovy.transform.CompileStatic import static de.dkfz.roddy.StringConstants.* /** - * Used to convert commands from cli to e.g. GenericJobInfo + * Used to convert commands from cli to e.g. ExtendedJobInfo * Created by heinold on 04.04.17. */ @CompileStatic @@ -128,12 +128,12 @@ class PBSCommandParser { } } - GenericJobInfo toGenericJobInfo() { - GenericJobInfo jInfo = new GenericJobInfo(jobName, script, jobID, parameters, dependencies) + ExtendedJobInfo toGenericJobInfo() { + ExtendedJobInfo jInfo = new ExtendedJobInfo(jobName, script, jobID, parameters, dependencies) ResourceSet askedResources = new ResourceSet(null, memory ? new BufferValue(memory as Integer, bufferUnit) : null, cores ? cores as Integer : null, nodes ? nodes as Integer : null, walltime ? new TimeUnit(walltime) : null, null, null, null) - jInfo.setAskedResources(askedResources) + jInfo.setRequestedResources(askedResources) return jInfo } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy index 5699decc..c9e9898d 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy @@ -7,16 +7,18 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs import com.google.common.collect.LinkedHashMultimap +import de.dkfz.roddy.BEException import de.dkfz.roddy.StringConstants import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService -import de.dkfz.roddy.execution.jobs.BEJob -import de.dkfz.roddy.execution.jobs.GenericJobInfo -import de.dkfz.roddy.execution.jobs.JobManagerOptions -import de.dkfz.roddy.execution.jobs.JobState +import de.dkfz.roddy.execution.io.ExecutionResult +import de.dkfz.roddy.execution.jobs.* import de.dkfz.roddy.execution.jobs.cluster.GridEngineBasedJobManager import de.dkfz.roddy.tools.BufferUnit +import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.TimeUnit +import groovy.transform.PackageScope +import groovy.util.slurpersupport.GPathResult /** * @author michael @@ -34,8 +36,8 @@ class PBSJobManager extends GridEngineBasedJobManager { } @Override - GenericJobInfo parseGenericJobInfo(String commandString) { - return new PBSCommandParser(commandString).toGenericJobInfo(); + ExtendedJobInfo parseGenericJobInfo(String commandString) { + return new PBSCommandParser(commandString).toGenericJobInfo() } /** @@ -79,12 +81,12 @@ class PBSJobManager extends GridEngineBasedJobManager { } @Override - String getQueryJobStatesCommand() { + String getQueryCommandForJobInfo() { return "qstat -t" } @Override - String getExtendedQueryJobStatesCommand() { + String getQueryCommandForExtendedJobInfo() { return "qstat -x -f" } @@ -157,4 +159,96 @@ class PBSJobManager extends GridEngineBasedJobManager { return "PBS_O_WORKDIR" } + + @Override + Map queryExtendedJobInfo(List jobIds) { + Map queriedExtendedStates = [:] + String qStatCommand = getQueryCommandForExtendedJobInfo() + qStatCommand += " " + jobIds.collect { it }.join(" ") + // For the reviewer: This does not make sense right? We have a list of given ids which we want to + // query anyway. + // if (isTrackingOfUserJobsEnabled) + // qStatCommand += " -u $userIDForQueries " + + ExecutionResult er = executionService.execute(qStatCommand.toString()) + if (er != null && er.successful) { + GPathResult parsedJobs = new XmlSlurper().parseText(er.resultLines.join("\n")) + for (BEJobID jobID : jobIds) { + def jobInfo = parsedJobs.children().find { it["Job_Id"].toString().startsWith("${jobID.id}.") } + queriedExtendedStates[jobID] = this.processQstatOutputFromXMLNodeChild(jobInfo) + } + } else { + throw new BEException("Extended job states couldn't be retrieved. \n Returned status code:${er.exitCode} \n ${qStatCommand.toString()} \n\t result:${er.resultLines.join("\n\t")}") + } + return queriedExtendedStates + } + + /** + * Reads the qstat output and creates ExtendedJobInfo objects + * @param resultLines - Input of ExecutionResult object + * @return map with jobid as key + */ + @PackageScope + ExtendedJobInfo processQstatOutputFromXMLNodeChild(GPathResult job) { + String jobIdRaw = job["Job_Id"] as String + BEJobID jobID = toJobID(jobIdRaw) + + List jobDependencies = getJobDependencies(job["depend"]) + ExtendedJobInfo jobInfo = new ExtendedJobInfo(job["Job_Name"] as String, null, jobID, null, jobDependencies) + + /** Common */ + jobInfo.user = (job["Job_Owner"] as String)?.split(StringConstants.SPLIT_AT)[0] + jobInfo.userGroup = job["egroup"] + jobInfo.priority = job["Priority"] + jobInfo.submissionHost = job["submit_host"] + jobInfo.server = job["server"] + jobInfo.umask = job["umask"] + jobInfo.account = job["Account_Name"] + jobInfo.startCount = safelyCastToInteger(job["start_count"]) + + /** Resources + * Note: Both nodes and nodect are deprecated values + * nodect is the number of requested nodes and of type integer + */ + jobInfo.executionHosts = getExecutionHosts(job["exec_host"]) + String queue = job["queue"] as String + + def requestedResources = job["Resource_List"] + String resourcesListNodes = requestedResources["nodes"] + Integer requestedNodes = safelyCastToInteger(requestedResources["nodect"]) + Integer requestedCores = safelyCastToInteger(resourcesListNodes?.find("ppn=.*")?.find(/(\d+)/)) + String requestedAdditionalNodeFlag = withCaughtAndLoggedException { resourcesListNodes?.find(/(\d+):(\.*)/) { fullMatch, nCores, feature -> return feature } } + TimeUnit requestedWalltime = safelyCastToTimeUnit(requestedResources["walltime"]) + BufferValue requestedMemory = safelyCastToBufferValue(requestedResources["mem"]) + + def usedResources = job["resources_used"] + TimeUnit usedWalltime = safelyCastToTimeUnit(usedResources["walltime"]) + BufferValue usedMemory = safelyCastToBufferValue(usedResources["mem"]) + + jobInfo.requestedResources = new ResourceSet(null, requestedMemory, requestedCores, requestedNodes, requestedWalltime, null, queue, requestedAdditionalNodeFlag) + jobInfo.usedResources = new ResourceSet(null, usedMemory, null, null, usedWalltime, null, queue, null) + jobInfo.rawResourceRequest = job["submit_args"] + jobInfo.runTime = safelyCastToDuration(job["total_runtime"]) + jobInfo.cpuTime = safelyParseColonSeparatedDuration(usedResources["cput"]) + + /** Status info */ + jobInfo.jobState = parseJobState(job["job_state"] as String) + jobInfo.exitCode = safelyCastToInteger(job["exit_status"]) + + /** Directories and files */ + jobInfo.logFile = safelyGetQstatFile(job["Output_Path"] as String, jobIdRaw) + jobInfo.errorLogFile = safelyGetQstatFile(job["Error_Path"] as String, jobIdRaw) + + /** Timestamps */ + jobInfo.submitTime = safelyParseTime(job["qtime"]) // The time that the job entered the current queue. + jobInfo.startTime = safelyParseTime(job["start_time"]) // The timepoint the job was started. + jobInfo.endTime = safelyParseTime(job["comp_time"]) // The timepoint the job was completed. + jobInfo.eligibleTime = safelyParseTime(job["etime"]) // The time that the job became eligible to run, i.e. in a queued state while residing in an execution queue. + // http://docs.adaptivecomputing.com/torque/4-2-7/Content/topics/9-accounting/accountingRecords.htm + // Left is ctime: The the time the job was created + + return jobInfo + } + + } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManager.groovy index 613eb0ad..d6c9c58a 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManager.groovy @@ -7,22 +7,19 @@ package de.dkfz.roddy.execution.jobs.cluster.sge import com.google.common.collect.LinkedHashMultimap -import de.dkfz.roddy.BEException import de.dkfz.roddy.StringConstants import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.execution.io.ExecutionResult import de.dkfz.roddy.execution.jobs.* -import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager import de.dkfz.roddy.execution.jobs.cluster.GridEngineBasedJobManager -import de.dkfz.roddy.tools.* +import de.dkfz.roddy.tools.BufferUnit +import de.dkfz.roddy.tools.BufferValue +import de.dkfz.roddy.tools.TimeUnit import groovy.util.slurpersupport.GPathResult import java.time.Duration -import java.time.Instant -import java.time.LocalDateTime -import java.time.ZoneId -import java.util.regex.Matcher +import java.time.ZonedDateTime /** * @author michael @@ -40,17 +37,17 @@ class SGEJobManager extends GridEngineBasedJobManager { } @Override - String getQueryJobStatesCommand() { - return "qstat -g d -j" + String getQueryCommandForJobInfo() { + return "qstat" } @Override - String getExtendedQueryJobStatesCommand() { + String getQueryCommandForExtendedJobInfo() { return "qstat -xml -ext -f -j" } @Override - GenericJobInfo parseGenericJobInfo(String command) { + ExtendedJobInfo parseGenericJobInfo(String command) { return null } @@ -134,4 +131,157 @@ class SGEJobManager extends GridEngineBasedJobManager { return null } + @Override + String assembleJobInfoQueryCommand(List jobIDs) { + + // SGE will output quite a bunch of information if you query by job. Thus said, just querying all jobs, + // maybe filtered by the user, will possible result in less lines per query. E.g. a query with around + // 8000 active jobs will result in approximately 830kByte, whereas a query with qstat -j ID will + // result in 30 lines per job multiplied by (guessed) 40Byte per line = 1,2kByte per Job. We could overcome this + // by putting in a filter with grep, but then again, the state is displayed differently than in the tabluar + // qstat (e.g. scheduling info: Job is in hold state), which makes it a little more difficult to use + // => We'll stick to a simple "qstat (-u)" for now. + String command = getQueryCommandForJobInfo() + + if (isTrackingOfUserJobsEnabled) + command += " -u $userIDForQueries " + + return command + } + + Map convertJobInfoMapToExtendedMap(Map jobInfoMap) { + return jobInfoMap.collectEntries { + BEJobID jobID, JobInfo jobInfo -> + [jobID, new ExtendedJobInfo(jobID, jobInfo.jobState)] + } as Map + } + + @Override + Map queryExtendedJobInfo(List jobIDs) { + // Unfortunately, SGEs qstat does not show some information in extended mode, e.g.: + // - The start time + // - The job state (ok it sometimes displays something. But its some sort of message and it is not directly + // attached to the job AND e.g. running is not shown at all + // - Finish time + // - Execution host + // To overcome at least the missing job state, we call queryJobInfo() for this information first. + + Map shortInfo = jobIDs == null ? queryAllJobInfo() : queryJobInfoByID(jobIDs) + + // If no jobs were found or all are of state unknown, we can actually just omit the rest. + // This will make things a lot easier because: + // - If there is NO job info in the result xml of the extended query, the result will be different and return + // invalid! (for my SGE tests, see the test files) xml code! + // - If there is at least one job found, you will not see info about unknown jobs! + // - So either preprocess the xml and correct it or check first, if we have info at + // all and omit the query then. + if (shortInfo.size() == 0 || shortInfo.values().every { it.jobState == JobState.UNKNOWN }) + return convertJobInfoMapToExtendedMap(shortInfo) + + // Now we at least have some states, regardless of their existence in our list of job ids + // Lets retrieve and parse our extended states. + ExecutionResult er = executionService.execute(getQueryCommandForExtendedJobInfo()) + + if (!er.successful) + return convertJobInfoMapToExtendedMap(shortInfo) + + Map result = [:] + + // Finally we can parse our hopefully correct xml info + def xml = new XmlSlurper().parseText(er.resultLines.join("\n")) + + xml["djob_info"]["element"].each { + GPathResult element -> + ExtendedJobInfo jobInfo = parseGPathJobInfoElement(element) + jobInfo.jobState = shortInfo[jobInfo.jobID]?.jobState ?: JobState.UNKNOWN + result[jobInfo.jobID] = jobInfo + } + + // If a result is missing, it will be filled in later by the methods in BatchEuphoriaJobManager. + return result + } + + /** + * The job state will NOT be parsed in this method! It is solely to be used as a helper for queryExtendedJobInfo + */ + ExtendedJobInfo parseGPathJobInfoElement(GPathResult job) { + ExtendedJobInfo jobInfo = new ExtendedJobInfo(new BEJobID(job["JB_job_number"] as String)) + + /** Common */ + jobInfo.user = job["JB_owner"] + jobInfo.userGroup = job["JB_group"] + jobInfo.priority = job["JB_priority"] + // jobInfo.submissionHost = job["JB_submit_host"] // Info is not available + // jobInfo.server = job["JB_server"] // Info is not available + // jobInfo.umask = job["JB_umask"] // Info is not available + jobInfo.account = job["JB_account"] + jobInfo.startCount = safelyCastToInteger(job["JB_restart"]) + + /** Resources */ + // jobInfo.executionHosts = getExecutionHosts(job["JB_exec_host"]) // Info is not available + // jobInfo.queue // Info is not available + + // Getting cores and nodes is a bit tricks. Nodes are not in the xml, cores via JB_pe_range + String requestedCoresRaw = job["JB_pe_range"]["ranges"]["RN_min"]?.toString() + Integer requestedCores = requestedCoresRaw ? requestedCoresRaw as Integer : null + + def requestedResourcesRaw = ((job["JB_hard_resource_list"]["qstat_l_requests"] as GPathResult).children() as List) + .findAll { it.name() == "qsat_l_requests" }.collectEntries { + [it["CE_name"], it["CE_doubleval"] as Integer] + } + Long requestedMemoryRaw = requestedResourcesRaw["h_rss"] as Long + BufferValue requestedMemory + if (requestedMemoryRaw != null) + requestedMemory = new BufferValue((Integer) (requestedMemoryRaw / 1024L), BufferUnit.k) + + Integer requestedWalltimeRaw = requestedResourcesRaw["h_rt"] as Integer + Duration requestedWalltime = requestedWalltimeRaw ? Duration.ofSeconds(requestedWalltimeRaw) : null + jobInfo.requestedResources = new ResourceSet(requestedMemory, requestedCores, null, requestedWalltime, null, null, null) + + Long usedWalltimeRaw = (extractStatisticsValue(job, "ru_wallclock") as Double) as Long + Duration usedWalltime = null + if (usedWalltime != null) + Duration.ofSeconds(usedWalltimeRaw) + Double usedMemoryRaw = extractStatisticsValue(job, "ru_maxrss") as Double + BufferValue usedMemory = null + if (usedMemoryRaw) + usedMemory = new BufferValue((usedMemoryRaw / 1024) as Integer, BufferUnit.k) + jobInfo.usedResources = new ResourceSet(usedMemory, requestedCores, null, usedWalltime, null, null, null) + + // jobInfo.cpuTime // acct_cpu?? + // jobInfo.runTime = safelyCastToDuration(job["JB_total_runtime"]) // ? + + /** Status info */ + // State is not set here! + jobInfo.exitCode = safelyCastToInteger(extractStatisticsValue(job, "exit_status")) + + /** Directories and files */ + boolean mergedLogs = job["JB_merge_stderr"] as Boolean + def outPath = job["JB_stdout_path_list"]["path_list"]["PN_path"] + jobInfo.logFile = outPath ? new File(outPath.toString()) : null + if (!mergedLogs) { + def errPath = job["JB_stderr_path_list"]["path_list"]["PN_path"] + jobInfo.errorLogFile = errPath ? new File(errPath.toString()) : null + } + + /** Timestamps */ + jobInfo.submitTime = extractAndParseTime(job, "submission_time") // The time the job entered the current queue. + jobInfo.startTime = extractAndParseTime(job, "start_time") // The time the job was started. + jobInfo.endTime = extractAndParseTime(job, "end_time") // The time the job was completed. + + return jobInfo + } + + ZonedDateTime extractAndParseTime(GPathResult job, String id) { + safelyParseTime((extractStatisticsValue(job, id) as Double) as Integer) + } + + Object extractStatisticsValue(GPathResult job, String id) { + GPathResult usageList = job["JB_ja_tasks"]["ulong_sublist"]["JAT_scaled_usage_list"] as GPathResult + GPathResult times = usageList.children().findAll { GPathResult it -> it.name() == "scaled" } + def entry = times.collectEntries { + [it["UA_name"]?.toString(), it["UA_value"].toString()] + }[id] + entry + } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy index e71567c4..7808f099 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy @@ -9,14 +9,7 @@ package de.dkfz.roddy.execution.jobs.cluster.slurm import com.google.common.collect.LinkedHashMultimap import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService -import de.dkfz.roddy.execution.io.ExecutionResult -import de.dkfz.roddy.execution.jobs.BEJob -import de.dkfz.roddy.execution.jobs.BEJobID -import de.dkfz.roddy.execution.jobs.Command -import de.dkfz.roddy.execution.jobs.GenericJobInfo -import de.dkfz.roddy.execution.jobs.JobManagerOptions -import de.dkfz.roddy.execution.jobs.JobState -import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager +import de.dkfz.roddy.execution.jobs.* import de.dkfz.roddy.execution.jobs.cluster.GridEngineBasedJobManager import groovy.transform.CompileStatic @@ -58,17 +51,17 @@ class SlurmJobManager extends GridEngineBasedJobManager { } @Override - String getQueryJobStatesCommand() { + String getQueryCommandForJobInfo() { return null } @Override - String getExtendedQueryJobStatesCommand() { + String getQueryCommandForExtendedJobInfo() { return null } @Override - GenericJobInfo parseGenericJobInfo(String command) { + ExtendedJobInfo parseGenericJobInfo(String command) { return null } @@ -111,4 +104,9 @@ class SlurmJobManager extends GridEngineBasedJobManager { void createComputeParameter(ResourceSet resourceSet, LinkedHashMultimap parameters) { } + + @Override + Map queryExtendedJobInfo(List jobIDs) { + return null + } } \ No newline at end of file diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy index 8d907050..ae4607e9 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy @@ -24,22 +24,22 @@ class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManager queryJobStates(List jobIDs) { + Map queryJobInfo(List jobIDs) { + return [:] + } + + @Override + Map queryExtendedJobInfo(List jobIDs) { return [:] } @@ -158,17 +158,17 @@ class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManager queryExtendedJobStateById(List jobIds) { + Map queryExtendedJobStateById(List jobIds) { return [:] } } diff --git a/src/test/groovy/de/dkfz/roddy/batcheuphoria/BETestBaseSpec.groovy b/src/test/groovy/de/dkfz/roddy/batcheuphoria/BETestBaseSpec.groovy new file mode 100644 index 00000000..261442ff --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/batcheuphoria/BETestBaseSpec.groovy @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). + * + * Distributed under the MIT License (license terms are at https://www.github.com/TheRoddyWMS/BatchEuphoria/LICENSE.txt). + */ + +package de.dkfz.roddy.batcheuphoria + + +import spock.lang.Specification + +/** + * Base Spock spec for BE Spock specs + * Contains commonly used test methods + */ +class BETestBaseSpec extends Specification { + + static final File getResourceFile(Class _class, String file) { + String subDir = _class.package.name.replace(".", "/") + new File("src/test/resources/${subDir}/", file) + } + + def "test getResourceFile"() { + when: + File resourceFile = getResourceFile(BETestBaseSpec, "getResourceFileTest.txt") + String text = resourceFile.text + + then: + resourceFile.exists() + text == "abc" + } +} diff --git a/src/test/groovy/de/dkfz/roddy/batcheuphoria/jobs/JobManagerOptionsTest.groovy b/src/test/groovy/de/dkfz/roddy/batcheuphoria/jobs/JobManagerOptionsTest.groovy index f031b2b6..28ca2e02 100644 --- a/src/test/groovy/de/dkfz/roddy/batcheuphoria/jobs/JobManagerOptionsTest.groovy +++ b/src/test/groovy/de/dkfz/roddy/batcheuphoria/jobs/JobManagerOptionsTest.groovy @@ -23,7 +23,6 @@ class JobManagerOptionsTest { def parms = JobManagerOptions.create().build() assert parms.trackOnlyStartedJobs == false assert parms.updateInterval == Duration.ofMinutes(5) - assert parms.createDaemon == false assert !parms.passEnvironment } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/BEIntegrationTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/BEIntegrationTest.groovy index 0a6afc5d..15d7b26c 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/BEIntegrationTest.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/BEIntegrationTest.groovy @@ -18,7 +18,6 @@ import de.dkfz.roddy.tools.BufferValue import groovy.transform.CompileStatic import org.junit.BeforeClass import org.junit.Test -import org.xml.sax.SAXParseException import java.time.Duration @@ -71,7 +70,6 @@ class BEIntegrationTest { return system.loadClass().getDeclaredConstructor(BEExecutionService, JobManagerOptions) .newInstance(getExecutionServiceFor(system), JobManagerOptions.create() - .setCreateDaemon(false) .build() ) as BatchEuphoriaJobManager } @@ -148,14 +146,14 @@ class BEIntegrationTest { List lastStates = [] while (sleep > 0 && !allJobsInCorrectState) { lastStates.clear() - def status = jobManager.queryJobStatus(jobList, true) + def jobInfo = jobManager.queryJobInfoByJob(jobList) allJobsInCorrectState = true for (BEJob job in jobList) { - allJobsInCorrectState &= listOfStatesToCheck.contains(status[job]) - lastStates << status[job] + allJobsInCorrectState &= listOfStatesToCheck.contains(jobInfo[job]) + lastStates << jobInfo[job].jobState } if (!allJobsInCorrectState) { - assert status.values().join(" ").find(JobState.FAILED.name()) != JobState.FAILED.name() + assert jobInfo.values().join(" ").find(JobState.FAILED.name()) != JobState.FAILED.name() sleep-- } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManagerSpec.groovy new file mode 100644 index 00000000..0ecb7b5b --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManagerSpec.groovy @@ -0,0 +1,174 @@ +package de.dkfz.roddy.execution.jobs + +import de.dkfz.roddy.TestExecutionService +import de.dkfz.roddy.execution.jobs.cluster.JobManagerImplementationBaseSpec +import spock.lang.Shared + +import static de.dkfz.roddy.execution.jobs.JobState.* + +class BatchEuphoriaJobManagerSpec extends JobManagerImplementationBaseSpec { + + /** + * We will only test non abstract methods in the BatchEuphoriaJobManager class (which is abstract for reasons) + * For this, we will first create a custom test class, overriding some special methods necessary for our test + * cases. Afterwards, we create a Spy() of this class, which can then be utilized in our tests. + */ + abstract class TestBEJobManager extends BatchEuphoriaJobManager { + TestBEJobManager() { + super( + new TestExecutionService("test", "test"), + new JobManagerOptionsBuilder().build() + ) + } + + @Override + Map queryJobInfo(List jobIDs) { + // For the tests, it actually does not matter if the result contains JobInfo or ExtendedJobInfo objects + return queryExtendedJobInfo(jobIDs) as Map + } + + @Override + Map queryExtendedJobInfo(List jobIDs) { + [ + "1000": RUNNING, + "1001": SUSPENDED, + "1002": COMPLETED_SUCCESSFUL, + "1003": FAILED, + ].collectEntries { + String _id, JobState state -> + BEJobID id = new BEJobID(_id) + [id, new ExtendedJobInfo(id, state)] + } as Map + } + } + + def "test assertListIsValid"(list) { + when: + BatchEuphoriaJobManager.assertListIsValid(list) + + then: + thrown(AssertionError) + + where: + list | _ + null | _ + [] | _ + [null] | _ + } + + @Shared + TestBEJobManager jobManager = Spy() + + def "test queryJobInfoByJob (query for single object, also tests query by ID!)"(String id, JobState expectedState) { + when: + JobInfo result = jobManager.queryJobInfoByJob(new BEJob(new BEJobID(id), jobManager)) + + then: + result // There is ALWAYS a result! + result.jobState == expectedState + + where: + id | expectedState + "999" | UNKNOWN + "1000" | RUNNING + "1001" | SUSPENDED + "1002" | COMPLETED_SUCCESSFUL + "1003" | FAILED + } + + def "test queryJobInfoByJob"(String id, JobState expectedState) { + given: + def jobID = new BEJobID(id) + def job = new BEJob(jobID, jobManager) + + when: + def result = jobManager.queryJobInfoByJob([job]) + + then: + result.size() == 1 + result[job].jobState == expectedState + + where: + id | expectedState + "999" | UNKNOWN + "1000" | RUNNING + } + + def "test queryJobInfoByJob with null id or empty list"(List input) { + + when: + jobManager.queryJobInfoByJob(input) + + then: + thrown(AssertionError) + + where: + input | _ + null | _ + [] | _ + [null] | _ + } + + def "test queryAllJobInfo (same like above with null or [])"() { + expect: + jobManager.queryAllJobInfo().size() == 4 + } + + def "test queryExtendedJobInfoByJob (query for single object, also tests query by ID!)"() { + when: + JobInfo result = jobManager.queryExtendedJobInfoByJob(new BEJob(new BEJobID(id), jobManager)) + + then: + result // There is ALWAYS a result! + result.jobState == expectedState + + where: + id | expectedState + "999" | UNKNOWN + "1000" | RUNNING + "1001" | SUSPENDED + "1002" | COMPLETED_SUCCESSFUL + "1003" | FAILED + } + + def "test queryExtendedJobInfoByJob (will also test query by id"() { + given: + def jobID = new BEJobID(id) + def job = new BEJob(jobID, jobManager) + + when: + def result = jobManager.queryExtendedJobInfoByJob([job]) + + then: + result.size() == 1 + result[job].jobState == expectedState + + where: + id | expectedState + "999" | UNKNOWN + "1000" | RUNNING + "1001" | SUSPENDED + "1002" | COMPLETED_SUCCESSFUL + "1003" | FAILED + } + + def "test queryExtendedJobInfoByJob with null id or empty list"(List input) { + + when: + jobManager.queryExtendedJobInfoByJob(input) + + then: + thrown(AssertionError) + + where: + input | _ + null | _ + [] | _ + [null] | _ + } + + def "test queryAllExtendedJobInfo"() { + expect: + jobManager.queryAllExtendedJobInfo().size() == 4 + } +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfoTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfoTest.groovy new file mode 100644 index 00000000..7b2c0bc1 --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfoTest.groovy @@ -0,0 +1,14 @@ +package de.dkfz.roddy.execution.jobs + +import spock.lang.Specification + +class ExtendedJobInfoTest extends Specification { + def "test @EqualsAndHashCode annotation with differing base class"() { + when: + def g1 = new ExtendedJobInfo(new BEJobID("1000")) + def g2 = new ExtendedJobInfo(new BEJobID("1001")) + + then: + g1 != g2 + } +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommandTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommandTest.groovy index 645ddf15..aaea9629 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommandTest.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommandTest.groovy @@ -14,7 +14,7 @@ class SubmissionCommandTest extends Specification { JobManagerOptions.create().setPassEnvironment(passEnvironment).build()) { @Override - Map queryExtendedJobStateById(List jobIds) { + Map queryExtendedJobStateById(List jobIds) { return null } @@ -54,12 +54,12 @@ class SubmissionCommandTest extends Specification { } @Override - String getQueryJobStatesCommand() { + String getQueryCommandForJobInfo() { return null } @Override - String getExtendedQueryJobStatesCommand() { + String getQueryCommandForExtendedJobInfo() { return null } @@ -69,7 +69,7 @@ class SubmissionCommandTest extends Specification { } @Override - GenericJobInfo parseGenericJobInfo(String command) { + ExtendedJobInfo parseGenericJobInfo(String command) { return null } @@ -99,7 +99,12 @@ class SubmissionCommandTest extends Specification { } @Override - protected Map queryJobStates(List jobIDs) { + Map queryJobInfo(List jobIDs) { + return null + } + + @Override + Map queryExtendedJobInfo(List jobIDs) { return null } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManagerSpec.groovy index 02272a77..e556ddc0 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManagerSpec.groovy @@ -6,36 +6,55 @@ package de.dkfz.roddy.execution.jobs.cluster +import de.dkfz.roddy.BEException import org.slf4j.Logger import spock.lang.Specification import java.lang.reflect.Field import java.lang.reflect.Method import java.lang.reflect.Modifier +import java.time.Duration class ClusterJobManagerSpec extends Specification { - private static Method prepareLogger(Logger log) { - Method method = ClusterJobManager.class.getDeclaredMethod("withCaughtAndLoggedException", Closure) - method.setAccessible(true) + void "test parseColonSeparatedHHMMSSDuration, parse duration"() { + expect: + ClusterJobManager.parseColonSeparatedHHMMSSDuration(input) == parsedDuration - Field f = ClusterJobManager.class.getDeclaredField("log") + where: + input | parsedDuration + "00:00:00" | Duration.ofSeconds(0) + "24:00:00" | Duration.ofHours(24) + "119:00:00" | Duration.ofHours(119) + } + + void "test parseColonSeparatedHHMMSSDuration, parse duration fails"() { + when: + ClusterJobManager.parseColonSeparatedHHMMSSDuration("02:42") + + then: + BEException e = thrown(BEException) + e.message == "Duration string is not of the format HH+:MM:SS: '02:42'" + } + + private static Method prepareClassLogger(Class _class, Logger log) { + Field f = _class.getDeclaredField("log") f.setAccessible(true) - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(f, f.getModifiers() & ~Modifier.FINAL); + + Field modifiersField = f.class.getDeclaredField("modifiers") + modifiersField.setAccessible(true) + modifiersField.setInt(f, f.getModifiers() & ~Modifier.FINAL) f.set(null, log) - return method } - def "test catchExceptionAndLog throws exception"() { + def "test withCaughtAndLoggedException throws exception"() { given: Logger log = Mock(Logger) - Method method = prepareLogger(log) + prepareClassLogger(ClusterJobManager, log) when: - Object result = method.invoke(null, { throw new Exception("123"); return "ABC" }) + Object result = ClusterJobManager.withCaughtAndLoggedException { throw new Exception("123") } then: result == null @@ -43,13 +62,13 @@ class ClusterJobManagerSpec extends Specification { 1 * log.warn(_) } - def "test catchExceptionAndLog returns value"() { + def "test withCaughtAndLoggedException returns value"() { given: Logger log = Mock(Logger) - Method method = prepareLogger(log) + prepareClassLogger(ClusterJobManager, log) when: - Object result = method.invoke(null, { return "ABC" }) + Object result = ClusterJobManager.withCaughtAndLoggedException { return "ABC" } then: result == "ABC" diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBaseJobManagerTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBaseJobManagerTest.groovy deleted file mode 100644 index 1a581c54..00000000 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBaseJobManagerTest.groovy +++ /dev/null @@ -1,200 +0,0 @@ -package de.dkfz.roddy.execution.jobs.cluster - -import com.google.common.collect.LinkedHashMultimap -import de.dkfz.roddy.config.JobLog -import de.dkfz.roddy.config.ResourceSet -import de.dkfz.roddy.config.ResourceSetSize -import de.dkfz.roddy.execution.jobs.BEJob -import de.dkfz.roddy.execution.jobs.Command -import de.dkfz.roddy.execution.jobs.GenericJobInfo -import de.dkfz.roddy.execution.jobs.JobManagerOptions -import de.dkfz.roddy.execution.jobs.JobState -import de.dkfz.roddy.execution.jobs.TestHelper -import de.dkfz.roddy.execution.jobs.cluster.pbs.PBSCommand -import de.dkfz.roddy.execution.jobs.cluster.pbs.PBSJobManager -import de.dkfz.roddy.tools.BufferUnit -import de.dkfz.roddy.tools.BufferValue -import de.dkfz.roddy.tools.TimeUnit -import groovy.transform.CompileStatic -import org.junit.Before -import org.junit.Test - -@CompileStatic -class GridEngineBaseJobManagerTest { - - GridEngineBasedJobManager jobManager - - @Before - void setUp() throws Exception { - jobManager = new GridEngineBasedJobManager(TestHelper.makeExecutionService(), JobManagerOptions.create().build()) { - @Override - void createComputeParameter(ResourceSet resourceSet, LinkedHashMultimap parameters) { - - } - - @Override - void createQueueParameter(LinkedHashMultimap parameters, String queue) { - - } - - @Override - void createWalltimeParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) { - - } - - @Override - void createMemoryParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) { - - } - - @Override - void createStorageParameters(LinkedHashMultimap parameters, ResourceSet resourceSet) { - - } - - @Override - String getJobIdVariable() { - return null - } - - @Override - String getJobNameVariable() { - return null - } - - @Override - String getQueueVariable() { - return null - } - - @Override - String getNodeFileVariable() { - return null - } - - @Override - String getSubmitHostVariable() { - return null - } - - @Override - String getSubmitDirectoryVariable() { - return null - } - - @Override - String getQueryJobStatesCommand() { - return null - } - - @Override - String getExtendedQueryJobStatesCommand() { - return null - } - - @Override - GenericJobInfo parseGenericJobInfo(String command) { - return null - } - - @Override - protected Command createCommand(BEJob job) { - return null - } - - @Override - protected String parseJobID(String commandOutput) { - return null - } - - @Override - protected JobState parseJobState(String stateString) { - return null - } - } - } - - private BEJob makeJob(Map mapOfParameters) { - BEJob job = new BEJob(null, "Test", new File("/tmp/test.sh"), null, null, new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 4, 1, new TimeUnit("1h"), null, null, null), [], mapOfParameters, jobManager, JobLog.none(), null) - job - } - - @Test - void testAssembleDependencyStringWithoutDependencies() throws Exception { - def mapOfVars = ["a": "a", "b": "b"] - GridEngineBasedCommand cmd = new GridEngineBasedCommand(jobManager, makeJob(mapOfVars), - "jobName", null, mapOfVars, null, "/tmp/test.sh") { - @Override - protected String getDependsSuperParameter() { - return null - } - - @Override - protected String getDependencyParameterName() { - return null - } - - @Override - protected String getDependencyOptionSeparator() { - return null - } - - @Override - protected String getDependencyIDSeparator() { - return null - } - - @Override - protected String getJobNameParameter() { - return null - } - - @Override - protected String getHoldParameter() { - return null - } - - @Override - protected String getAccountParameter(String account) { - return null - } - - @Override - protected String getWorkingDirectory() { - return null - } - - @Override - protected String getLoggingParameter(JobLog jobLog) { - return null - } - - @Override - protected String getEmailParameter(String address) { - return null - } - - @Override - protected String getGroupListParameter(String groupList) { - return null - } - - @Override - protected String getUmaskString(String umask) { - return null - } - - @Override - protected String getAdditionalCommandParameters() { - return null - } - - @Override - protected String assembleVariableExportParameters() { - return null - } - } - assert cmd.assembleDependencyString([]) == "" - } - -} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManagerSpec.groovy index d714efdc..c3a75f14 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManagerSpec.groovy @@ -9,17 +9,18 @@ class GridEngineBasedJobManagerSpec extends Specification { output == GridEngineBasedJobManager.getExecutionHosts(input) where: - input || output - null || null - "" || null - "asdf" || ["asdf"] - "asdf+asdf" || ["asdf"] - "asdf+qwertz" || ["asdf", "qwertz"] - "asdf/5+asdf" || ["asdf"] - "asdf/10+asdf/1" || ["asdf"] - "asdf/3+qwertz/7" || ["asdf", "qwertz"] - "asdf+qwertz/2" || ["asdf", "qwertz"] - "asdf+qwertz/2+yxcv/6" || ["asdf", "qwertz", "yxcv"] + input || output + null || [] + "" || [] + "asdf" || ["asdf"] + "asdf+asdf" || ["asdf"] + "asdf+qwertz" || ["asdf", "qwertz"] + "asdf/5+asdf" || ["asdf"] + "asdf/10+asdf/1" || ["asdf"] + "asdf/3+qwertz/7" || ["asdf", "qwertz"] + "asdf+qwertz/2" || ["asdf", "qwertz"] + "asdf+qwertz/2+yxcv/6" || ["asdf", "qwertz", "yxcv"] + "exec-host/0+exec-host/1+exec-host/2" || ["exec-host"] } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineQstatParserTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineQstatParserTest.groovy deleted file mode 100644 index 4b628a96..00000000 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineQstatParserTest.groovy +++ /dev/null @@ -1,429 +0,0 @@ -/* - * Copyright (c) 2017 eilslabs. - * - * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). - */ - -package de.dkfz.roddy.execution.jobs.cluster - -import de.dkfz.roddy.TestExecutionService -import de.dkfz.roddy.execution.jobs.JobManagerOptions -import de.dkfz.roddy.execution.jobs.cluster.pbs.PBSJobManager -import groovy.transform.CompileDynamic -import groovy.transform.CompileStatic -import org.junit.Test - -/** - * Created by heinold on 26.03.17. - */ -@CompileStatic -class GridEngineQstatParserTest { - - final static String output1 = """\ -Job Id: 14973441.tbi-pbs-ng.inet.dkfz-heidelberg.de - Job_Name = r170623_063520503_A056-XQH5TD_createControlBafPlots - Job_Owner = otproddy@tbi-pbs4.inet.dkfz-heidelberg.de - resources_used.cput = 00:02:38 - resources_used.energy_used = 0 - resources_used.mem = 472772kb - resources_used.vmem = 756656kb - resources_used.walltime = 00:03:44 - job_state = C - queue = medium - server = tbi-pbs-ng.inet.dkfz-heidelberg.de - Checkpoint = u - ctime = Fri Jun 23 06:35:59 2017 - Error_Path = tbi-pbs4:/icgc/dkfzlsdf/project/hipo/hipo_A056/sequencing/who -\tle_genome_sequencing/view-by-pid/A056-XQH5TD/cnv_results/paired/metast -\tasis_blood/results_ACEseqWorkflow-1.2.8-1_v1_0_2017-06-23_06h33_+0200/ -\troddyExecutionStore/exec_170623_063520503_otproddy_WGS/r170623_0635205 -\t03_A056-XQH5TD_createControlBafPlots.e14973441 - exec_host = tbi-dsx50/22 - group_list = B080 - Hold_Types = n - Join_Path = oe - Keep_Files = n - Mail_Points = a - mtime = Fri Jun 23 10:59:30 2017 - Output_Path = tbi-pbs4:/icgc/dkfzlsdf/project/hipo/hipo_A056/sequencing/wh -\tole_genome_sequencing/view-by-pid/A056-XQH5TD/cnv_results/paired/metas -\ttasis_blood/results_ACEseqWorkflow-1.2.8-1_v1_0_2017-06-23_06h33_+0200 -\t/roddyExecutionStore/exec_170623_063520503_otproddy_WGS/r170623_063520 -\t503_A056-XQH5TD_createControlBafPlots.o14973441 - Priority = 0 - qtime = Sat Jun 3 06:35:59 2017 - Rerunable = True - Resource_List.mem = 5120mb - Resource_List.walltime = 01:00:00 - session_id = 34432 - euser = otproddy - egroup = B080 - queue_type = E - etime = Fri Jun 23 10:54:49 2017 - exit_status = 0 - submit_args = -N r170623_063520503_A056-XQH5TD_createControlBafPlots -o /i -\tcgc/dkfzlsdf/project/hipo/hipo_A056/sequencing/whole_genome_sequencing -\t/view-by-pid/A056-XQH5TD/cnv_results/paired/metastasis_blood/results_A -\tCEseqWorkflow-1.2.8-1_v1_0_2017-06-23_06h33_+0200/roddyExecutionStore/ -\texec_170623_063520503_otproddy_WGS -j oe -W group_list=B080 -W umask=0 -\t07 -l mem=5120M -l walltime=00:01:00:00 -W depend=afterok:14973440.tbi -\t-pbs-ng.inet.dkfz-heidelberg.de:14973412.tbi-pbs-ng.inet.dkfz-heidelbe -\trg.de -v WRAPPED_SCRIPT=/icgc/dkfzlsdf/project/hipo/hipo_A056/sequenci -\tng/whole_genome_sequencing/view-by-pid/A056-XQH5TD/cnv_results/paired/ -\tmetastasis_blood/results_ACEseqWorkflow-1.2.8-1_v1_0_2017-06-23_06h33_ -\t+0200/roddyExecutionStore/exec_170623_063520503_otproddy_WGS/analysisT -\tools/copyNumberEstimationWorkflow/createControlBafPlots.sh, -\tPARAMETER_FILE=/icgc/dkfzlsdf/project/hipo/hipo_A056/sequencing/whole -\t_genome_sequencing/view-by-pid/A056-XQH5TD/cnv_results/paired/metastas -\tis_blood/results_ACEseqWorkflow-1.2.8-1_v1_0_2017-06-23_06h33_+0200/ro -\tddyExecutionStore/exec_170623_063520503_otproddy_WGS/r170623_063520503 -\t_A056-XQH5TD_createControlBafPlots_120.parameters /icgc/dkfzlsdf/proje -\tct/hipo/hipo_A056/sequencing/whole_genome_sequencing/view-by-pid/A056- -\tXQH5TD/cnv_results/paired/metastasis_blood/results_ACEseqWorkflow-1.2. -\t8-1_v1_0_2017-06-23_06h33_+0200/roddyExecutionStore/exec_170623_063520 -\t503_otproddy_WGS/analysisTools/roddyTools/wrapInScript.sh - umask = 7 - start_time = Fri Jun 23 10:55:46 2017 - start_count = 1 - fault_tolerant = False - comp_time = Fri Jun 23 10:59:30 2017 - job_radix = 0 - total_runtime = 279.826458 - submit_host = tbi-pbs4.inet.dkfz-heidelberg.de - request_version = 1 - -Job Id: 14973792.tbi-pbs-ng.inet.dkfz-heidelberg.de - Job_Name = yapima - Job_Owner = pastor@tbi-worker.inet.dkfz-heidelberg.de - job_state = Q - queue = long - server = tbi-pbs-ng.inet.dkfz-heidelberg.de - Checkpoint = u - ctime = Fri Jun 23 11:02:42 2017 - Error_Path = tbi-worker:/ibios/co02/xavier/eo/yapima/yapima.e14973792 - Hold_Types = n - Join_Path = oe - Keep_Files = n - Mail_Points = a - Mail_Users = output2.pastorhostench@dkfz-heidelberg.de - mtime = Fri Jun 23 11:02:42 2017 - Output_Path = tbi-worker:/ibios/co02/xavier/eo/yapima/yapima.o14973792 - Priority = 0 - qtime = Fri Jun 23 11:02:42 2017 - Rerunable = True - Resource_List.mem = 10gb - Resource_List.nodect = 1 - Resource_List.nodes = 1:ppn=1 - Resource_List.walltime = 03:00:00 - euser = pastor - egroup = B080 - queue_type = E - etime = Fri Jun 23 11:02:42 2017 - submit_args = -o /ibios/co02/xavier/eo/yapima -j oe -M output2.pastorhostench@dk -\tfz-heidelberg.de -N yapima -l walltime=3:00:00,nodes=1:ppn=1, -\tmem=10g -v CONFIG_FILE=/icgc/dkfzlsdf/analysis/hipo/hipo_043/methylat -\tion_data_H043/config_yapima.sh /home/pastor/pipelines/run/yapima/proce -\tss450k.sh - fault_tolerant = False - job_radix = 0 - submit_host = tbi-worker.inet.dkfz-heidelberg.de - request_version = 1 - -Job Id: 14973766.tbi-pbs-ng.inet.dkfz-heidelberg.de - Job_Name = r170623_105203487_A017-WQL2_snvCalling - Job_Owner = otproddy@tbi-pbs4.inet.dkfz-heidelberg.de - resources_used.cput = 00:10:06 - resources_used.energy_used = 0 - resources_used.mem = 146892kb - resources_used.vmem = 261756kb - resources_used.walltime = 00:10:03 - job_state = R - queue = verylong - server = tbi-pbs-ng.inet.dkfz-heidelberg.de - Checkpoint = u - ctime = Fri Jun 23 10:52:20 2017 - depend = beforeok:14973775.tbi-pbs-ng.inet.dkfz-heidelberg.de - Error_Path = tbi-pbs4:/icgc/dkfzlsdf/project/hipo/hipo_A017/sequencing/who -\tle_genome_sequencing/view-by-pid/A017-WQL2/snv_results/paired/tumor_co -\tntrol02/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_10h50_+02 -\t00/roddyExecutionStore/exec_170623_105203487_otproddy_WGS/r170623_1052 -\t03487_A017-WQL2_snvCalling.e14973766 - exec_host = tbi-dsx55/25 - group_list = B080 - Hold_Types = n - Join_Path = oe - Keep_Files = n - Mail_Points = a - mtime = Fri Jun 23 10:53:47 2017 - Output_Path = tbi-pbs4:/icgc/dkfzlsdf/project/hipo/hipo_A017/sequencing/wh -\tole_genome_sequencing/view-by-pid/A017-WQL2/snv_results/paired/tumor_c -\tontrol02/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_10h50_+0 -\t200/roddyExecutionStore/exec_170623_105203487_otproddy_WGS/r170623_105 -\t203487_A017-WQL2_snvCalling.o14973766 - Priority = 0 - qtime = Fri Jun 23 10:52:20 2017 - Rerunable = True - Resource_List.mem = 4096mb - Resource_List.nodect = 1 - Resource_List.nodes = 1:ppn=1 - Resource_List.walltime = 48:00:00 - session_id = 45434 - euser = otproddy - egroup = B080 - queue_type = E - etime = Fri Jun 23 10:52:20 2017 - submit_args = -N r170623_105203487_A017-WQL2_snvCalling -o /icgc/dkfzlsdf/ -\tproject/hipo/hipo_A017/sequencing/whole_genome_sequencing/view-by-pid/ -\tA017-WQL2/snv_results/paired/tumor_control02/results_SNVCallingWorkflo -\tw-1.0.166-1_v1_0_2017-06-23_10h50_+0200/roddyExecutionStore/exec_17062 -\t3_105203487_otproddy_WGS -j oe -W group_list=B080 -W umask=007 -l mem= -\t4096m -l nodes=1:ppn=1 -l walltime=48:00:00 -v WRAPPED_SCRIPT=/icgc/dk -\tfzlsdf/project/hipo/hipo_A017/sequencing/whole_genome_sequencing/view- -\tby-pid/A017-WQL2/snv_results/paired/tumor_control02/results_SNVCalling -\tWorkflow-1.0.166-1_v1_0_2017-06-23_10h50_+0200/roddyExecutionStore/exe -\tc_170623_105203487_otproddy_WGS/analysisTools/snvPipeline/snvCalling.output1 -\th, -\tPARAMETER_FILE=/icgc/dkfzlsdf/project/hipo/hipo_A017/sequencing/whole -\t_genome_sequencing/view-by-pid/A017-WQL2/snv_results/paired/tumor_cont -\trol02/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_10h50_+0200 -\t/roddyExecutionStore/exec_170623_105203487_otproddy_WGS/r170623_105203 -\t487_A017-WQL2_snvCalling_50.parameters /icgc/dkfzlsdf/project/hipo/hip -\to_A017/sequencing/whole_genome_sequencing/view-by-pid/A017-WQL2/snv_re -\tsults/paired/tumor_control02/results_SNVCallingWorkflow-1.0.166-1_v1_0 -\t_2017-06-23_10h50_+0200/roddyExecutionStore/exec_170623_105203487_otpr -\toddy_WGS/analysisTools/roddyTools/wrapInScript.sh - umask = 7 - start_time = Fri Jun 23 10:53:47 2017 - Walltime.Remaining = 172137 - start_count = 1 - fault_tolerant = False - job_radix = 0 - submit_host = tbi-pbs4.inet.dkfz-heidelberg.de - request_version = 1 - -Job Id: 14973745.tbi-pbs-ng.inet.dkfz-heidelberg.de - Job_Name = r170623_10495311_A017-39867J_snvAnnotation - Job_Owner = otproddy@tbi-pbs4.inet.dkfz-heidelberg.de - job_state = H - queue = verylong - server = tbi-pbs-ng.inet.dkfz-heidelberg.de - Checkpoint = u - ctime = Fri Jun 23 10:50:14 2017 - depend = afterok:14973744.tbi-pbs-ng.inet.dkfz-heidelberg.de, -\tbeforeok:14973746.tbi-pbs-ng.inet.dkfz-heidelberg.de - Error_Path = tbi-pbs4:/icgc/dkfzlsdf/project/hipo/hipo_A017/sequencing/who -\tle_genome_sequencing/view-by-pid/A017-39867J/snv_results/paired/metast -\tasis_control02/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_10 -\th48_+0200/roddyExecutionStore/exec_170623_10495311_otproddy_WGS/r17062 -\t3_10495311_A017-39867J_snvAnnotation.e14973745 - group_list = B080 - Hold_Types = output1 - Join_Path = oe - Keep_Files = n - Mail_Points = a - mtime = Fri Jun 23 10:50:15 2017 - Output_Path = tbi-pbs4:/icgc/dkfzlsdf/project/hipo/hipo_A017/sequencing/wh -\tole_genome_sequencing/view-by-pid/A017-39867J/snv_results/paired/metas -\ttasis_control02/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_1 -\t0h48_+0200/roddyExecutionStore/exec_170623_10495311_otproddy_WGS/r1706 -\t23_10495311_A017-39867J_snvAnnotation.o14973745 - Priority = 0 - qtime = Fri Jun 23 10:50:14 2017 - Rerunable = True - Resource_List.mem = 4096mb - Resource_List.nodect = 1 - Resource_List.nodes = 1:ppn=2 - Resource_List.walltime = 180:00:00 - euser = otproddy - egroup = B080 - queue_type = E - submit_args = -N r170623_10495311_A017-39867J_snvAnnotation -o /icgc/dkfzl -\tsdf/project/hipo/hipo_A017/sequencing/whole_genome_sequencing/view-by- -\tpid/A017-39867J/snv_results/paired/metastasis_control02/results_SNVCal -\tlingWorkflow-1.0.166-1_v1_0_2017-06-23_10h48_+0200/roddyExecutionStore -\t/exec_170623_10495311_otproddy_WGS -j oe -W group_list=B080 -W umask=0 -\t07 -l mem=4096m -l nodes=1:ppn=2 -l walltime=180:00:00 -W depend=after -\tok:14973744.tbi-pbs-ng.inet.dkfz-heidelberg.de -v WRAPPED_SCRIPT=/icgc -\t/dkfzlsdf/project/hipo/hipo_A017/sequencing/whole_genome_sequencing/vi -\tew-by-pid/A017-39867J/snv_results/paired/metastasis_control02/results_ -\tSNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_10h48_+0200/roddyExecutio -\tnStore/exec_170623_10495311_otproddy_WGS/analysisTools/snvPipeline/snv -\tAnnotation.sh, -\tPARAMETER_FILE=/icgc/dkfzlsdf/project/hipo/hipo_A017/sequencing/whole -\t_genome_sequencing/view-by-pid/A017-39867J/snv_results/paired/metastas -\tis_control02/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_10h4 -\t8_+0200/roddyExecutionStore/exec_170623_10495311_otproddy_WGS/r170623_ -\t10495311_A017-39867J_snvAnnotation_60.parameters /icgc/dkfzlsdf/projec -\tt/hipo/hipo_A017/sequencing/whole_genome_sequencing/view-by-pid/A017-3 -\t9867J/snv_results/paired/metastasis_control02/results_SNVCallingWorkfl -\tow-1.0.166-1_v1_0_2017-06-23_10h48_+0200/roddyExecutionStore/exec_1706 -\t23_10495311_otproddy_WGS/analysisTools/roddyTools/wrapInScript.sh - umask = 7 - fault_tolerant = False - job_radix = 0 - submit_host = tbi-pbs4.inet.dkfz-heidelberg.de - request_version = 1 - -""" - - - final static String output2 = """ -Job Id: 14973826.tbi-pbs-ng.inet.dkfz-heidelberg.de - Job_Name = r170623_111000536_XI061_9EP29_snvDeepAnnotation - Job_Owner = otproddy@tbi-pbs4.inet.dkfz-heidelberg.de - job_state = H - queue = long - server = tbi-pbs-ng.inet.dkfz-heidelberg.de - Checkpoint = u - ctime = Fri Jun 23 11:10:25 2017 - depend = afterok:14973825.tbi-pbs-ng.inet.dkfz-heidelberg.de, - beforeok:14973827.tbi-pbs-ng.inet.dkfz-heidelberg.de - Error_Path = tbi-pbs4:/icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/sequ - encing/whole_genome_sequencing/view-by-pid/XI061_9EP29/snv_results/pai - red/tumor_blood/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_1 - 1h08_+0200/roddyExecutionStore/exec_170623_111000536_otproddy_WGS/r170 - 623_111000536_XI061_9EP29_snvDeepAnnotation.e14973826 - group_list = B080 - Hold_Types = output1 - Join_Path = oe - Keep_Files = n - Mail_Points = a - mtime = Fri Jun 23 11:10:25 2017 - Output_Path = tbi-pbs4:/icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/seq - uencing/whole_genome_sequencing/view-by-pid/XI061_9EP29/snv_results/pa - ired/tumor_blood/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_ - 11h08_+0200/roddyExecutionStore/exec_170623_111000536_otproddy_WGS/r17 - 0623_111000536_XI061_9EP29_snvDeepAnnotation.o14973826 - Priority = 0 - qtime = Fri Jun 23 11:10:25 2017 - Rerunable = True - Resource_List.mem = 4096mb - Resource_List.nodect = 1 - Resource_List.nodes = 1:ppn=3 - Resource_List.walltime = 04:00:00 - euser = otproddy - egroup = B080 - queue_type = E - submit_args = -N r170623_111000536_XI061_9EP29_snvDeepAnnotation -o /icgc/ - dkfzlsdf/project/Xintern/XI061_ependymoma/sequencing/whole_genome_sequ - encing/view-by-pid/XI061_9EP29/snv_results/paired/tumor_blood/results_ - SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_11h08_+0200/roddyExecutio - nStore/exec_170623_111000536_otproddy_WGS -j oe -W group_list=B080 -W - umask=007 -l mem=4096m -l nodes=1:ppn=3 -l walltime=4:00:00 -W depend= - afterok:14973825.tbi-pbs-ng.inet.dkfz-heidelberg.de -v WRAPPED_SCRIPT= - /icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/sequencing/whole_genom - e_sequencing/view-by-pid/XI061_9EP29/snv_results/paired/tumor_blood/re - sults_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_11h08_+0200/roddyEx - ecutionStore/exec_170623_111000536_otproddy_WGS/analysisTools/tools/vc - f_pipeAnnotator.sh, - PARAMETER_FILE=/icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/sequen - cing/whole_genome_sequencing/view-by-pid/XI061_9EP29/snv_results/paire - d/tumor_blood/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_11h - 08_+0200/roddyExecutionStore/exec_170623_111000536_otproddy_WGS/r17062 - 3_111000536_XI061_9EP29_snvDeepAnnotation_61.parameters /icgc/dkfzlsdf - /project/Xintern/XI061_ependymoma/sequencing/whole_genome_sequencing/v - iew-by-pid/XI061_9EP29/snv_results/paired/tumor_blood/results_SNVCalli - ngWorkflow-1.0.166-1_v1_0_2017-06-23_11h08_+0200/roddyExecutionStore/e - xec_170623_111000536_otproddy_WGS/analysisTools/roddyTools/wrapInScrip - t.sh - umask = 7 - fault_tolerant = False - job_radix = 0 - submit_host = tbi-pbs4.inet.dkfz-heidelberg.de - request_version = 1 - -Job Id: 14973827.tbi-pbs-ng.inet.dkfz-heidelberg.de - Job_Name = r170623_111000536_XI061_9EP29_snvFilter - Job_Owner = otproddy@tbi-pbs4.inet.dkfz-heidelberg.de - job_state = H - queue = long - server = tbi-pbs-ng.inet.dkfz-heidelberg.de - Checkpoint = u - ctime = Fri Jun 23 11:10:25 2017 - depend = afterok:14973826.tbi-pbs-ng.inet.dkfz-heidelberg.de:14973824.tbi- - pbs-ng.inet.dkfz-heidelberg.de - Error_Path = tbi-pbs4:/icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/sequ - encing/whole_genome_sequencing/view-by-pid/XI061_9EP29/snv_results/pai - red/tumor_blood/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_1 - 1h08_+0200/roddyExecutionStore/exec_170623_111000536_otproddy_WGS/r170 - 623_111000536_XI061_9EP29_snvFilter.e14973827 - group_list = B080 - Hold_Types = output1 - Join_Path = oe - Keep_Files = n - Mail_Points = a - mtime = Fri Jun 23 11:10:25 2017 - Output_Path = tbi-pbs4:/icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/seq - uencing/whole_genome_sequencing/view-by-pid/XI061_9EP29/snv_results/pa - ired/tumor_blood/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_ - 11h08_+0200/roddyExecutionStore/exec_170623_111000536_otproddy_WGS/r17 - 0623_111000536_XI061_9EP29_snvFilter.o14973827 - Priority = 0 - qtime = Fri Jun 23 11:10:25 2017 - Rerunable = True - Resource_List.mem = 4096mb - Resource_List.nodect = 1 - Resource_List.nodes = 1:ppn=1 - Resource_List.walltime = 04:00:00 - euser = otproddy - egroup = B080 - queue_type = E - submit_args = -N r170623_111000536_XI061_9EP29_snvFilter -o /icgc/dkfzlsdf - /project/Xintern/XI061_ependymoma/sequencing/whole_genome_sequencing/v - iew-by-pid/XI061_9EP29/snv_results/paired/tumor_blood/results_SNVCalli - ngWorkflow-1.0.166-1_v1_0_2017-06-23_11h08_+0200/roddyExecutionStore/e - xec_170623_111000536_otproddy_WGS -j oe -W group_list=B080 -W umask=00 - 7 -l mem=4096m -l nodes=1:ppn=1 -l walltime=4:00:00 -W depend=afterok: - 14973826.tbi-pbs-ng.inet.dkfz-heidelberg.de:14973824.tbi-pbs-ng.inet.d - kfz-heidelberg.de -v WRAPPED_SCRIPT=/icgc/dkfzlsdf/project/Xintern/XI0 - 61_ependymoma/sequencing/whole_genome_sequencing/view-by-pid/XI061_9EP - 29/snv_results/paired/tumor_blood/results_SNVCallingWorkflow-1.0.166-1 - _v1_0_2017-06-23_11h08_+0200/roddyExecutionStore/exec_170623_111000536 - _otproddy_WGS/analysisTools/snvPipeline/filter_vcf.sh, - PARAMETER_FILE=/icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/sequen - cing/whole_genome_sequencing/view-by-pid/XI061_9EP29/snv_results/paire - d/tumor_blood/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_11h - 08_+0200/roddyExecutionStore/exec_170623_111000536_otproddy_WGS/r17062 - 3_111000536_XI061_9EP29_snvFilter_62.parameters /icgc/dkfzlsdf/project - /Xintern/XI061_ependymoma/sequencing/whole_genome_sequencing/view-by-p - id/XI061_9EP29/snv_results/paired/tumor_blood/results_SNVCallingWorkfl - ow-1.0.166-1_v1_0_2017-06-23_11h08_+0200/roddyExecutionStore/exec_1706 - 23_111000536_otproddy_WGS/analysisTools/roddyTools/wrapInScript.sh - umask = 7 - fault_tolerant = False - job_radix = 0 - submit_host = tbi-pbs4.inet.dkfz-heidelberg.de - request_version = 1 - -""" - - protected List getTestQstat() { - return Arrays.asList( - "job - ID prior name user jobState submit / start at queue slots ja -task - ID", - "---------------------------------------------------------------------------------------------------------------- -", - " 1187 0.75000 r140710_09 seqware r 07 / 10 / 2014 09:51:55 main.q @worker3 1", - " 1188 0.41406 r140710_09 seqware r 07 / 10 / 2014 09:51:40 main.q @worker1 1", - " 1190 0.25000 r140710_09 seqware r 07 / 10 / 2014 09:51:55 main.q @worker2 1", - " 1189 0.00000 r140710_09 seqware hqw 07 / 10 / 2014 09:51:27 1", - " 1191 0.00000 r140710_09 seqware hqw 07 / 10 / 2014 09:51:48 1", - " 1192 0.00000 r140710_09 seqware hqw 07 / 10 / 2014 09:51:48 1") - } - - @CompileDynamic - @Test - void testProcessQstatOutputFromPlainText() throws Exception { - TestExecutionService executionService = new TestExecutionService("", "") - PBSJobManager jm = new PBSJobManager(executionService, JobManagerOptions.create() - .setCreateDaemon(false) - .setUserIdForJobQueries("asdf") - .build()) - - Map> qstatReaderResultOutput1 = (Map>) GridEngineBasedJobManager.processQstatOutputFromPlainText(output1) - Map> qstatReaderResultOutput2 = (Map>) GridEngineBasedJobManager.processQstatOutputFromPlainText(output2) - - assert qstatReaderResultOutput1.size() == 4 - assert qstatReaderResultOutput1.keySet() as List == [ "14973441", "14973792", "14973766", "14973745"] - assert qstatReaderResultOutput2.size() == 2 - } - -} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpec.groovy new file mode 100644 index 00000000..fab02f28 --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpec.groovy @@ -0,0 +1,123 @@ +package de.dkfz.roddy.execution.jobs.cluster + +import com.google.common.reflect.TypeToken +import de.dkfz.roddy.TestExecutionService +import de.dkfz.roddy.batcheuphoria.BETestBaseSpec +import de.dkfz.roddy.execution.BEExecutionService +import de.dkfz.roddy.execution.io.ExecutionResult +import de.dkfz.roddy.execution.jobs.BEJobID +import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager +import de.dkfz.roddy.execution.jobs.JobManagerOptions + +import java.lang.reflect.Constructor + +/** + * Base test specification for job manager implementations. + * + * Used to enforce a common test structure for all subclasses. + */ +abstract class JobManagerImplementationBaseSpec extends BETestBaseSpec { + + protected static final BEJobID testJobID = new BEJobID("22005") + + /** + * https://stackoverflow.com/questions/3403909/get-generic-type-of-class-at-runtime + */ + private final TypeToken typeToken = new TypeToken(getClass()) {} + + private final Class jobManagerClass = typeToken.getRawType() + + T _newJobManager(BEExecutionService testExecutionService, JobManagerOptions parms) { + Constructor c = jobManagerClass.getConstructor(BEExecutionService, JobManagerOptions) + return c.newInstance(testExecutionService, parms) + } + + /** + * Create a job manager instance of type T. + * The job manager will feature a modified instance of BEExecutionService which will return + * the contents of the resource file jobStateQueryResultFile when execute() is called. + * + * For this feature to work, your JobManager implementation must have a constructor with + * BEExecutionService and JobManagerOptions as parameters. + * + * @param maxTrackingTime Duration used for job tracking filter tests + * @param resourceFileForExecuteResult The file whose content will be returned for execute() calls + * @return + */ + final T createJobManagerWithModifiedExecutionService(String resourceFileForExecuteResult) { + JobManagerOptions parms = JobManagerOptions.create().build() + final Class _class = jobManagerClass + BEExecutionService testExecutionService = [ + execute: { + String s -> new ExecutionResult(true, 0, getResourceFile(_class, resourceFileForExecuteResult).readLines(), null) + } + ] as BEExecutionService + return _newJobManager(testExecutionService, parms) + } + + final T createJobManager() { + JobManagerOptions parms = JobManagerOptions.create().build() + TestExecutionService testExecutionService = new TestExecutionService("test", "test") + return _newJobManager(testExecutionService, parms) + } + + /////////////////////////////////////////////////////////////////////////// + // Unfortunately it is not possible to override Spock test features. + // Or at least, I was not able to find a suitable way. What I propose here + // is, that we just list all necessary tests and hope for the best in the + // future. Maybe the override support comes at some point. Until then, + // we have a list in form of a big comment. + /////////////////////////////////////////////////////////////////////////// + + /* + abstract def "test submitJob"() + + abstract def "test addToListOfStartedJobs"() + + abstract def "test startHeldJobs"() + + abstract def "test killJobs"() + + // Most of the logic for the query(Extended)JobInfo methods is already done in the BatchEuphoriaJobManager spec + // Concentrate on only a couple tests for query... in your JobManager test spec. + abstract def "test queryJobInfoByJob (also tests multiple requests and by ID)"(String id, expectedState) + + abstract def "test queryAllJobInfo"() + + abstract def "test queryExtendedJobInfoByJob (also tests multiple requests and by ID)"(String id, expectedState) + + abstract def "test queryAllExtendedJobInfo"() + + abstract def "test getEnvironmentVariableGlobs"() + + abstract def "test getDefaultForHoldJobsEnabled"() + + abstract def "test isHoldJobsEnabled"() + + abstract def "test getUserEmail"() + + abstract def "test getUserMask"() + + abstract def "test getUserGroup"() + + abstract def "test getUserAccount"() + + abstract def "test executesWithoutJobSystem"() + + abstract def "test convertResourceSet"() + + abstract def "test collectJobIDsFromJobs"() + + abstract def "test extractAndSetJobResultFromExecutionResult"() + + abstract def "test createCommand"() + + abstract def "test parseJobID"() + + abstract def "test parseJobState"() + + abstract def "test executeStartHeldJobs"() + + abstract def "test executeKillJobs"() + */ +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpecSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpecSpec.groovy new file mode 100644 index 00000000..6c449c15 --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpecSpec.groovy @@ -0,0 +1,150 @@ +package de.dkfz.roddy.execution.jobs.cluster + +import de.dkfz.roddy.TestExecutionService +import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager +import de.dkfz.roddy.execution.jobs.cluster.pbs.PBSJobManager + +/** + * Test spec for JobManagerImplementationBaseSpecSpec + */ +class JobManagerImplementationBaseSpecSpec extends JobManagerImplementationBaseSpec { + + def "test createJobManagerWithModifiedExecutionService"() { + when: + BatchEuphoriaJobManager jm = createJobManagerWithModifiedExecutionService("createJobManagerWithModifiedExecutionServiceTest.txt") + def expected = ["Line1", "Line2", "Line3"] + def res0 = jm.executionService.execute("blabla").resultLines + def res1 = jm.executionService.execute("something else").resultLines + def res2 = jm.executionService.execute("just a test").resultLines + + then: + res0 == expected + res1 == expected + res2 == expected + } + + def "test createJobManager"() { + when: + BatchEuphoriaJobManager jm = createJobManager() + + then: + jm instanceof PBSJobManager + jm.getExecutionService() instanceof TestExecutionService + } + + def "test submitJob"() { + return null + } + + def "test addToListOfStartedJobs"() { + return null + } + + def "test startHeldJobs"() { + return null + } + + def "test killJobs"() { + return null + } + + def "test queryJobStateByJob"() { + return null + } + + def "test queryJobStateByID"() { + return null + } + + def "test queryJobStatesByJob"() { + return null + } + + def "test queryJobStatesByID"() { + return null + } + + def "test queryAllJobStates"() { + return null + } + + def "test queryJobStates with overdue finish date"() { + return null + } + + def "test queryExtendedJobStatesByJob"() { + return null + } + + def "test queryExtendedJobStatesById"() { + return null + } + + def "test queryExtendedJobStatesById with overdue date"() { + return null + } + + def "test getEnvironmentVariableGlobs"() { + return null + } + + def "test getDefaultForHoldJobsEnabled"() { + return null + } + + def "test isHoldJobsEnabled"() { + return null + } + + def "test getUserEmail"() { + return null + } + + def "test getUserMask"() { + return null + } + + def "test getUserGroup"() { + return null + } + + def "test getUserAccount"() { + return null + } + + def "test executesWithoutJobSystem"() { + return null + } + + def "test convertResourceSet"() { + return null + } + + def "test collectJobIDsFromJobs"() { + return null + } + + def "test extractAndSetJobResultFromExecutionResult"() { + return null + } + + def "test createCommand"() { + return null + } + + def "test parseJobID"() { + return null + } + + def "test parseJobState"() { + return null + } + + def "test executeStartHeldJobs"() { + return null + } + + def "test executeKillJobs"() { + return null + } +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy index a5c1c07c..5b51abb1 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy @@ -1,62 +1,33 @@ /* - * Copyright (c) 2017 eilslabs. + * Copyright (c) 2019 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). * - * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). + * Distributed under the MIT License (license terms are at https://www.github.com/TheRoddyWMS/BatchEuphoria/LICENSE.txt). */ package de.dkfz.roddy.execution.jobs.cluster.lsf import de.dkfz.roddy.TestExecutionService -import de.dkfz.roddy.execution.BEExecutionService -import de.dkfz.roddy.execution.io.ExecutionResult -import de.dkfz.roddy.execution.jobs.BEJobID -import de.dkfz.roddy.execution.jobs.GenericJobInfo -import de.dkfz.roddy.execution.jobs.JobManagerOptions -import de.dkfz.roddy.execution.jobs.JobState +import de.dkfz.roddy.config.ResourceSet +import de.dkfz.roddy.execution.jobs.* +import de.dkfz.roddy.execution.jobs.cluster.JobManagerImplementationBaseSpec import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.BufferValue import groovy.json.JsonSlurper import groovy.transform.CompileStatic import spock.lang.Ignore -import spock.lang.Specification import java.time.Duration import java.time.LocalDateTime import java.time.ZoneId import java.time.ZonedDateTime import java.time.format.DateTimeFormatter -import java.time.temporal.ChronoUnit +import java.time.format.DateTimeParseException -class LSFJobManagerSpec extends Specification { +class LSFJobManagerSpec extends JobManagerImplementationBaseSpec { - static final File getResourceFile(String file) { - new File("src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/", file) - } - - void "Test convertJobDetailsMapToGenericJobInfoObject"() { - - given: - def parms = JobManagerOptions.create().build() - TestExecutionService testExecutionService = new TestExecutionService("test", "test") - LSFJobManager jm = new LSFJobManager(testExecutionService, parms) - - Object parsedJson = new JsonSlurper().parseText(getResourceFile(resourceFile).text) - List records = (List) parsedJson.getAt("RECORDS") - - when: - GenericJobInfo jobInfo = jm.convertJobDetailsMapToGenericJobInfoObject(records.get(0)) - - then: - jobInfo != null - jobInfo.jobID.toString() == "22005" - jobInfo.command == expectedCommand - - where: - resourceFile | expectedJobId | expectedCommand - "queryExtendedJobStateByIdTest.json" | "22005" | "ls -l" - "queryExtendedJobStateByIdWithoutListsTest.json" | "22005" | "ls -l" - "queryExtendedJobStateByIdEmptyTest.json" | "22005" | null - } + ////////////////////////////////////// + // Time zone helpers and tests. + ////////////////////////////////////// @CompileStatic String zonedDateTimeToString(ZonedDateTime date) { @@ -90,134 +61,162 @@ class LSFJobManagerSpec extends Specification { LocalDateTime.of(2000, 5, 7, 10, 21) | "May 7 10:21 L" } - void "test parseTime"() { + ////////////////////////////////////// + // Class tests + ////////////////////////////////////// + + def "test parseTime"(String time, ZonedDateTime expected) { + expect: + createJobManager().parseTime(time) == expected + + where: + time | expected + "Jan 1 10:21" | ZonedDateTime.of(LocalDateTime.now().year, 1, 1, 10, 21, 0, 0, ZoneId.systemDefault()) + "Jan 2 10:21 L" | ZonedDateTime.of(LocalDateTime.now().year, 1, 2, 10, 21, 0, 0, ZoneId.systemDefault()) + "May 3 10:21 2016" | ZonedDateTime.of(2016, 5, 3, 10, 21, 0, 0, ZoneId.systemDefault()) + "May 4 10:21 2016 L" | ZonedDateTime.of(2016, 5, 4, 10, 21, 0, 0, ZoneId.systemDefault()) + } + + def "test parseTime with malformatted string"(String time, expected) { + when: + createJobManager().parseTime(time) + + then: + thrown(expected) + + where: + time | expected + null | DateTimeParseException + "" | DateTimeParseException + "Hfjfj" | DateTimeParseException + "Mai 1 10:21" | DateTimeParseException + } + + def "test stripAwayStatusInfo"() { + expect: + LSFJobManager.stripAwayStatusInfo(time) == expected + + where: + time | expected + null | null + "" | "" + "Jan 1 10:21" | "Jan 1 10:21" + "Jan 1 10:21 1" | "Jan 1 10:21" + "Jan 1 10:21 E" | "Jan 1 10:21" + "Jan 1 10:21 F" | "Jan 1 10:21" + } + + def "test submitJob"() { + return null + } + + def "test addToListOfStartedJobs"() { + return null + } + + def "test startHeldJobs"() { + return null + } + + def "test killJobs"() { + return null + } + + def "test queryJobStateByJob"() { + return null + } + + def "test queryJobStateByID"() { + given: + def jm = createJobManagerWithModifiedExecutionService("queryJobStateByIdTest.json") + + when: + def result = jm.queryJobInfoByID(testJobID) + + then: + result.jobState == JobState.COMPLETED_SUCCESSFUL + } + + def "test queryJobStatesByJob"() { + return null + } + + def "test queryJobStatesByID"() { given: - def jsonFile = getResourceFile("queryExtendedJobStateByIdTest.json") + def jm = createJobManagerWithModifiedExecutionService("queryJobStateByIdTest.json") - JobManagerOptions parms = JobManagerOptions.create().build() - BEExecutionService testExecutionService = [ - execute: { String s -> new ExecutionResult(true, 0, jsonFile.readLines(), null) } - ] as BEExecutionService - LSFJobManager manager = new LSFJobManager(testExecutionService, parms) when: - ZonedDateTime refTime = ZonedDateTime.now() - ZonedDateTime earlierTime = refTime.minusDays(1) - ZonedDateTime laterTime = refTime.plusDays(1) - ZonedDateTime laterLastYear = laterTime.minusYears(1) + def result = jm.queryJobInfoByID([testJobID]) then: - manager.parseTime(zonedDateTimeToString(earlierTime)).truncatedTo(ChronoUnit.MINUTES).equals(earlierTime.truncatedTo(ChronoUnit.MINUTES)) - manager.parseTime(zonedDateTimeToString(laterTime)).truncatedTo(ChronoUnit.MINUTES).equals(laterLastYear.truncatedTo(ChronoUnit.MINUTES)) + result.size() == 1 + result[testJobID].jobState == JobState.COMPLETED_SUCCESSFUL + } + + def "test queryAllJobStates"() { + return null } - void "test queryExtendedJobStateById with overdue date"() { + def "test queryExtendedJobStatesByJob"() { given: - JobManagerOptions parms = JobManagerOptions.create().build() - def jsonFile = getResourceFile("queryExtendedJobStateByIdTest.json") - BEExecutionService testExecutionService = [ - execute: { String s -> new ExecutionResult(true, 0, jsonFile.readLines(), null) } - ] as BEExecutionService - LSFJobManager manager = new LSFJobManager(testExecutionService, parms) + def jobManager = createJobManagerWithModifiedExecutionService("queryExtendedJobStateByIdTest.json") + BEJob job = new BEJob(testJobID, jobManager) when: - Map result = manager.queryExtendedJobStateById([new BEJobID("22005")]) + def result = jobManager.queryExtendedJobInfoByJob([job]) then: - result.size() == 0 + result.size() == 1 + result[job] } - void "test queryExtendedJobStateById"() { + def "test queryExtendedJobStatesById"() { + given: - JobManagerOptions parms = JobManagerOptions.create().setMaxTrackingTimeForFinishedJobs(Duration.ofDays(360000)).build() - def jsonFile = getResourceFile("queryExtendedJobStateByIdTest.json") - BEExecutionService testExecutionService = [ - execute: { String s -> new ExecutionResult(true, 0, jsonFile.readLines(), null) } - ] as BEExecutionService - LSFJobManager manager = new LSFJobManager(testExecutionService, parms) + def jobManager = createJobManagerWithModifiedExecutionService("queryExtendedJobStateByIdTest.json") + + ExtendedJobInfo expected = new ExtendedJobInfo(testJobID, JobState.COMPLETED_SUCCESSFUL) + expected.requestedResources = new ResourceSet(null, null, null, null, Duration.ofMinutes(10), null, "short-dmg", null) + expected.usedResources = new ResourceSet(null, new BufferValue(5452595, BufferUnit.k), null, 1, Duration.ofSeconds(1), null, "short-dmg", null) + expected.jobName = "ls -l" + expected.command = "/home/testuser/somescript.sh" + expected.submissionHost = "from-host" + expected.executionHosts = ["exec-host"] + expected.user = "otptest" + expected.rawResourceRequest = 'select[type == local] order[r15s:pg] ' + expected.parentJobIDs = ["22004"] + expected.execHome = "/some/test" + expected.processesInJob = ["46782", "46796", "46798", "46915", "47458", "47643"] + expected.exitCode = 0 + expected.execCwd = "/some/test" + expected.cwd = '$HOME' + expected.projectName = "default" + expected.cpuTime = Duration.ofSeconds(1) + expected.runTime = Duration.ofSeconds(1) when: - Map result = manager.queryExtendedJobStateById([new BEJobID("22005")]) + Map result = jobManager.queryExtendedJobInfoByID([testJobID]) + ExtendedJobInfo jobInfo = result[testJobID] then: result.size() == 1 - GenericJobInfo jobInfo = result.get(new BEJobID("22005")) - jobInfo - jobInfo.askedResources.size == null - jobInfo.askedResources.mem == null - jobInfo.askedResources.cores == null - jobInfo.askedResources.nodes == null - jobInfo.askedResources.walltime == Duration.ofMinutes(10) - jobInfo.askedResources.storage == null - jobInfo.askedResources.queue == "short-dmg" - jobInfo.askedResources.nthreads == null - jobInfo.askedResources.swap == null - - jobInfo.usedResources.size == null - jobInfo.usedResources.mem == new BufferValue(5452595, BufferUnit.k) - jobInfo.usedResources.cores == null - jobInfo.usedResources.nodes == 1 - jobInfo.usedResources.walltime == Duration.ofSeconds(1) - jobInfo.usedResources.storage == null - jobInfo.usedResources.queue == "short-dmg" - jobInfo.usedResources.nthreads == null - jobInfo.usedResources.swap == null - - jobInfo.jobName == "ls -l" - jobInfo.command == "ls -l" - jobInfo.jobID == new BEJobID("22005") - - // The year-parsing/inferrence is checked in another test. Here just take the parsed value. + + when: + // Small hack to get the right year. LSF won't always report years in its various date fields. + // This needs to be configured by the LSF cluster administrators. ZonedDateTime testTime = ZonedDateTime.of(jobInfo.submitTime.year, 12, 28, 19, 56, 0, 0, ZoneId.systemDefault()) - jobInfo.submitTime == testTime - jobInfo.eligibleTime == null - jobInfo.startTime == testTime - jobInfo.endTime == testTime - jobInfo.executionHosts == ["exec-host", "exec-host"] - jobInfo.submissionHost == "from-host" - jobInfo.priority == null - jobInfo.logFile == null - jobInfo.errorLogFile == null - jobInfo.inputFile == null - jobInfo.user == "otptest" - jobInfo.userGroup == null - jobInfo.resourceReq == 'select[type == local] order[r15s:pg] ' - jobInfo.startCount == null - jobInfo.account == null - jobInfo.server == null - jobInfo.umask == null - jobInfo.parameters == null - jobInfo.parentJobIDs == ["22004"] - jobInfo.otherSettings == null - jobInfo.jobState == JobState.COMPLETED_SUCCESSFUL - jobInfo.userTime == null - jobInfo.systemTime == null - jobInfo.pendReason == null - jobInfo.execHome == "/some/test" - jobInfo.execUserName == null - jobInfo.pidStr == ["46782", "46796", "46798", "46915", "47458", "47643"] - jobInfo.pgidStr == null - jobInfo.exitCode == 0 - jobInfo.jobGroup == null - jobInfo.description == null - jobInfo.execCwd == "/some/test" - jobInfo.askedHostsStr == null - jobInfo.cwd == '$HOME' - jobInfo.projectName == "default" - jobInfo.cpuTime == Duration.ofSeconds(1) - jobInfo.runTime == Duration.ofSeconds(1) - jobInfo.timeUserSuspState == null - jobInfo.timePendState == null - jobInfo.timePendSuspState == null - jobInfo.timeSystemSuspState == null - jobInfo.timeUnknownState == null - jobInfo.timeOfCalculation == null - } + expected.submitTime = testTime + expected.startTime = testTime + expected.endTime = testTime + then: + jobInfo == expected + } def "test convertBJobsResultLinesToResultMap"() { given: - def jsonFile = getResourceFile("convertBJobsResultLinesToResultMapTest.json") + def jsonFile = getResourceFile(LSFJobManagerSpec,"convertBJobsResultLinesToResultMapTest.json") def json = jsonFile.text when: @@ -233,35 +232,12 @@ class LSFJobManagerSpec extends Specification { map[jobId]["FINISH_TIME"] == "Jan 7 09:59 L" } - def "test filterJobMapByAge"() { - given: - def jsonFile = getResourceFile("convertBJobsResultLinesToResultMapTest.json") - def json = jsonFile.text - - when: - LocalDateTime referenceTime = LocalDateTime.now() - int minutesToSubtract = 20 - def records = LSFJobManager.convertBJobsJsonOutputToResultMap(json) - records.each { - def id, def record -> - def timeForRecord = LocalDateTime.of(referenceTime.year, referenceTime.month, referenceTime.dayOfMonth, referenceTime.hour, referenceTime.minute, referenceTime.second).minusMinutes(minutesToSubtract) - minutesToSubtract -= 4 - record["FINISH_TIME"] = localDateTimeToLSFString(timeForRecord) - } - records = LSFJobManager.filterJobMapByAge(records, Duration.ofMinutes(10)) - def id = records.keySet()[0] - - then: - records.size() == 3 - id.id == "491861" - } - /** * This test should not be run by default, as it runs quite a while (on purpose). * Reenable it, if you run into memory leaks. */ @Ignore - def testMassiveConvertBJobsResultLinesToResultMap(def _entries, def value) { + def "test massive ConvertBJobsResultLinesToResultMap"(def _entries, def value) { when: int entries = _entries[0] String template1 = getResourceFile("bjobsJobTemplatePart1.txt").text @@ -301,4 +277,92 @@ class LSFJobManagerSpec extends Specification { [8000] | true [16000] | true } + + def "test convertJobDetailsMapToGenericJobInfoObject"() { + given: + def parms = JobManagerOptions.create().build() + TestExecutionService testExecutionService = new TestExecutionService("test", "test") + LSFJobManager jm = new LSFJobManager(testExecutionService, parms) + + Object parsedJson = new JsonSlurper().parseText(getResourceFile(LSFJobManagerSpec, resourceFile).text) + List records = (List) parsedJson.getAt("RECORDS") + + when: + ExtendedJobInfo jobInfo = jm.convertJobDetailsMapToGenericJobInfoObject(records.get(0)) + + then: + jobInfo != null + jobInfo.jobID.toString() == "22005" + jobInfo.command == expectedCommand + + where: + resourceFile | expectedJobId | expectedCommand + "queryExtendedJobStateByIdTest.json" | "22005" | "/home/testuser/somescript.sh" + "queryExtendedJobStateByIdWithoutListsTest.json" | "22005" | "/home/testuser/somescript.sh" + "queryExtendedJobStateByIdEmptyTest.json" | "22005" | null + } + + def "test getEnvironmentVariableGlobs"() { + return null + } + + def "test getDefaultForHoldJobsEnabled"() { + return null + } + + def "test isHoldJobsEnabled"() { + return null + } + + def "test getUserEmail"() { + return null + } + + def "test getUserMask"() { + return null + } + + def "test getUserGroup"() { + return null + } + + def "test getUserAccount"() { + return null + } + + def "test executesWithoutJobSystem"() { + return null + } + + def "test convertResourceSet"() { + return null + } + + def "test collectJobIDsFromJobs"() { + return null + } + + def "test extractAndSetJobResultFromExecutionResult"() { + return null + } + + def "test createCommand"() { + return null + } + + def "test parseJobID"() { + return null + } + + def "test parseJobState"() { + return null + } + + def "test executeStartHeldJobs"() { + return null + } + + def "test executeKillJobs"() { + return null + } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManagerSpec.groovy new file mode 100644 index 00000000..acd1f2c0 --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManagerSpec.groovy @@ -0,0 +1,121 @@ +package de.dkfz.roddy.execution.jobs.cluster.lsf.rest + +import de.dkfz.roddy.execution.jobs.cluster.JobManagerImplementationBaseSpec + +class LSFRestJobManagerSpec extends JobManagerImplementationBaseSpec { + def "test submitJob"() { + return null + } + + def "test addToListOfStartedJobs"() { + return null + } + + def "test startHeldJobs"() { + return null + } + + def "test killJobs"() { + return null + } + + def "test queryJobStateByJob"() { + return null + } + + def "test queryJobStateByID"() { + return null + } + + def "test queryJobStatesByJob"() { + return null + } + + def "test queryJobStatesByID"() { + return null + } + + def "test queryAllJobStates"() { + return null + } + + def "test queryJobStates with overdue finish date"() { + return null + } + + def "test queryExtendedJobStatesByJob"() { + return null + } + + def "test queryExtendedJobStatesByID"() { + return null + } + + def "test queryExtendedJobStatesByID with overdue date"() { + return null + } + + def "test getEnvironmentVariableGlobs"() { + return null + } + + def "test getDefaultForHoldJobsEnabled"() { + return null + } + + def "test isHoldJobsEnabled"() { + return null + } + + def "test getUserEmail"() { + return null + } + + def "test getUserMask"() { + return null + } + + def "test getUserGroup"() { + return null + } + + def "test getUserAccount"() { + return null + } + + def "test executesWithoutJobSystem"() { + return null + } + + def "test convertResourceSet"() { + return null + } + + def "test collectJobIDsFromJobs"() { + return null + } + + def "test extractAndSetJobResultFromExecutionResult"() { + return null + } + + def "test createCommand"() { + return null + } + + def "test parseJobID"() { + return null + } + + def "test parseJobState"() { + return null + } + + def "test executeStartHeldJobs"() { + return null + } + + def "test executeKillJobs"() { + return null + } +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParserTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParserTest.groovy index cfd8308e..30ae01c5 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParserTest.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParserTest.groovy @@ -6,10 +6,8 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs -import de.dkfz.roddy.execution.BEExecutionService -import de.dkfz.roddy.execution.io.ExecutionResult +import de.dkfz.roddy.TestExecutionService import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager -import de.dkfz.roddy.execution.jobs.Command import de.dkfz.roddy.execution.jobs.JobManagerOptions import groovy.transform.CompileStatic import org.junit.BeforeClass @@ -27,42 +25,7 @@ class PBSCommandParserTest { @BeforeClass static void setup() { - testJobManager = new PBSJobManager(new BEExecutionService() { - @Override - ExecutionResult execute(Command command) { - return null - } - - @Override - ExecutionResult execute(Command command, boolean waitFor) { - return null - } - - @Override - ExecutionResult execute(String command) { - return null - } - - @Override - ExecutionResult execute(String command, boolean waitFor) { - return null - } - - @Override - ExecutionResult execute(String command, boolean waitForIncompatibleClassChangeError, OutputStream outputStream) { - return null - } - - @Override - boolean isAvailable() { - return false - } - - @Override - File queryWorkingDirectory() { - return null - } - }, JobManagerOptions.create().setCreateDaemon(false).build()) + testJobManager = new PBSJobManager(new TestExecutionService("testuser", "localhost"), JobManagerOptions.create().build()) } @Test @@ -86,18 +49,16 @@ class PBSCommandParserTest { assert commandParser.memory == "16384" assert commandParser.nodes == "1" assert commandParser.cores == "8" - assert commandParser.parameters == ["PARAMETER_FILE":"/data/michael/temp/roddyLocalTest/testproject/rpp/A100/roddyExecutionStore/exec_170402_171935425_heinold_indelCalling/r170402_171935425_A100_indelCalling_1.parameters"] + assert commandParser.parameters == ["PARAMETER_FILE": "/data/michael/temp/roddyLocalTest/testproject/rpp/A100/roddyExecutionStore/exec_170402_171935425_heinold_indelCalling/r170402_171935425_A100_indelCalling_1.parameters"] assert commandParser.dependencies == ["120015"] def gji = testJobManager.parseGenericJobInfo(commandString) assert gji.jobName == "r170402_171935425_A100_indelCalling" - assert gji.askedResources.getCores() == 8 - assert gji.askedResources.getNodes() == 1 - assert gji.askedResources.getWalltime() == Duration.ofDays(2).plusHours(2) - assert gji.askedResources.getMem().toLong() == 16384 - assert gji.parameters == ["PARAMETER_FILE":"/data/michael/temp/roddyLocalTest/testproject/rpp/A100/roddyExecutionStore/exec_170402_171935425_heinold_indelCalling/r170402_171935425_A100_indelCalling_1.parameters"] + assert gji.requestedResources.getCores() == 8 + assert gji.requestedResources.getNodes() == 1 + assert gji.requestedResources.getWalltime() == Duration.ofDays(2).plusHours(2) + assert gji.requestedResources.getMem().toLong() == 16384 + assert gji.parameters == ["PARAMETER_FILE": "/data/michael/temp/roddyLocalTest/testproject/rpp/A100/roddyExecutionStore/exec_170402_171935425_heinold_indelCalling/r170402_171935425_A100_indelCalling_1.parameters"] assert gji.parentJobIDs == ["120015"] } - - } \ No newline at end of file diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManagerSpec.groovy index b8744b60..b9b069af 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManagerSpec.groovy @@ -6,326 +6,245 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs -import de.dkfz.roddy.TestExecutionService + +import de.dkfz.roddy.config.ResourceSet +import de.dkfz.roddy.config.ResourceSetSize import de.dkfz.roddy.execution.jobs.BEJobID -import de.dkfz.roddy.execution.jobs.GenericJobInfo -import de.dkfz.roddy.execution.jobs.JobManagerOptions +import de.dkfz.roddy.execution.jobs.ExtendedJobInfo import de.dkfz.roddy.execution.jobs.JobState -import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager +import de.dkfz.roddy.execution.jobs.cluster.JobManagerImplementationBaseSpec import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.BufferValue -import spock.lang.Specification +import de.dkfz.roddy.tools.TimeUnit -import java.lang.reflect.InvocationTargetException -import java.lang.reflect.Method import java.time.Duration +import java.time.ZoneId import java.time.ZoneOffset import java.time.ZonedDateTime -class PBSJobManagerSpec extends Specification { - PBSJobManager manager - - String rawXmlExample = ''' - - - 15020227.testServer - workflow_test - testOwner - H - fast - afterok:15624736.testServer - testServer - u - 1499432861 - logging_root_path/clusterLog/2017-07-07/workflow_test - u - oe - n - a - 1499432861 - 1514572025 - logging_root_path/clusterLog/2017-07-07/workflow_test.o15020227 - 0 - 1499432861 - True - 71497 - - 5120mb - 1 - 1:ppn=1 - 00:20:00 - - - 04:50:19 - 0 - 2449672kb - 2524268kb - 03:07:41 - - PBS_O_QUEUE=default,PBS_O_HOME=/home/test,PBS_O_LOGNAME=test,PBS_O_SHELL=/bin/bash,PBS_O_LANG=en_GB.UTF-8 - test - testGroup - E - -N workflow_test -h -o /clusterLog/2017-07-07 -j oe -W umask=027 -l mem=5120M -l walltime=00:00:20:00 -l nodes=1:ppn=1 - 23 - False - 0 - sub.testServer - 1 - 1 - - - ''' - - String outputQueued = '''4499334.pbsserverr180328_183957634_pid_4_starAlignmentotp-data@subm.example.comQotppbsserveru1522255162subm:/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment.o$PBS_JOBIDnoena1522255162subm:/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment.o$PBS_JOBID01522255162True168:00:0037888mb1511:ppn=405:00:00PBS_O_QUEUE=otp,PBS_O_HOME=/home/otp-data,PBS_O_LOGNAME=otp-data,PBS_O_PATH=/usr/local/bin:/usr/bin:/bin:/usr/games,PBS_O_MAIL=/var/mail/otp-data,PBS_O_SHELL=/bin/bash,PBS_O_LANG=en_US.utf8,TOOL_ID=starAlignment,PARAMETER_FILE=/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment_3.parameters,CONFIG_FILE=/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment_3.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme,PBS_O_WORKDIR=/home/otp-data,PBS_O_HOST=subm.example.com,PBS_O_SERVER=pbsserver1522255162-v TOOL_ID=starAlignment,PARAMETER_FILE=/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment_3.parameters,CONFIG_FILE=/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment_3.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N r180328_183957634_pid_4_starAlignment -h -w /home/otp-data -j oe -o /net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment.o$PBS_JOBID -l mem=37888M -l walltime=00:05:00:00 -l nodes=1:ppn=4 -q otp /net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/analysisTools/roddyTools/wrapInScript.shFalse0subm.example.com/home/otp-data - -''' - - String outputFinished = '''4564045.pbsserverr180405_163953553_stds_snvJoinVcfFilesotp-data@subm.example.com00:00:006064kb454232kb00:00:06Cotppbsserveru1522939158afterok:6059780.pbsserver@pbsserver:6059781.pbsserver@pbsserver,beforeok:6059788.pbsserver@pbsserversubm:/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o$PBS_JOBIDdenbi5-int/0+denbi5-int/1+denbi5-int/215003noena1522939194subm:/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o$PBS_JOBID01522939158True168:00:001024mb1511:ppn=304:00:0014506PBS_O_QUEUE=otp,PBS_O_HOME=/home/otp-data,PBS_O_LOGNAME=otp-data,PBS_O_PATH=/usr/local/bin:/usr/bin:/bin:/usr/games,PBS_O_MAIL=/var/mail/otp-data,PBS_O_SHELL=/bin/bash,PBS_O_LANG=en_US.utf8,TOOL_ID=snvJoinVcfFiles,PARAMETER_FILE=/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles_9.parameters,CONFIG_FILE=/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles_9.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme,PBS_O_WORKDIR=/home/otp-data,PBS_O_HOST=subm.example.com,PBS_O_SERVER=pbsserverPost job file processing error; job 4564045.pbsserver on host denbi5-int/1 - -Unable to copy file /var/spool/torque/spool/4564045.pbsserver.OU to otp-data@subm:/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o4564045.pbsserver -*** error from copy -Host key verification failed. -lost connection -*** end error output -Output retained on that host in: /var/spool/torque/undelivered/4564045.pbsserver.OU15229391820-v TOOL_ID=snvJoinVcfFiles,PARAMETER_FILE=/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles_9.parameters,CONFIG_FILE=/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles_9.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N r180405_163953553_stds_snvJoinVcfFiles -h -w /home/otp-data -j oe -o /net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o$PBS_JOBID -l mem=1024M -l walltime=00:04:00:00 -l nodes=1:ppn=3 -q otp -W depend=afterok:6059780:6059781 /net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/analysisTools/roddyTools/wrapInScript.sh15229391831False1522939194010.431524subm.example.com/home/otp-data -''' - - void setup() { - JobManagerOptions parms = JobManagerOptions.create().build() - TestExecutionService testExecutionService = new TestExecutionService("test", "test") - manager = new PBSJobManager(testExecutionService, parms) - } - - void "test processQstatOutputFromXML with queued job"() { +class PBSJobManagerSpec extends JobManagerImplementationBaseSpec { + + /** + * convertResourceSet is implemented in ClusterJobManager but calls methods in the actual JobManager instance + */ + void "test convertResourceSet"(String expected, Integer mem, Integer cores, Integer nodes, String walltime) { + given: + BufferValue _mem = mem ? new BufferValue(mem, BufferUnit.m) : null + TimeUnit _walltime = walltime ? new TimeUnit(walltime) : null + ResourceSet test = new ResourceSet(ResourceSetSize.l, _mem, cores, nodes, (TimeUnit) _walltime, null, null, null) + + expect: + createJobManager().convertResourceSet(null, test).processingCommandString == expected + + where: + expected | mem | cores | nodes | walltime + "-l mem=1024M -l walltime=00:01:00:00 -l nodes=1:ppn=2" | 1024 | 2 | 1 | "h" + "-l walltime=00:01:00:00 -l nodes=1:ppn=1" | null | null | 1 | "h" + "-l mem=1024M -l nodes=1:ppn=2" | 1024 | 2 | null | null + } + + ExtendedJobInfo getGenericJobInfoObjectForTests() { + ExtendedJobInfo expected = new ExtendedJobInfo(testJobID, JobState.QUEUED) + expected.requestedResources = new ResourceSet(null, new BufferValue(37888, BufferUnit.M), 4, 1, Duration.ofHours(5), null, "debug", null) + expected.usedResources = new ResourceSet(null, null, null, null, (Duration) null, null, "debug", null) + expected.jobName = "ATestJob" + expected.user = "testuser" + expected.userGroup = "" + expected.account = "" + expected.umask = "" + expected.submissionHost = "subm-host.example.com" + expected.executionHosts = [] + expected.submitTime = ZonedDateTime.of(2018, 03, 28, 18, 39, 22, 0, ZoneOffset.systemDefault()) + expected.eligibleTime = ZonedDateTime.of(2018, 03, 28, 18, 39, 22, 0, ZoneOffset.systemDefault()) + expected.priority = "0" + expected.logFile = new File("/somefolder/job.o22005.pbsserver") + expected.errorLogFile = new File("/somefolder/job.o22005.pbsserver") + expected.rawResourceRequest = '-v TOOL_ID=starAlignment,PARAMETER_FILE=/somefolder/exec/ATestJob.parameters,CONFIG_FILE=/somefolder/exec/ATestJob.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N ATestJob -h -w /home/otp-data -j oe -o /somefolder/job.o$PBS_JOBID -l mem=37888M -l walltime=00:05:00:00 -l nodes=1:ppn=4 -q otp /somefolder/wrapInScript.sh' + expected.server = "pbsserver" + expected.parentJobIDs = [] + expected + } + + def "test submitJob"() { + return null + } + + def "test addToListOfStartedJobs"() { + return null + } + + def "test startHeldJobs"() { + return null + } + + def "test killJobs"() { + return null + } + + def "test queryJobStateByJob"() { + return null + } + + def "test queryJobStateByID"() { + return null + } + + def "test queryJobInfByJob"() { + return null + } + + def "test queryJobInfByID"() { + return null + } + + def "test queryAllJobInf"() { + return null + } + + def "test queryExtendedJobInfByJob"() { + return null + } + + def "test queryExtendedJobInfByID"() { + return null + } + + void "test queryExtendedJobInfByID with queued job"() { + + given: + + def jobManager = createJobManagerWithModifiedExecutionService("processExtendedQstatXMLOutputWithQueuedJob.xml") + + ExtendedJobInfo expected = getGenericJobInfoObjectForTests() + when: - Map result = manager.processQstatOutputFromXML(outputQueued) + Map result = jobManager.queryExtendedJobInfo([testJobID]) + ExtendedJobInfo jobInfo = result.get(testJobID) then: result.size() == 1 - GenericJobInfo jobInfo = result.get(new BEJobID("4499334")) - jobInfo - jobInfo.askedResources.size == null - jobInfo.askedResources.mem == new BufferValue(37888, BufferUnit.M) - jobInfo.askedResources.cores == 4 - jobInfo.askedResources.nodes == 1 - jobInfo.askedResources.walltime == Duration.ofHours(5) - jobInfo.askedResources.storage == null - jobInfo.askedResources.queue == "otp" - jobInfo.askedResources.nthreads == null - jobInfo.askedResources.swap == null - - jobInfo.usedResources.size == null - jobInfo.usedResources.mem == null - jobInfo.usedResources.cores == null - jobInfo.usedResources.nodes == null - jobInfo.usedResources.walltime == null - jobInfo.usedResources.storage == null - jobInfo.usedResources.queue == "otp" - jobInfo.usedResources.nthreads == null - jobInfo.usedResources.swap == null - - jobInfo.jobName == "r180328_183957634_pid_4_starAlignment" - jobInfo.command == null - jobInfo.jobID == new BEJobID("4499334") - jobInfo.submitTime.isEqual ZonedDateTime.of(2018, 03, 28, 16, 39, 22, 0, ZoneOffset.UTC) - jobInfo.eligibleTime.isEqual ZonedDateTime.of(2018, 03, 28, 16, 39, 22, 0, ZoneOffset.UTC) - jobInfo.startTime == null - jobInfo.endTime == null - jobInfo.executionHosts == null - jobInfo.submissionHost == "subm.example.com" - jobInfo.priority == "0" - jobInfo.logFile == new File("/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment.o4499334.pbsserver") - jobInfo.errorLogFile == new File("/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment.o4499334.pbsserver") - jobInfo.inputFile == null - jobInfo.user == null - jobInfo.userGroup == null - jobInfo.resourceReq == '-v TOOL_ID=starAlignment,PARAMETER_FILE=/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment_3.parameters,CONFIG_FILE=/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment_3.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N r180328_183957634_pid_4_starAlignment -h -w /home/otp-data -j oe -o /net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment.o$PBS_JOBID -l mem=37888M -l walltime=00:05:00:00 -l nodes=1:ppn=4 -q otp /net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/analysisTools/roddyTools/wrapInScript.sh' - jobInfo.startCount == null - jobInfo.account == null - jobInfo.server == "pbsserver" - jobInfo.umask == null - jobInfo.parameters == null - jobInfo.parentJobIDs == [] - jobInfo.otherSettings == null - jobInfo.jobState == JobState.QUEUED - jobInfo.userTime == null - jobInfo.systemTime == null - jobInfo.pendReason == null - jobInfo.execHome == null - jobInfo.execUserName == null - jobInfo.pidStr == null - jobInfo.pgidStr == null - jobInfo.exitCode == null - jobInfo.jobGroup == null - jobInfo.description == null - jobInfo.execCwd == null //??? - jobInfo.askedHostsStr == null - jobInfo.cwd == null - jobInfo.projectName == null - jobInfo.cpuTime == null - jobInfo.runTime == null - jobInfo.timeUserSuspState == null - jobInfo.timePendState == null - jobInfo.timePendSuspState == null - jobInfo.timeSystemSuspState == null - jobInfo.timeUnknownState == null - jobInfo.timeOfCalculation == null - } - - void "test processQstatOutputFromXML with finished job, with newline"() { + jobInfo.requestedResources == expected.requestedResources + jobInfo.usedResources == expected.usedResources + jobInfo.submitTime == expected.submitTime + jobInfo == expected + } + + void "test queryExtendedJobInfByID with finished job"() { + + given: + + def jobManager = createJobManagerWithModifiedExecutionService("processExtendedQstatXMLOutputWithFinishedJob.xml") + + ExtendedJobInfo expected = getGenericJobInfoObjectForTests() + + expected.usedResources = new ResourceSet(null, new BufferValue(6064, BufferUnit.k), null, null, Duration.ofSeconds(6), null, "otp", null) + expected.startTime = ZonedDateTime.of(2018, 4, 5, 16, 39, 43, 0, ZoneId.systemDefault()) + expected.endTime = ZonedDateTime.of(2018, 4, 5, 16, 39, 54, 0, ZoneId.systemDefault()) + expected.executionHosts = ["exec-host"] + expected.startCount = 1 + expected.parentJobIDs = ["6059780", "6059781"] + expected.jobState = JobState.COMPLETED_UNKNOWN + expected.exitCode = 0 + expected.cpuTime = Duration.ZERO + expected.runTime = Duration.ofSeconds(10) + when: - Map result = manager.processQstatOutputFromXML(outputFinished) + Map result = jobManager.queryExtendedJobInfo([testJobID]) + ExtendedJobInfo jobInfo = result.get(testJobID) then: result.size() == 1 - GenericJobInfo jobInfo = result.get(new BEJobID("4564045")) - jobInfo - jobInfo.askedResources.size == null - jobInfo.askedResources.mem == new BufferValue(1024, BufferUnit.M) - jobInfo.askedResources.cores == 3 - jobInfo.askedResources.nodes == 1 - jobInfo.askedResources.walltime == Duration.ofHours(4) - jobInfo.askedResources.storage == null - jobInfo.askedResources.queue == "otp" - jobInfo.askedResources.nthreads == null - jobInfo.askedResources.swap == null - - jobInfo.usedResources.size == null - jobInfo.usedResources.mem == new BufferValue(6064, BufferUnit.k) - jobInfo.usedResources.cores == null - jobInfo.usedResources.nodes == null - jobInfo.usedResources.walltime == Duration.ofSeconds(6) - jobInfo.usedResources.storage == null - jobInfo.usedResources.queue == "otp" - jobInfo.usedResources.nthreads == null - jobInfo.usedResources.swap == null - - jobInfo.jobName == "r180405_163953553_stds_snvJoinVcfFiles" - jobInfo.command == null - jobInfo.jobID == new BEJobID("4564045") - jobInfo.submitTime.isEqual ZonedDateTime.of(2018, 4, 5, 14, 39, 18, 0, ZoneOffset.UTC) - jobInfo.eligibleTime.isEqual ZonedDateTime.of(2018, 4, 5, 14, 39, 42, 0, ZoneOffset.UTC) - jobInfo.startTime.isEqual ZonedDateTime.of(2018, 4, 5, 14, 39, 43, 0, ZoneOffset.UTC) - jobInfo.endTime.isEqual ZonedDateTime.of(2018, 4, 5, 14, 39, 54, 0, ZoneOffset.UTC) - jobInfo.executionHosts == ["denbi5-int"] - jobInfo.submissionHost == "subm.example.com" - jobInfo.priority == "0" - jobInfo.logFile == new File("/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o4564045.pbsserver") - jobInfo.errorLogFile == new File("/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o4564045.pbsserver") - jobInfo.inputFile == null - jobInfo.user == null - jobInfo.userGroup == null - jobInfo.resourceReq == '-v TOOL_ID=snvJoinVcfFiles,PARAMETER_FILE=/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles_9.parameters,CONFIG_FILE=/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles_9.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N r180405_163953553_stds_snvJoinVcfFiles -h -w /home/otp-data -j oe -o /net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o$PBS_JOBID -l mem=1024M -l walltime=00:04:00:00 -l nodes=1:ppn=3 -q otp -W depend=afterok:6059780:6059781 /net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/analysisTools/roddyTools/wrapInScript.sh' - jobInfo.startCount == 1 - jobInfo.account == null - jobInfo.server == "pbsserver" - jobInfo.umask == null - jobInfo.parameters == null - jobInfo.parentJobIDs == ["6059780", "6059781"] - jobInfo.otherSettings == null - jobInfo.jobState == JobState.COMPLETED_UNKNOWN - jobInfo.userTime == null - jobInfo.systemTime == null - jobInfo.pendReason == null - jobInfo.execHome == null - jobInfo.execUserName == null - jobInfo.pidStr == null - jobInfo.pgidStr == null - jobInfo.exitCode == 0 - jobInfo.jobGroup == null - jobInfo.description == null - jobInfo.execCwd == null - jobInfo.askedHostsStr == null - jobInfo.cwd == null - jobInfo.projectName == null - jobInfo.cpuTime == Duration.ZERO - jobInfo.runTime == Duration.ofSeconds(10) - jobInfo.timeUserSuspState == null - jobInfo.timePendState == null - jobInfo.timePendSuspState == null - jobInfo.timeSystemSuspState == null - jobInfo.timeUnknownState == null - jobInfo.timeOfCalculation == null - } - - void "processQstatOutput, qstat with empty XML output"() { + jobInfo == expected + } + + void "test queryExtendedJobInfoByID with empty XML output"() { given: - String rawXMLOutput =''' - - - 15020227.testServer - - - ''' + def jm = createJobManagerWithModifiedExecutionService("queryExtendedJobInfoWithEmptyXML.xml") when: - Map jobInfo = manager.processQstatOutputFromXML(rawXMLOutput) + Map result = jm.queryExtendedJobInfo([testJobID]) + def jobInfo = result.get(testJobID) then: - jobInfo.size() == 1 + result.size() == 1 + jobInfo } - void "processQstatOutput, qstat with XML output"() { + void "test queryExtendedJobInfoByID, replace placeholder PBS_JOBID in logFile and errorLogFile with job id "() { + given: + def jm = createJobManagerWithModifiedExecutionService("queryExtendedJobInfoWithPlaceholderReplacement.xml") + when: - Map jobInfo = manager.processQstatOutputFromXML(rawXmlExample) + Map result = jm.queryExtendedJobInfo([testJobID]) + def jobInfo = result.get(testJobID) then: - jobInfo.size() == 1 - jobInfo.get(new BEJobID("15020227")).jobName == "workflow_test" + result.size() == 1 + jobInfo.logFile.absolutePath == "/logging_root_path/logfile.o22005.testServer" + jobInfo.errorLogFile.absolutePath == "/logging_root_path/logfile.e22005.testServer" } + def "test getEnvironmentVariableGlobs"() { + return null + } - void "processQstatOutput, replace placeholder PBS_JOBID in logFile and errorLogFile with job id "() { - given: - String rawXMLOutput=''' - - - 15976927.testServer - /logging_root_path/clusterLog/2017-07-07/workflow_test/snvFilter.o$PBS_JOBID - tbi-pbs:/logging_root_path/clusterLog/2017-07-07/workflow_test/snvFilter.e$PBS_JOBID - - - ''' + def "test getDefaultForHoldJobsEnabled"() { + return null + } - when: - Map jobInfo = manager.processQstatOutputFromXML(rawXMLOutput) + def "test isHoldJobsEnabled"() { + return null + } - then: - jobInfo.size() == 1 - jobInfo.get(new BEJobID("15976927")).logFile.toString() == "/logging_root_path/clusterLog/2017-07-07/workflow_test/snvFilter.o15976927.testServer" - jobInfo.get(new BEJobID("15976927")).errorLogFile.toString() == "/logging_root_path/clusterLog/2017-07-07/workflow_test/snvFilter.e15976927.testServer" + def "test getUserEmail"() { + return null } + def "test getUserMask"() { + return null + } - void "parseColonSeparatedHHMMSSDuration, parse duration"() { + def "test getUserGroup"() { + return null + } - given: - Method method = ClusterJobManager.class.getDeclaredMethod("parseColonSeparatedHHMMSSDuration", String) - method.setAccessible(true) + def "test getUserAccount"() { + return null + } - expect: - parsedDuration == method.invoke(null, input) + def "test executesWithoutJobSystem"() { + return null + } - where: - input || parsedDuration - "00:00:00" || Duration.ofSeconds(0) - "24:00:00" || Duration.ofHours(24) - "119:00:00" || Duration.ofHours(119) + def "test convertResourceSet"() { + return null } - - void "parseColonSeparatedHHMMSSDuration, parse duration fails"() { + def "test collectJobIDsFromJobs"() { + return null + } - given: - Method method = ClusterJobManager.class.getDeclaredMethod("parseColonSeparatedHHMMSSDuration", String) - method.setAccessible(true) + def "test extractAndSetJobResultFromExecutionResult"() { + return null + } - when: - method.invoke(null, "02:42") + def "test createCommand"() { + return null + } - then: - InvocationTargetException e = thrown(InvocationTargetException) - e.targetException.message == "Duration string is not of the format HH+:MM:SS: '02:42'" + def "test parseJobID"() { + return null + } + + def "test parseJobState"() { + return null + } + + def "test executeStartHeldJobs"() { + return null + } + + def "test executeKillJobs"() { + return null } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerSpec.groovy new file mode 100644 index 00000000..986c549c --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerSpec.groovy @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2017 eilslabs. + * + * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). + */ + +package de.dkfz.roddy.execution.jobs.cluster.sge + +import de.dkfz.roddy.execution.BEExecutionService +import de.dkfz.roddy.execution.io.ExecutionResult +import de.dkfz.roddy.execution.jobs.* +import de.dkfz.roddy.execution.jobs.cluster.JobManagerImplementationBaseSpec + +import static de.dkfz.roddy.execution.jobs.JobState.* + +/** + */ +class SGEJobManagerSpec extends JobManagerImplementationBaseSpec { + + + def "test submitJob"() { + return null + } + + def "test addToListOfStartedJobs"() { + return null + } + + def "test startHeldJobs"() { + return null + } + + def "test killJobs"() { + return null + } + + def "test queryJobInfoByJob (also tests multiple requests and by ID)"(String id, expectedState) { + given: + def jobManager = createJobManagerWithModifiedExecutionService("simpleQStatOutput.txt") + def jobID = new BEJobID(id) + def job = new BEJob(jobID, jobManager) + + when: + JobInfo info = jobManager.queryJobInfoByJob(job) + + then: + info.jobID == jobID + info.jobState == expectedState + + where: + id | expectedState + "22000" | UNKNOWN + "22003" | RUNNING + "22005" | HOLD + } + + def "test queryAllJobInfo"() { + given: + def jobManager = createJobManagerWithModifiedExecutionService("simpleQStatOutput.txt") + def jobIDs = ["22003", "22005", "22006", "22007"].collect { new BEJobID(it) } + when: + def result = jobManager.queryAllJobInfo() + + then: + result.size() == 4 + result.keySet() as List == jobIDs + result[jobIDs[0]].jobState == RUNNING + result[jobIDs[1]].jobState == HOLD + result[jobIDs[2]].jobState == HOLD + result[jobIDs[3]].jobState == HOLD + } + + SGEJobManager createSGEJobManagerForExtendedInfoTests() { + // This one is special, as query extended states in SGE utilizes queryJobInfo + BEExecutionService testExecutionService = [ + execute: { + String s -> + if (s.startsWith("qstat -xml")) + new ExecutionResult(true, 0, getResourceFile(SGEJobManagerSpec, "queryExtendedJobStatesWithQueuedJobs.xml").readLines(), null) + else + new ExecutionResult(true, 0, getResourceFile(SGEJobManagerSpec, "simpleQStatOutput.txt").readLines(), null) + } + ] as BEExecutionService + def jobManager = new SGEJobManager(testExecutionService, new JobManagerOptionsBuilder().build()) + return jobManager + } + + def "test queryExtendedJobInfoByJob (also tests multiple requests and by ID)"(String id, expectedState) { + given: + def jobManager = createSGEJobManagerForExtendedInfoTests() + def jobID = new BEJobID(id) + def job = new BEJob(jobID, jobManager) + + when: + JobInfo info = jobManager.queryExtendedJobInfoByJob(job) + + then: + info.jobID == jobID && info.jobState == expectedState + + where: + id | expectedState + "22000" | UNKNOWN + "22003" | RUNNING + "22005" | HOLD + } + + def "test queryAllExtendedJobInfo"() { + given: + def jobManager = createSGEJobManagerForExtendedInfoTests() + def jobIDs = ["22003", "22005", "22006", "22007"].collect { new BEJobID(it) } + + when: + Map result = jobManager.queryAllExtendedJobInfo() + + then: + result.size() == 4 + result[jobIDs[0]].jobID == jobIDs[0] + result[jobIDs[0]].jobState == RUNNING + + result[jobIDs[1]].jobID == jobIDs[1] + result[jobIDs[1]].jobState == HOLD + + result[jobIDs[2]].jobID == jobIDs[2] + result[jobIDs[2]].jobState == HOLD + + result[jobIDs[3]].jobID == jobIDs[3] + result[jobIDs[3]].jobState == HOLD + } + + def "test getEnvironmentVariableGlobs"() { + return null + } + + def "test getDefaultForHoldJobsEnabled"() { + return null + } + + def "test isHoldJobsEnabled"() { + return null + } + + def "test getUserEmail"() { + return null + } + + def "test getUserMask"() { + return null + } + + def "test getUserGroup"() { + return null + } + + def "test getUserAccount"() { + return null + } + + def "test executesWithoutJobSystem"() { + return null + } + + def "test convertResourceSet"() { + return null + } +// @Test +// public void testConvertToolEntryToPBSCommandParameters() { + // Roddy specific tests! +// ResourceSet rset1 = new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 2, 1, new TimeUnit("h"), null, null, null); +// ResourceSet rset2 = new ResourceSet(ResourceSetSize.l, null, null, 1, new TimeUnit("h"), null, null, null); +// ResourceSet rset3 = new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 2, null, null, null, null, null); +// +// Configuration cfg = new Configuration(new InformationalConfigurationContent(null, Configuration.ConfigurationType.OTHER, "test", "", "", null, "", ResourceSetSize.l, null, null, null, null)); +// +// BatchEuphoriaJobManager cFactory = new SGEJobManager(false); +// PBSResourceProcessingParameters test = (PBSResourceProcessingParameters) cFactory.convertResourceSet(cfg, rset1); +// assert test.getProcessingCommandString().trim().equals("-V -l s_data=1024M"); +// +// test = (PBSResourceProcessingParameters) cFactory.convertResourceSet(cfg, rset2); +// assert test.getProcessingCommandString().equals(" -V"); +// +// test = (PBSResourceProcessingParameters) cFactory.convertResourceSet(cfg, rset3); +// assert test.getProcessingCommandString().equals(" -V -l s_data=1024M"); +// } + + def "test collectJobIDsFromJobs"() { + return null + } + + def "test extractAndSetJobResultFromExecutionResult"() { + return null + } + + def "test createCommand"() { + return null + } + + def "test parseJobID"() { + return null + } + + def "test parseJobState"() { + return null + } + + def "test executeStartHeldJobs"() { + return null + } + + def "test executeKillJobs"() { + return null + } +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerTest.java b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerTest.java deleted file mode 100644 index 438ed5c9..00000000 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2017 eilslabs. - * - * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). - */ - -package de.dkfz.roddy.execution.jobs.cluster.sge; - -import org.junit.Test; - -/** - */ -public class SGEJobManagerTest { - - @Test - public void testConvertToolEntryToPBSCommandParameters() { - // Roddy specific tests! -// ResourceSet rset1 = new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 2, 1, new TimeUnit("h"), null, null, null); -// ResourceSet rset2 = new ResourceSet(ResourceSetSize.l, null, null, 1, new TimeUnit("h"), null, null, null); -// ResourceSet rset3 = new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 2, null, null, null, null, null); -// -// Configuration cfg = new Configuration(new InformationalConfigurationContent(null, Configuration.ConfigurationType.OTHER, "test", "", "", null, "", ResourceSetSize.l, null, null, null, null)); -// -// BatchEuphoriaJobManager cFactory = new SGEJobManager(false); -// PBSResourceProcessingParameters test = (PBSResourceProcessingParameters) cFactory.convertResourceSet(cfg, rset1); -// assert test.getProcessingCommandString().trim().equals("-V -l s_data=1024M"); -// -// test = (PBSResourceProcessingParameters) cFactory.convertResourceSet(cfg, rset2); -// assert test.getProcessingCommandString().equals(" -V"); -// -// test = (PBSResourceProcessingParameters) cFactory.convertResourceSet(cfg, rset3); -// assert test.getProcessingCommandString().equals(" -V -l s_data=1024M"); - } - - -} diff --git a/src/test/resources/de/dkfz/roddy/batcheuphoria/getResourceFileTest.txt b/src/test/resources/de/dkfz/roddy/batcheuphoria/getResourceFileTest.txt new file mode 100644 index 00000000..f2ba8f84 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/batcheuphoria/getResourceFileTest.txt @@ -0,0 +1 @@ +abc \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/createJobManagerWithModifiedExecutionServiceTest.txt b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/createJobManagerWithModifiedExecutionServiceTest.txt new file mode 100644 index 00000000..0b008121 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/createJobManagerWithModifiedExecutionServiceTest.txt @@ -0,0 +1,3 @@ +Line1 +Line2 +Line3 \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdTest.json b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdTest.json index 03f86de3..295eabef 100644 --- a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdTest.json +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdTest.json @@ -34,7 +34,7 @@ "EXEC_HOME": "/some/test", "SLOTS": "1", "ERROR_FILE": "", - "COMMAND": "ls -l", + "COMMAND": "/home/testuser/somescript.sh", "DEPENDENCY": "done(22004)" } ] diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdWithoutListsTest.json b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdWithoutListsTest.json index ad2cff1c..1b24940a 100644 --- a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdWithoutListsTest.json +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdWithoutListsTest.json @@ -4,17 +4,17 @@ "RECORDS": [ { "JOBID": "22005", - "JOB_NAME": "ls -l", + "JOB_NAME": "ATestJob", "STAT": "DONE", - "USER": "otptest", - "QUEUE": "short-dmg", + "USER": "testuser", + "QUEUE": "debug", "JOB_DESCRIPTION": "", "PROJ_NAME": "default", "JOB_GROUP": "", "JOB_PRIORITY": "", "PIDS": "51904", "EXIT_CODE": "1", - "FROM_HOST": "from-host", + "FROM_HOST": "subm-host", "EXEC_HOST": "exec-host", "SUBMIT_TIME": "Dec 28 19:56", "START_TIME": "Dec 28 19:56", @@ -31,10 +31,10 @@ "OUTPUT_FILE": "/sequencing/whole_genome_sequencing/coveragePlotSingle.o30060", "INPUT_FILE": "", "EFFECTIVE_RESREQ": "select[type == local] order[r15s:pg] ", - "EXEC_HOME": "/some/test", + "EXEC_HOME": "/home/testuser", "SLOTS": "1", "ERROR_FILE": "", - "COMMAND": "ls -l", + "COMMAND": "/home/testuser/somescript.sh", "DEPENDENCY": "done(22004)" } ] diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryJobStateByIdTest.json b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryJobStateByIdTest.json new file mode 100644 index 00000000..80f55881 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryJobStateByIdTest.json @@ -0,0 +1,12 @@ +{ + "COMMAND": "bjobs", + "JOBS": 1, + "RECORDS": [ + { + "JOBID": "22005", + "JOB_NAME": "ls -l", + "STAT": "DONE", + "FINISH_TIME": "Dec 28 19:56 L" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/createJobManagerWithModifiedExecutionServiceTest.txt b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/createJobManagerWithModifiedExecutionServiceTest.txt new file mode 100644 index 00000000..0b008121 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/createJobManagerWithModifiedExecutionServiceTest.txt @@ -0,0 +1,3 @@ +Line1 +Line2 +Line3 \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputTests.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputTests.xml new file mode 100644 index 00000000..657e6102 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputTests.xml @@ -0,0 +1,52 @@ + + + + 15020227.testServer + workflow_test + testOwner + H + fast + afterok:15624736.testServer + testServer + u + 1499432861 + logging_root_path/clusterLog/2017-07-07/workflow_test + u + oe + n + a + 1499432861 + 1514572025 + logging_root_path/clusterLog/2017-07-07/workflow_test.o15020227 + 0 + 1499432861 + True + + 71497 + + + 5120mb + 1 + 1:ppn=1 + 00:20:00 + + + 04:50:19 + 0 + 2449672kb + 2524268kb + 03:07:41 + + PBS_O_QUEUE=default,PBS_O_HOME=/home/test,PBS_O_LOGNAME=test,PBS_O_SHELL=/bin/bash,PBS_O_LANG=en_GB.UTF-8 + test + testGroup + E + -N workflow_test -h -o /clusterLog/2017-07-07 -j oe -W umask=027 -l mem=5120M -l walltime=00:00:20:00 -l nodes=1:ppn=1 + 23 + False + 0 + sub.testServer + 1 + 1 + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithFinishedJob.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithFinishedJob.xml new file mode 100644 index 00000000..b53b4dd9 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithFinishedJob.xml @@ -0,0 +1,65 @@ + + + + 22005.pbsserver + ATestJob + testuser@subm-host.example.com + + 00:00:00 + 6064kb + 454232kb + 00:00:06 + + C + debug + pbsserver + u + 1522255162 + afterok:6059780.pbsserver@pbsserver:6059781.pbsserver@pbsserver,beforeok:6059788.pbsserver@pbsserver + subm:/somefolder/job.o$PBS_JOBID + exec-host/0+exec-host/1+exec-host/2 + 15003 + n + oe + n + a + 1522255162 + subm:/somefolder/job.o$PBS_JOBID + 0 + 1522255162 + True + + 168:00:00 + 37888mb + 1 + 5 + 1 + 1:ppn=4 + 05:00:00 + + 51904 + PBS_O_QUEUE=otp,PBS_O_HOME=/home/otp-data,PBS_O_LOGNAME=otp-data,PBS_O_PATH=/usr/local/bin:/usr/bin:/bin:/usr/games,PBS_O_MAIL=/var/mail/otp-data,PBS_O_SHELL=/bin/bash,PBS_O_LANG=en_US.utf8,TOOL_ID=starAlignment,PARAMETER_FILE=/somefolder/exec/ATestJob.parameters,CONFIG_FILE=/somefolder/exec/ATestJob.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme,PBS_O_WORKDIR=/home/otp-data,PBS_O_HOST=subm.example.com,PBS_O_SERVER=pbsserver + Post job file processing error; job 4564045.pbsserver on host exec-host/1 + + Unable to copy file /var/spool/torque/spool/4564045.pbsserver.OU to + otp-data@subm:/somefolder/results/r180405_163953553_stds_snvJoinVcfFiles.o4564045.pbsserver + *** error from copy + Host key verification failed. + lost connection + *** end error output + Output retained on that host in: /var/spool/torque/undelivered/4564045.pbsserver.OU + + 1522255162 + 0 + -v TOOL_ID=starAlignment,PARAMETER_FILE=/somefolder/exec/ATestJob.parameters,CONFIG_FILE=/somefolder/exec/ATestJob.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N ATestJob -h -w /home/otp-data -j oe -o /somefolder/job.o$PBS_JOBID -l mem=37888M -l walltime=00:05:00:00 -l nodes=1:ppn=4 -q otp /somefolder/wrapInScript.sh + 1522939183 + 1 + False + 1522939194 + 0 + 10.431524 + subm-host.example.com + /home/testuser + + diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithQueuedJob.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithQueuedJob.xml new file mode 100644 index 00000000..6ca8491a --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithQueuedJob.xml @@ -0,0 +1,40 @@ + + + + 22005.pbsserver + ATestJob + testuser@subm-host.example.com + Q + debug + pbsserver + u + 1522255162 + subm:/somefolder/job.o$PBS_JOBID + n + oe + n + a + 1522255162 + subm:/somefolder/job.o$PBS_JOBID + 0 + 1522255162 + True + + 168:00:00 + 37888mb + 1 + 5 + 1 + 1:ppn=4 + 05:00:00 + + PBS_O_QUEUE=otp,PBS_O_HOME=/home/otp-data,PBS_O_LOGNAME=otp-data,PBS_O_PATH=/usr/local/bin:/usr/bin:/bin:/usr/games,PBS_O_MAIL=/var/mail/otp-data,PBS_O_SHELL=/bin/bash,PBS_O_LANG=en_US.utf8,TOOL_ID=starAlignment,PARAMETER_FILE=/somefolder/exec/ATestJob.parameters,CONFIG_FILE=/somefolder/exec/ATestJob.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme,PBS_O_WORKDIR=/home/otp-data,PBS_O_HOST=subm.example.com,PBS_O_SERVER=pbsserver + 1522255162 + -v TOOL_ID=starAlignment,PARAMETER_FILE=/somefolder/exec/ATestJob.parameters,CONFIG_FILE=/somefolder/exec/ATestJob.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N ATestJob -h -w /home/otp-data -j oe -o /somefolder/job.o$PBS_JOBID -l mem=37888M -l walltime=00:05:00:00 -l nodes=1:ppn=4 -q otp /somefolder/wrapInScript.sh + False + 0 + subm-host.example.com + /home/testuser + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithEmptyXML.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithEmptyXML.xml new file mode 100644 index 00000000..3c688c0c --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithEmptyXML.xml @@ -0,0 +1,5 @@ + + + 22005.testServer + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithPlaceholderReplacement.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithPlaceholderReplacement.xml new file mode 100644 index 00000000..1ddf87b4 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithPlaceholderReplacement.xml @@ -0,0 +1,7 @@ + + + 22005.testServer + testServer:/logging_root_path/logfile.o$PBS_JOBID + testServer:/logging_root_path/logfile.e$PBS_JOBID + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/createJobManagerWithModifiedExecutionServiceTest.txt b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/createJobManagerWithModifiedExecutionServiceTest.txt new file mode 100644 index 00000000..0b008121 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/createJobManagerWithModifiedExecutionServiceTest.txt @@ -0,0 +1,3 @@ +Line1 +Line2 +Line3 \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithFinishedJob.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithFinishedJob.xml new file mode 100644 index 00000000..d68ed0bb --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithFinishedJob.xml @@ -0,0 +1,317 @@ + + + + + 22005 + 0 + 1548935890 + testuser + 1000 + testuser + 1000 + sge + true + + + testuser + localhost + + + false + RoddyTest_testScript + + + /somefolder/job.o$JOB_ID + + + false + + + 0 + + + h_rss + 4 + 37888M + 39728447488.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + h_rt + 3 + 18000 + 18000.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + + + /bin/bash + + + false + + + + + __SGE_PREFIX__O_HOME + /home/testuser + + + __SGE_PREFIX__O_LOGNAME + testuser + + + __SGE_PREFIX__O_PATH + /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + + + __SGE_PREFIX__O_SHELL + /bin/bash + + + __SGE_PREFIX__O_MAIL + /var/mail/testuser + + + __SGE_PREFIX__O_HOST + localhost + + + __SGE_PREFIX__O_WORKDIR + /home/testuser + + + PARAMETER_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript_1.parameters + + + TOOL_ID + testScript + + + CONFIG_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript_1.parameters + + + debugWrapInScript + false + + + baseEnvironmentScript + /etc/profile + + + + + 65536 + 1 + + + io + 0.000000 + + + iow + 0.000000 + + + mem + 0.000000 + + + cpu + 0.016000 + + + vmem + 0.000000 + + + maxvmem + 0.000000 + + + submission_time + 1548935890.000000 + + + priority + 0.000000 + + + exit_status + 1.000000 + + + signal + 0.000000 + + + start_time + 1548936457.000000 + + + end_time + 1548936457.000000 + + + ru_wallclock + 0.000000 + + + ru_utime + 0.016000 + + + ru_stime + 0.000000 + + + ru_maxrss + 3384.000000 + + + ru_ixrss + 0.000000 + + + ru_ismrss + 0.000000 + + + ru_idrss + 0.000000 + + + ru_isrss + 0.000000 + + + ru_minflt + 5188.000000 + + + ru_majflt + 0.000000 + + + ru_nswap + 0.000000 + + + ru_inblock + 0.000000 + + + ru_oublock + 32.000000 + + + ru_msgsnd + 0.000000 + + + ru_msgrcv + 0.000000 + + + ru_nsignals + 0.000000 + + + ru_nvcsw + 170.000000 + + + ru_nivcsw + 113.000000 + + + acct_cpu + 0.016000 + + + acct_mem + 0.000000 + + + acct_io + 0.000000 + + + acct_iow + 0.000000 + + + acct_maxvmem + 0.000000 + + + + + /home/testuser + + + 22006 + + + 22007 + + + 0 + 0 + 0 + 0 + false + 0 + 1024 + 0 + 0 + 0 + serial + + + 4 + 4 + 1 + + + 0 + 0 + 0 + 0 + 1 + + + 1 + 1 + 1 + + + 0 + + + + + + + 37 + queue instance "main.q@localhost" dropped because it is full + + + + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithJobWithPipedScript.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithJobWithPipedScript.xml new file mode 100644 index 00000000..00fe0a5c --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithJobWithPipedScript.xml @@ -0,0 +1,111 @@ + + + + + 22005 + 0 + job_scripts/145 + 1548934527 + testuser + 1000 + testuser + 1000 + sge + true + + + testuser + localhost + + + false + ATestJob + 0 + + + h_rss + 4 + 2G + 2147483648.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + + + __SGE_PREFIX__O_HOME + /home/testuser + + + __SGE_PREFIX__O_LOGNAME + testuser + + + __SGE_PREFIX__O_PATH + /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + + + __SGE_PREFIX__O_SHELL + /bin/bash + + + __SGE_PREFIX__O_MAIL + /var/mail/testuser + + + __SGE_PREFIX__O_HOST + localhost + + + __SGE_PREFIX__O_WORKDIR + /home/testuser + + + STDIN + + + 128 + 1 + + + 0 + 0 + 0 + 0 + false + 0 + 1024 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 1 + 1 + 1 + + + 0 + + + + + + + 37 + queue instance "main.q@localhost" dropped because it is full + + + + + diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithQueuedJobs.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithQueuedJobs.xml new file mode 100644 index 00000000..1e2ec829 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithQueuedJobs.xml @@ -0,0 +1,632 @@ + + + + + 22003 + 0 + job_scripts/22003 + 1548935890 + testuser + 1000 + testuser + 1000 + sge + true + + + testuser + localhost + + + false + ATestJob + + + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript.o$JOB_ID + + + false + + + 0 + + + h_rss + 4 + 1024M + 1073741824.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + h_rt + 3 + 18000 + 18000.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + + + /bin/bash + + + false + + + + + __SGE_PREFIX__O_HOME + /home/testuser + + + __SGE_PREFIX__O_LOGNAME + testuser + + + __SGE_PREFIX__O_PATH + /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + + + __SGE_PREFIX__O_SHELL + /bin/bash + + + __SGE_PREFIX__O_MAIL + /var/mail/testuser + + + __SGE_PREFIX__O_HOST + localhost + + + __SGE_PREFIX__O_WORKDIR + /home/testuser + + + PARAMETER_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript_1.parameters + + + TOOL_ID + testScript + + + CONFIG_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript_1.parameters + + + debugWrapInScript + false + + + baseEnvironmentScript + /etc/profile + + + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/analysisTools/roddyTools/wrapInScript.sh + /home/testuser + + + 22006 + + + 22007 + + + 0 + 0 + 0 + 0 + false + 0 + 1024 + 0 + 0 + 0 + serial + + + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + + + 1 + 1 + 1 + + + 0 + + + 22005 + 0 + job_scripts/22005 + 1549272907 + heinold + 1000 + heinold + 1000 + sge + true + + + heinold + localhost + + + false + RoddyTest_testScript + + + /home/heinold/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190204_103439242_heinold_testCorrect2/RoddyTest_testScript.o$JOB_ID + + + false + + + 0 + + + h_rss + 4 + 3788M + 3972005888.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + h_rt + 3 + 18000 + 18000.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + + + /bin/bash + + + false + + + + + __SGE_PREFIX__O_HOME + /home/heinold + + + __SGE_PREFIX__O_LOGNAME + heinold + + + __SGE_PREFIX__O_PATH + /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + + + __SGE_PREFIX__O_SHELL + /bin/bash + + + __SGE_PREFIX__O_MAIL + /var/mail/heinold + + + __SGE_PREFIX__O_HOST + localhost + + + __SGE_PREFIX__O_WORKDIR + /home/heinold + + + PARAMETER_FILE + /home/heinold/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190204_103439242_heinold_testCorrect2/RoddyTest_testScript_1.parameters + + + TOOL_ID + testScript + + + CONFIG_FILE + /home/heinold/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190204_103439242_heinold_testCorrect2/RoddyTest_testScript_1.parameters + + + debugWrapInScript + false + + + baseEnvironmentScript + /etc/profile + + + /home/heinold/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190204_103439242_heinold_testCorrect2/analysisTools/roddyTools/wrapInScript.sh + /home/heinold + + + 162 + + + 163 + + + 0 + 0 + 0 + 0 + false + 0 + 1024 + 0 + 0 + 0 + serial + + + 4 + 4 + 1 + + + 0 + 0 + 0 + 0 + 0 + + + 1 + 1 + 1 + + + 0 + + + 22006 + 0 + job_scripts/147 + 1548935902 + testuser + 1000 + testuser + 1000 + sge + true + + + testuser + localhost + + + false + ATestJob + + + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript.o$JOB_ID + + + false + + + 0 + + + h_rss + 4 + 1024M + 1073741824.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + h_rt + 3 + 18000 + 18000.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + + + /bin/bash + + + false + + + + + __SGE_PREFIX__O_HOME + /home/testuser + + + __SGE_PREFIX__O_LOGNAME + testuser + + + __SGE_PREFIX__O_PATH + /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + + + __SGE_PREFIX__O_SHELL + /bin/bash + + + __SGE_PREFIX__O_MAIL + /var/mail/testuser + + + __SGE_PREFIX__O_HOST + localhost + + + __SGE_PREFIX__O_WORKDIR + /home/testuser + + + PARAMETER_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript_2.parameters + + + TOOL_ID + testScript + + + CONFIG_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript_2.parameters + + + debugWrapInScript + false + + + baseEnvironmentScript + /etc/profile + + + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/analysisTools/roddyTools/wrapInScript.sh + /home/testuser + + + 146 + + + + + 148 + + + 0 + 0 + 0 + 0 + false + 0 + 1024 + 0 + 0 + 0 + serial + + + 1 + 1 + 1 + + + + + 0 + 146 + + + 0 + 0 + 0 + 0 + 0 + + + 1 + 1 + 1 + + + 0 + + + 22007 + 0 + job_scripts/148 + 1548935913 + testuser + 1000 + testuser + 1000 + sge + true + + + testuser + localhost + + + false + ATestJob + + + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScriptMultiInMixedParameters.o$JOB_ID + + + false + + + 0 + + + /bin/bash + + + false + + + + + __SGE_PREFIX__O_HOME + /home/testuser + + + __SGE_PREFIX__O_LOGNAME + testuser + + + __SGE_PREFIX__O_PATH + /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + + + __SGE_PREFIX__O_SHELL + /bin/bash + + + __SGE_PREFIX__O_MAIL + /var/mail/testuser + + + __SGE_PREFIX__O_HOST + localhost + + + __SGE_PREFIX__O_WORKDIR + /home/testuser + + + PARAMETER_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScriptMultiInMixedParameters_3.parameters + + + TOOL_ID + testScriptMultiInMixedParameters + + + CONFIG_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScriptMultiInMixedParameters_3.parameters + + + debugWrapInScript + false + + + baseEnvironmentScript + /etc/profile + + + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/analysisTools/roddyTools/wrapInScript.sh + /home/testuser + + + 146 + + + 147 + + + 0 + 0 + 0 + 0 + false + 0 + 1024 + 0 + 0 + 0 + + + 0 + 146 + + + 0 + 147 + + + 0 + 0 + 0 + 0 + 0 + + + 1 + 1 + 1 + + + 0 + + + + + + + + + 146 + + + 147 + + + 148 + + + 33 + Job is in hold state + + + + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithUnknownJobs.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithUnknownJobs.xml new file mode 100644 index 00000000..11130f92 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithUnknownJobs.xml @@ -0,0 +1,9 @@ + + + <> + 153 + + <> + 159 + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/simpleQStatOutput.txt b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/simpleQStatOutput.txt new file mode 100644 index 00000000..7997162d --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/simpleQStatOutput.txt @@ -0,0 +1,6 @@ +job-ID prior name user state submit/start at queue slots ja-task-ID +----------------------------------------------------------------------------------------------------------------- + 22003 0.50000 RoddyTest_ testuser r 01/31/2019 13:07:37 main.q@localhost 1 + 22005 0.50000 RoddyTest_ testuser hqw 01/31/2019 13:07:37 1 + 22006 0.00000 RoddyTest_ testuser hqw 01/31/2019 12:58:22 1 + 22007 0.00000 RoddyTest_ testuser hqw 01/31/2019 12:58:33 1