diff --git a/build.gradle b/build.gradle index f25fcbf8..e237802e 100644 --- a/build.gradle +++ b/build.gradle @@ -46,6 +46,9 @@ configurations.all { } dependencies { + compile 'com.fasterxml.jackson.core:jackson-core:2.9.7' + compile 'com.fasterxml.jackson.core:jackson-annotations:2.9.7' + compile 'com.fasterxml.jackson.core:jackson-databind:2.9.7' compile 'org.codehaus.groovy:groovy-all:2.4.9' testCompile 'junit:junit:4.12' testCompile 'org.spockframework:spock-core:1.1-groovy-2.4' @@ -60,8 +63,7 @@ dependencies { compile group: 'commons-cli', name: 'commons-cli', version: '1.2' compile group: 'org.apache.commons', name: 'commons-text', version: '1.1' compile 'com.google.guava:guava:23.0' - // compile 'com.github.eilslabs:RoddyToolLib:master-SNAPSHOT' - compile 'com.github.theroddywms:RoddyToolLib:2.0.0' + compile 'com.github.theroddywms:RoddyToolLib:2.1.1' } task writePom { diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index ed88a042..457aad0d 100755 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 9ec837b2..558870da 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-all.zip diff --git a/gradlew b/gradlew index cccdd3d5..af6708ff 100755 --- a/gradlew +++ b/gradlew @@ -28,7 +28,7 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" diff --git a/gradlew.bat b/gradlew.bat index e95643d6..0f8d5937 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome diff --git a/src/main/groovy/de/dkfz/roddy/config/ResourceSet.groovy b/src/main/groovy/de/dkfz/roddy/config/ResourceSet.groovy index d23b18c3..53d3c354 100644 --- a/src/main/groovy/de/dkfz/roddy/config/ResourceSet.groovy +++ b/src/main/groovy/de/dkfz/roddy/config/ResourceSet.groovy @@ -4,10 +4,12 @@ import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.RoddyConversionHelperMethods import de.dkfz.roddy.tools.TimeUnit import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode import java.time.Duration @CompileStatic +@EqualsAndHashCode class ResourceSet { private final String queue private ResourceSetSize size diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEFakeJobID.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEFakeJobID.groovy index 42af20e2..7de70c22 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEFakeJobID.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEFakeJobID.groovy @@ -13,7 +13,7 @@ import static de.dkfz.roddy.execution.jobs.BEFakeJobID.FakeJobReason.values /** * Created by heinold on 23.02.17. */ -class BEFakeJobID extends BEJobID.FakeJobID { +class BEFakeJobID extends BEJobID { /** * Various reasons why a job was not executed and is a fake job. */ diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobID.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobID.groovy index e37083fc..61cfea3b 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobID.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobID.groovy @@ -53,13 +53,6 @@ class BEJobID implements Comparable { return id } - @Deprecated - static class FakeJobID extends BEJobID { - FakeJobID(String id) { - super(id) - } - } - @Override int compareTo(BEJobID o) { return this.id.compareTo(o.id) diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy index 53ef9cac..ed197c65 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy @@ -14,8 +14,6 @@ import groovy.transform.CompileStatic import org.slf4j.Logger import org.slf4j.LoggerFactory -import java.time.Duration -import java.time.LocalDateTime import java.util.concurrent.TimeoutException /** @@ -29,7 +27,7 @@ abstract class BatchEuphoriaJobManager { final static Logger log = LoggerFactory.getLogger(BatchEuphoriaJobManager) - protected final BEExecutionService executionService + final BEExecutionService executionService protected boolean isTrackingOfUserJobsEnabled @@ -45,30 +43,12 @@ abstract class BatchEuphoriaJobManager { private String userAccount - private final Map activeJobs = [:] - - private Thread updateDaemonThread - - /** - * Set this to true to tell the job manager, that an existing update daemon shall be closed, e.g. because - * the application is in the process to exit. - */ - protected boolean updateDaemonShallBeClosed /** * Set this to true, if you do not want to allow any further job submission. */ protected boolean forbidFurtherJobSubmission - private Map cachedStates = [:] - private final Object cacheStatesLock = new Object() - private LocalDateTime lastCacheUpdate - private Duration cacheUpdateInterval - - public boolean surveilledJobsHadErrors = false - - private final List updateDaemonListeners = [] - boolean requestMemoryIsEnabled boolean requestWalltimeIsEnabled boolean requestQueueIsEnabled @@ -89,7 +69,6 @@ abstract class BatchEuphoriaJobManager { this.userGroup = parms.userGroup this.userAccount = parms.userAccount this.userMask = parms.userMask - this.cacheUpdateInterval = parms.updateInterval this.requestMemoryIsEnabled = parms.requestMemoryIsEnabled this.requestWalltimeIsEnabled = parms.requestWalltimeIsEnabled @@ -99,9 +78,6 @@ abstract class BatchEuphoriaJobManager { this.passEnvironment = parms.passEnvironment this.holdJobsIsEnabled = Optional.ofNullable(parms.holdJobIsEnabled).orElse(getDefaultForHoldJobsEnabled()) - if (parms.createDaemon) { - createUpdateDaemonThread() - } } /** @@ -122,6 +98,10 @@ abstract class BatchEuphoriaJobManager { return job.runResult } + @Deprecated + void addToListOfStartedJobs(BEJob job) { + } + /** * Resume given job * @param job @@ -139,8 +119,13 @@ abstract class BatchEuphoriaJobManager { } } + static final assertListIsValid(List list) { + assert list && list.every { it != null } + } + /** * Try to abort a list of jobs + * * @param jobs */ void killJobs(List jobs) { @@ -156,6 +141,16 @@ abstract class BatchEuphoriaJobManager { } } + final JobInfo queryJobInfoByJob(BEJob job) { + assert job + queryJobInfoByID(job.jobID) + } + + final JobInfo queryJobInfoByID(BEJobID jobID) { + assert jobID + return queryJobInfoByID([jobID])[jobID] + } + /** * Queries the status of all jobs in the list. * @@ -166,36 +161,66 @@ abstract class BatchEuphoriaJobManager { * @param jobs * @return */ - Map queryJobStatus(List jobs, boolean forceUpdate = false) { - Map result = queryJobStatesUsingCache(jobs*.jobID, forceUpdate) - return jobs.collectEntries { BEJob job -> - [job, job.jobState == JobState.ABORTED ? JobState.ABORTED : - result[job.jobID] ?: JobState.UNKNOWN] + final Map queryJobInfoByJob(List jobs) { + assertListIsValid(jobs) + + Map jobsByID = jobs.collectEntries { + def job = it + def id = it.jobID + [id, job] } + + return queryJobInfoByID(jobs*.jobID) + .collectEntries { + BEJobID id, JobInfo info -> [jobsByID[id], info] + } as Map } /** - * Queries the status of all jobs in the list. + * Query a list of job states by their id. + * If the status of a job could not be determined, the query will return UNKNOWN as the state for this job. * - * Every job ID in the list is supposed to have an entry in the result map. If - * the manager cannot retrieve info about the job, the result will be UNKNOWN - * for this particular job. + * @param jobIDs + * @return A map of all job states accessible by their id. + */ + final Map queryJobInfoByID(List jobIDs) { + assertListIsValid(jobIDs) + + Map result = queryJobInfo(jobIDs) ?: [:] as Map + + // Collect the result and fill in empty entries with UNKNOWN + return jobIDs.collectEntries { + BEJobID jobID -> + [jobID, result[jobID] ?: new JobInfo(jobID)] + } + } + + /** + * Query the states of all jobs available limited by the settings provided in the constructor. * - * @param jobIds * @return */ - Map queryJobStatusById(List jobIds, boolean forceUpdate = false) { - Map result = queryJobStatesUsingCache(jobIds, forceUpdate) - return jobIds.collectEntries { BEJobID jobId -> [jobId, result[jobId] ?: JobState.UNKNOWN] } + final Map queryAllJobInfo() { + queryJobInfo(null) } /** - * Queries the status of all jobs. + * Needs to be overriden by JobManager implementations. + * jobIDs may be null or [] which means, that the query is not for specific jobs but for all available (except for + * the filter settings of the JobManager) jobs. * - * @return + * @param jobIDs A list of IDs OR null/[] */ - Map queryJobStatusAll(boolean forceUpdate = false) { - return queryJobStatesUsingCache(null, forceUpdate) + abstract Map queryJobInfo(List jobIDs) + + final ExtendedJobInfo queryExtendedJobInfoByJob(BEJob job) { + assert job + return queryExtendedJobInfoByID(job.jobID) + } + + final ExtendedJobInfo queryExtendedJobInfoByID(BEJobID id) { + assert id + return queryExtendedJobInfoByID([id])[id] } /** @@ -204,15 +229,17 @@ abstract class BatchEuphoriaJobManager { * - The used cores * - The used walltime * - * @param jobs - * @return + * If the jobid cannot be found, an empty extend job info object will be returned with the jobstate set to UNKNOWN */ - Map queryExtendedJobState(List jobs) { + final Map queryExtendedJobInfoByJob(List jobs) { + assertListIsValid(jobs) - Map queriedExtendedStates = queryExtendedJobStateById(jobs.collect { it.getJobID() }) - return (Map) queriedExtendedStates.collectEntries { - Map.Entry it -> [jobs.find { BEJob temp -> temp.getJobID() == it.key }, (GenericJobInfo) it.value] - } + Map jobsByID = jobs.collectEntries { [it.jobID, it] } + + return queryExtendedJobInfoByID(jobs*.jobID) + .collectEntries { + BEJobID id, JobInfo info -> [jobsByID[id], info] + } as Map } /** @@ -221,19 +248,34 @@ abstract class BatchEuphoriaJobManager { * - The used cores * - The used walltime * - * @param jobIds - * @return + * If the jobid cannot be found, an empty extend job info object will be returned with the jobstate set to UNKNOWN */ - abstract Map queryExtendedJobStateById(List jobIds) - - void addToListOfStartedJobs(BEJob job) { - if (updateDaemonThread) { - synchronized (activeJobs) { - activeJobs.put(job.getJobID(), job) - } - } + final Map queryExtendedJobInfoByID(List jobIDs) { + assertListIsValid(jobIDs) + + Map queriedExtendedStates = queryExtendedJobInfo(jobIDs) ?: [:] as Map + + jobIDs.collectEntries { + BEJobID jobID -> + [ + jobID, + queriedExtendedStates[jobID] ?: new ExtendedJobInfo(jobID) + ] + } as Map } + final Map queryAllExtendedJobInfo() { return queryExtendedJobInfo(null) } + + /** + * Needs to be overriden by JobManager implementations. + * jobIDs may be null or [] which means, that the query is not for specific jobs but for all available (except for + * the filter settings of the JobManager) jobs. + * + * @param jobIDs A list of IDs OR null/[] + * @return a list of jobs + */ + abstract Map queryExtendedJobInfo(List jobIDs) + abstract String getJobIdVariable() abstract String getJobNameVariable() @@ -278,9 +320,9 @@ abstract class BatchEuphoriaJobManager { abstract String getSubmissionCommand() - abstract String getQueryJobStatesCommand() + abstract String getQueryCommandForJobInfo() - abstract String getExtendedQueryJobStatesCommand() + abstract String getQueryCommandForExtendedJobInfo() ProcessingParameters convertResourceSet(BEJob job) { return convertResourceSet(job, job.resourceSet) @@ -295,7 +337,7 @@ abstract class BatchEuphoriaJobManager { * @param commandString * @return */ - abstract GenericJobInfo parseGenericJobInfo(String command) + abstract ExtendedJobInfo parseGenericJobInfo(String command) protected List collectJobIDsFromJobs(List jobs) { BEJob.jobsWithUniqueValidJobId(jobs).collect { it.runResult.getJobID() } @@ -315,9 +357,6 @@ abstract class BatchEuphoriaJobManager { job.resetJobID(jobID) jobResult = new BEJobResult(command, job, res, job.tool, job.parameters, job.parentJobs as List) job.setRunResult(jobResult) - synchronized (cacheStatesLock) { - cachedStates.put(jobID, isHoldJobsEnabled() ? JobState.HOLD : JobState.QUEUED) - } } else { def job = command.getJob() jobResult = new BEJobResult(command, job, res, job.tool, job.parameters, job.parentJobs as List) @@ -333,100 +372,59 @@ abstract class BatchEuphoriaJobManager { abstract protected JobState parseJobState(String stateString) - abstract protected Map queryJobStates(List jobIDs) - abstract protected ExecutionResult executeStartHeldJobs(List jobIDs) abstract protected ExecutionResult executeKillJobs(List jobIDs) - protected void createUpdateDaemonThread() { - updateDaemonThread = Thread.startDaemon("Job state update daemon.", { - while (!updateDaemonShallBeClosed) { - updateActiveJobList() - - waitForUpdateIntervalDuration() - } - }) - } - - final void addUpdateDaemonListener(UpdateDaemonListener listener) { - synchronized (updateDaemonListeners) { - updateDaemonListeners << listener - } - } - - boolean isDaemonAlive() { - return updateDaemonThread != null && updateDaemonThread.isAlive() - } - void waitForUpdateIntervalDuration() { - long duration = Math.max(cacheUpdateInterval.toMillis(), 10 * 1000) - // Sleep one second until the duration is reached. This allows the daemon to finish faster, when it shall stop - // (updateDaemonShallBeClosed == true) - for (long timer = duration; timer > 0 && !updateDaemonShallBeClosed; timer -= 1000) - Thread.sleep(1000) + @Deprecated + Map queryJobStatus(List jobs) { + queryJobInfoByJob(jobs).collectEntries { BEJob job, JobInfo ji -> [job, ji.jobState] } as Map } - void stopUpdateDaemon() { - updateDaemonShallBeClosed = true - updateDaemonThread?.join() + /** + * Queries the status of all jobs in the list. + * + * Every job ID in the list is supposed to have an entry in the result map. If + * the manager cannot retrieve info about the job, the result will be UNKNOWN + * for this particular job. + * + * @param jobIds + * @return + */ + @Deprecated + Map queryJobStatusById(List jobIds) { + queryJobInfoByID(jobIds).collectEntries { BEJobID id, JobInfo ji -> [id, ji.jobState] } as Map } - private void updateActiveJobList() { - List listOfRemovableJobs = [] - synchronized (activeJobs) { - Map states = queryJobStatesUsingCache(activeJobs.keySet() as List, true) - - for (BEJobID id : activeJobs.keySet()) { - JobState jobState = states.get(id) - BEJob job = activeJobs.get(id) - - job.setJobState(jobState) - if (!jobState.isPlannedOrRunning()) { - synchronized (updateDaemonListeners) { - updateDaemonListeners.each { it.jobEnded(job, jobState) } - } - if (!jobState.successful) { - surveilledJobsHadErrors = true - } - listOfRemovableJobs << id - } - } - listOfRemovableJobs.each { activeJobs.remove(it) } - } + /** + * Queries the status of all jobs. + * + * @return + */ + @Deprecated + Map queryJobStatusAll(boolean forceUpdate = false) { + return queryAllJobInfo().collectEntries { BEJobID id, JobInfo ji -> [id, ji.jobState] } as Map } - private Map queryJobStatesUsingCache(List jobIDs, boolean forceUpdate) { - if (forceUpdate || lastCacheUpdate == null || cacheUpdateInterval == Duration.ZERO || - Duration.between(lastCacheUpdate, LocalDateTime.now()) > cacheUpdateInterval) { - synchronized (cacheStatesLock) { - cachedStates = queryJobStates(jobIDs) - } - lastCacheUpdate = LocalDateTime.now() - } - return new HashMap(cachedStates) + @Deprecated + Map queryExtendedJobState(List jobs) { + queryExtendedJobInfoByJob(jobs) } /** - * The method will wait until all started jobs are finished (with or without errors). - * - * Note, that the method does not allow further job submission! As soon, as you call it, you cannot submit jobs! + * Will be used to gather extended information about a job like: + * - The used memory + * - The used cores + * - The used walltime * - * @return true, if there were NO errors, false, if there were any. + * @param jobIds + * @return */ - boolean waitForJobsToFinish() { - if (!updateDaemonThread) { - throw new BEException("The job manager must be created with JobManagerOption.createDaemon set to true to make waitForJobsToFinish() work.") - } - forbidFurtherJobSubmission = true - while (!updateDaemonShallBeClosed) { - synchronized (activeJobs) { - if (activeJobs.isEmpty()) { - break - } - } - waitForUpdateIntervalDuration() - } - return !surveilledJobsHadErrors + @Deprecated + Map queryExtendedJobStateById(List jobIds) { + return queryExtendedJobInfo(jobIds) } + + } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfo.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfo.groovy new file mode 100644 index 00000000..0762d523 --- /dev/null +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfo.groovy @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2017 eilslabs. + * + * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). + */ + +package de.dkfz.roddy.execution.jobs + +import de.dkfz.roddy.config.ResourceSet +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString + +import java.time.Duration +import java.time.ZonedDateTime + +/** + * Stores extended information like e.g. statistical data about a job + * + * Note, that the amount of gathered information depends on the target system and can vary with different job states. + * E.g. the fields + * + * This class is a value class. Normally, we'd use final fields for all values or some kind of + * builder interface to create instances of it. But due to the sheer amount of its fields, + * we will refrain from both. Please make sure, that you do net mess things up. + */ +@CompileStatic +@ToString(includeNames = true) +@EqualsAndHashCode(callSuper = true) +class ExtendedJobInfo extends JobInfo { + + // Common information + + String jobName + + String user + + String userGroup + + /** + * Umask with which files and directories are created + */ + String umask + + String execUserName + + String description + + String projectName + + String jobGroup + + /** + * The executed command + */ + String command + + /** + * Job parameters during submission + */ + Map parameters + + /** + * IDs of the jobs parent jobs + */ + List parentJobIDs + + String priority + + /** + * List of process ids within the job + */ + List processesInJob + + /** + * Currently active process group ID in a job. + */ + String processGroupID + + String submissionHost + + String account + + /** + * Submission server + */ + String server + + // Resources + + String rawHostQueryString + + List executionHosts + + /** + * Requested resources like e.g. walltime, max memory + */ + ResourceSet requestedResources + + /** + * Used resources like e.g. walltime, max memory + */ + ResourceSet usedResources + + /** + * resource requirements + */ + String rawResourceRequest + + /** + * Time in seconds that the job has been in the run state + */ + Duration runTime + + /** + * Cumulative total CPU time in seconds of all processes in a job + */ + Duration cpuTime + + // Status + + /** + * (UNIX) exit status of the job + */ + Integer exitCode + + /** + * Reason, why a job is on hold or suspended + */ + String pendReason + + /** + * How often was the job started + */ + Integer startCount + + // Directories + + /** + * Current working directory + */ + String cwd + + /** + * Executed current working directory + */ + String execCwd + + String execHome + + File logFile + + File errorLogFile + + File inputFile + + // Timestamps and timing info + + /** + * user time used + */ + String userTime + + /** + * system time used + */ + String systemTime + + /** + * The date-time the job entered the queue. + */ + ZonedDateTime submitTime + + /** + * The date-time the job became eligible to run when all conditions like job dependencies are met, i.e. in a queued state while residing in an execution queue. + */ + ZonedDateTime eligibleTime + + /** + * The date-time the job was started. + */ + ZonedDateTime startTime + + /** + * Suspended by its owner or the LSF administrator after being dispatched + */ + Duration timeUserSuspState + + /** + * Waiting in a queue for scheduling and dispatch + */ + Duration timePendState + + /** + * Suspended by its owner or the LSF administrator while in PEND state + */ + Duration timePendSuspState + + /** + * Suspended by the job system after being dispatched + */ + Duration timeSystemSuspState + + Duration timeUnknownState + + ZonedDateTime timeOfCalculation + + // Whatever... + + /** + * Guess what... don't know + */ + String otherSettings + + + ExtendedJobInfo(BEJobID jobID) { + super(jobID) + } + + ExtendedJobInfo(BEJobID jobID, JobState jobState) { + super(jobID) + this.jobState = jobState + } + + @Deprecated + ExtendedJobInfo(String jobName, String command, BEJobID jobID, Map parameters, List parentJobIDs) { + super(jobID) + this.jobName = jobName + this.command = command + this.jobID = jobID + this.parameters = parameters + this.parentJobIDs = parentJobIDs + } + +} diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy deleted file mode 100644 index 2a550e89..00000000 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/GenericJobInfo.groovy +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2017 eilslabs. - * - * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). - */ - -package de.dkfz.roddy.execution.jobs - -import de.dkfz.roddy.config.ResourceSet -import groovy.transform.CompileStatic -import groovy.transform.ToString - -import java.time.Duration -import java.time.ZonedDateTime - -/** - * Created by michael on 06.02.15. - */ -@CompileStatic -@ToString(includeNames = true) -class GenericJobInfo { - - ResourceSet askedResources - ResourceSet usedResources - String jobName - File tool - BEJobID jobID - - /** The date-time the job entered the queue. */ - ZonedDateTime submitTime - /** The date-time the job became eligible to run when all conditions like job dependencies are met, i.e. in a queued state while residing in an execution queue. */ - ZonedDateTime eligibleTime - /** The date-time the job was started. */ - ZonedDateTime startTime - /** The date-time the job was completed. */ - ZonedDateTime endTime - - List executionHosts - String submissionHost - String priority - - File logFile - File errorLogFile - File inputFile - - String user - String userGroup - String resourceReq // resource requirements - Integer startCount - - String account - String server - String umask - - Map parameters - List parentJobIDs - String otherSettings - JobState jobState - String userTime //user time used - String systemTime //system time used - String pendReason - String execHome - String execUserName - List pidStr - String pgidStr // Currently active process group ID in a job. - Integer exitCode // UNIX exit status of the job - String jobGroup - String description - String execCwd //Executed current working directory - String askedHostsStr - String cwd //Current working directory - String projectName - - Duration cpuTime //Cumulative total CPU time in seconds of all processes in a job - Duration runTime //Time in seconds that the job has been in the run state - Duration timeUserSuspState //Suspended by its owner or the LSF administrator after being dispatched - Duration timePendState //Waiting in a queue for scheduling and dispatch - Duration timePendSuspState // Suspended by its owner or the LSF administrator while in PEND state - Duration timeSystemSuspState //Suspended by the LSF system after being dispatched - Duration timeUnknownState - ZonedDateTime timeOfCalculation - - - GenericJobInfo(String jobName, File tool, BEJobID jobID, Map parameters, List parentJobIDs) { - this.jobName = jobName - this.tool = tool - this.jobID = jobID - this.parameters = parameters - this.parentJobIDs = parentJobIDs - } -} diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/JobInfo.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/JobInfo.groovy new file mode 100644 index 00000000..4c43ebf3 --- /dev/null +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/JobInfo.groovy @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). + * + * Distributed under the MIT License (license terms are at https://www.github.com/TheRoddyWMS/BatchEuphoria/LICENSE.txt). + */ + +package de.dkfz.roddy.execution.jobs + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode + +import java.time.ZonedDateTime + +/** + * The class contains the bare minimum of information about a job gathered by + * BatchEuphoriaJobManager.queryJobInfo() + * Note that all fields except jobID can be undefined / null! + * + * There are three different settings for the content of this class: + * - A job was not found by the manager, the state will be set to UNKNOWN, with no endTime + * - A job was found but is still running, the state will be set accordingly, with no endTime + * - A job was found and is finished, the state will be set accordingly with the endTime set + * + * This class is a value class. Normally, we'd use final fields for all values or some kind of + * builder interface to create instances of it. But due to the sheer size of ExtendedJobInfo, + * we will refrain from both. Please make sure, that you do net mess things up. + */ +@CompileStatic +@EqualsAndHashCode +class JobInfo { + + BEJobID jobID + + /** + * The date-time the job was completed. + * Can be null, e.g. if the batch system cannot track the job anymore. + * */ + ZonedDateTime endTime + + JobState jobState = JobState.UNKNOWN + + JobInfo(BEJobID jobID) { + this.jobID = jobID + } +} diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy index 57df38a8..9b6ddec5 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/JobManagerOptions.groovy @@ -17,15 +17,18 @@ import java.time.ZoneId @CompileStatic class JobManagerOptions { - boolean createDaemon Duration updateInterval String userIdForJobQueries + boolean trackOnlyStartedJobs String userGroup + String userAccount + String userEmail + String userMask /** @@ -84,13 +87,12 @@ class JobManagerOptionsBuilder { JobManagerOptionsBuilder() { trackOnlyStartedJobs = false updateInterval = Duration.ofMinutes(5) - createDaemon = false requestMemoryIsEnabled = true requestWalltimeIsEnabled = true requestQueueIsEnabled = true requestCoresIsEnabled = true - requestStorageIsEnabled = false // Defaults to false, not supported now. - passEnvironment = false // Setting this to true should be a conscious decision. Therefore the default 'false'. + requestStorageIsEnabled = false // Defaults to false, not supported now. + passEnvironment = false // Setting this to true should be a conscious decision. Therefore the default 'false'. additionalOptions = [:] timeZoneId = ZoneId.systemDefault() } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/QueryJobStatesFilter.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/QueryJobStatesFilter.groovy new file mode 100644 index 00000000..46a6abcf --- /dev/null +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/QueryJobStatesFilter.groovy @@ -0,0 +1,8 @@ +package de.dkfz.roddy.execution.jobs + +import groovy.transform.CompileStatic + +@CompileStatic +class QueryJobStatesFilter { + boolean keep() { return true }; +} diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommand.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommand.groovy index 9155590e..d589c782 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommand.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommand.groovy @@ -61,13 +61,6 @@ abstract class SubmissionCommand extends Command { passEnvironment.orElse(parentJobManager.passEnvironment) } - @Deprecated - @Override - String toString() { - // TODO toString() shouldn't be used for such specific things. Remove explicit calls to get the bash command representation. - return toBashCommandString() - } - @Override String toBashCommandString() { String email = parentJobManager.getUserEmail() diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManager.groovy index 5a26c61a..87b22b47 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManager.groovy @@ -16,6 +16,7 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import java.time.Duration +import java.time.ZonedDateTime /** * A class for processing backends running on a cluster. @@ -29,7 +30,7 @@ abstract class ClusterJobManager extends BatchEuphoriaJobMana super(executionService, parms) } - protected static T withCaughtAndLoggedException(final Closure closure) { + static T withCaughtAndLoggedException(final Closure closure) { try { return closure.call() } catch (Exception e) { @@ -46,7 +47,36 @@ abstract class ClusterJobManager extends BatchEuphoriaJobMana return null } - protected static Duration parseColonSeparatedHHMMSSDuration(String str) { + static BEJobID toJobID(String jobIdRaw) { + BEJobID jobID + try { + jobID = new BEJobID(jobIdRaw) + } catch (Exception exp) { + throw new BEException("Job ID '${jobIdRaw}' could not be transformed to BEJobID ") + } + jobID + } + + static Duration safelyParseColonSeparatedDuration(Object value) { + String _value = value as String + withCaughtAndLoggedException { + return _value ? parseColonSeparatedHHMMSSDuration(_value) : null + } + } + + ZonedDateTime safelyParseTime(Object time) { + String _time = time as String + if (time) + return withCaughtAndLoggedException { + return parseTime(_time) + } + return null + } + + abstract ZonedDateTime parseTime(String time) + + + static Duration parseColonSeparatedHHMMSSDuration(String str) { String[] hhmmss = str.split(":") if (hhmmss.size() != 3) { throw new BEException("Duration string is not of the format HH+:MM:SS: '${str}'") @@ -99,4 +129,6 @@ abstract class ClusterJobManager extends BatchEuphoriaJobMana abstract void createMemoryParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) abstract void createStorageParameters(LinkedHashMultimap parameters, ResourceSet resourceSet) + + } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManager.groovy index 0f2914fb..70d8e4e0 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManager.groovy @@ -1,8 +1,13 @@ +/* + * Copyright (c) 2019 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). + * + * Distributed under the MIT License (license terms are at https://www.github.com/TheRoddyWMS/BatchEuphoria/LICENSE.txt). + */ + package de.dkfz.roddy.execution.jobs.cluster import com.google.common.collect.LinkedHashMultimap import de.dkfz.roddy.BEException -import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.execution.io.ExecutionResult import de.dkfz.roddy.execution.jobs.* @@ -11,13 +16,11 @@ import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.RoddyConversionHelperMethods import de.dkfz.roddy.tools.TimeUnit import groovy.transform.CompileStatic -import groovy.util.slurpersupport.GPathResult import java.time.Duration import java.time.Instant import java.time.ZoneId import java.time.ZonedDateTime -import java.util.regex.Matcher @CompileStatic abstract class GridEngineBasedJobManager extends ClusterJobManager { @@ -25,6 +28,10 @@ abstract class GridEngineBasedJobManager extends ClusterJobMa public static final String WITH_DELIMITER = '(?=(%1$s))' private final ZoneId TIME_ZONE_ID + @Override + ZonedDateTime parseTime(String str) { + return ZonedDateTime.ofInstant(Instant.ofEpochSecond(str as long), TIME_ZONE_ID) + } GridEngineBasedJobManager(BEExecutionService executionService, JobManagerOptions parms) { super(executionService, parms) @@ -48,65 +55,54 @@ abstract class GridEngineBasedJobManager extends ClusterJobMa } @Override - protected Map queryJobStates(List jobIDs) { - StringBuilder queryCommand = new StringBuilder(getQueryJobStatesCommand()) - - if (jobIDs && jobIDs.size() < 10) { - queryCommand << " " << jobIDs*.id.join(" ") - } - - if (isTrackingOfUserJobsEnabled) - queryCommand << " -u $userIDForQueries " + Map queryJobInfo(List jobIDs) { + String queryCommand = assembleJobInfoQueryCommand(jobIDs) - ExecutionResult er = executionService.execute(queryCommand.toString()) + ExecutionResult er = executionService.execute(queryCommand) List resultLines = er.resultLines - Map result = [:] - - if (!er.successful) { + if (!er.successful) throw new BEException("The execution of ${queryCommand} failed.\n\t" + er.resultLines?.join("\n\t")?.toString()) - } else { - if (resultLines.size() > 2) { - for (String line : resultLines) { - line = line.trim() - if (line.length() == 0) continue - if (!RoddyConversionHelperMethods.isInteger(line.substring(0, 1))) - continue //Filter out lines which have been missed which do not start with a number. + if (resultLines.size() < 2) + return [:] - String[] split = line.split("\\s+") - final int ID = getColumnOfJobID() - final int JOBSTATE = getColumnOfJobState() + Map result = [:] + for (String line : resultLines) { + line = line.trim() + if (line.length() == 0) continue + if (!RoddyConversionHelperMethods.isInteger(line.substring(0, 1))) + continue //Filter out lines which have been missed which do not start with a number. - BEJobID jobID = new BEJobID(split[ID]) + String[] split = line.split("\\s+") + final int ID = getColumnOfJobID() + final int JOBSTATE = getColumnOfJobState() - JobState js = parseJobState(split[JOBSTATE]) - result.put(jobID, js) - } - } + BEJobID jobID = new BEJobID(split[ID]) + if (jobIDs && !jobIDs.contains(jobID)) + continue //Ignore ids which are not queried but keep them, if we don't use a filter (jobIDs). + + JobState jobState = parseJobState(split[JOBSTATE]) + def info = new JobInfo(jobID) + info.jobState = jobState + + result[jobID] = info } return result } - @Override - Map queryExtendedJobStateById(List jobIds) { - Map queriedExtendedStates - String qStatCommand = getExtendedQueryJobStatesCommand() - qStatCommand += " " + jobIds.collect { it }.join(" ") - - if (isTrackingOfUserJobsEnabled) - qStatCommand += " -u $userIDForQueries " + String assembleJobInfoQueryCommand(List jobIDs) { + StringBuilder queryCommand = new StringBuilder(getQueryCommandForJobInfo()) - ExecutionResult er = executionService.execute(qStatCommand.toString()) + if (jobIDs && jobIDs.size() < 10) + queryCommand << " " << jobIDs*.id.join(" ") - if (er != null && er.successful) { - queriedExtendedStates = this.processQstatOutputFromXML(er.resultLines.join("\n")) - } else { - throw new BEException("Extended job states couldn't be retrieved. \n Returned status code:${er.exitCode} \n ${qStatCommand.toString()} \n\t result:${er.resultLines.join("\n\t")}") - } - return queriedExtendedStates + if (isTrackingOfUserJobsEnabled) + queryCommand << " -u $userIDForQueries " + queryCommand.toString() } + @Override protected ExecutionResult executeStartHeldJobs(List jobIDs) { String command = "qrls ${jobIDs*.id.join(" ")}" @@ -119,162 +115,75 @@ abstract class GridEngineBasedJobManager extends ClusterJobMa return executionService.execute(command, false) } - /** - * Reads qstat output - * @param qstatOutput - * @return output of qstat in a map with jobid as key - */ - private static Map> processQstatOutputFromPlainText(String qstatOutput) { - return qstatOutput.split(String.format(WITH_DELIMITER, "\n\nJob Id: ")).collectEntries { - Matcher matcher = it =~ /^\s*Job Id: (?\d+)\..*\n/ - def result = new HashMap() - if (matcher) { - result[matcher.group("jobId")] = it - } - result - }.collectEntries { jobId, value -> - // join multi-line values - value = ((String) value).replaceAll("\n\t", "") - [(jobId): value] - }.collectEntries { jobId, value -> - Map p = ((String) value).readLines(). - findAll { it.startsWith(" ") && it.contains(" = ") }. - collectEntries { - String[] parts = it.split(" = ") - new MapEntry(parts.head().replaceAll(/^ {4}/, ""), parts.tail().join(' ')) - } - [(jobId): p] - } as Map> + BufferValue safelyCastToBufferValue(Object value) { + String _value = value as String + if (_value) + return withCaughtAndLoggedException { new BufferValue(Integer.valueOf(_value.find(/(\d+)/)), BufferUnit.valueOf(_value[-2])) } + return null } - private ZonedDateTime parseTime(String str) { - return withCaughtAndLoggedException { ZonedDateTime.ofInstant(Instant.ofEpochSecond(str as long), TIME_ZONE_ID) } + Integer safelyCastToInteger(Object value) { + if (value) + return withCaughtAndLoggedException { return Integer.valueOf(value as String) } + return null } - /** - * Reads the qstat output and creates GenericJobInfo objects - * @param resultLines - Input of ExecutionResult object - * @return map with jobid as key - */ - protected Map processQstatOutputFromXML(String result) { - Map queriedExtendedStates = [:] - if (result.isEmpty()) { - return [:] - } + Duration safelyCastToDuration(Object value) { + String _value = value as String + if (_value) + return withCaughtAndLoggedException { Duration.ofSeconds(Math.round(Double.parseDouble(_value)), 0) } + return null + } - GPathResult parsedJobs = new XmlSlurper().parseText(result) - for (job in parsedJobs.children()) { - String jobIdRaw = job["Job_Id"] as String - BEJobID jobID - try { - jobID = new BEJobID(jobIdRaw) - } catch (Exception exp) { - throw new BEException("Job ID '${jobIdRaw}' could not be transformed to BEJobID ") - } - - List jobDependencies = withCaughtAndLoggedException { getJobDependencies(job["depend"] as String) } - String jobName = job["Job_Name"] as String ?: null - GenericJobInfo gj = new GenericJobInfo(jobName, null, jobID, null, jobDependencies) - - BufferValue mem = null - Integer cores - Integer nodes - TimeUnit walltime = null - String additionalNodeFlag - - Object resourceList = job["Resource_List"] - String resourcesListMem = resourceList["mem"] as String - String resourcesListNoDect = resourceList["nodect"] as String - String resourcesListNodes = resourceList["nodes"] as String - String resourcesListWalltime = resourceList["walltime"] as String - if (resourcesListMem) - mem = withCaughtAndLoggedException { new BufferValue(Integer.valueOf(resourcesListMem.find(/(\d+)/)), BufferUnit.valueOf(resourcesListMem[-2])) } - if (resourcesListNoDect) - nodes = withCaughtAndLoggedException { Integer.valueOf(resourcesListNoDect) } - if (resourcesListNodes) - cores = withCaughtAndLoggedException { Integer.valueOf(resourcesListNodes.find("ppn=.*").find(/(\d+)/)) } - if (resourcesListNodes) - additionalNodeFlag = withCaughtAndLoggedException { resourcesListNodes.find(/(\d+):(\.*)/) { fullMatch, nCores, feature -> return feature } } - if (resourcesListWalltime) - walltime = withCaughtAndLoggedException { new TimeUnit(resourcesListWalltime) } - - BufferValue usedMem = null - TimeUnit usedWalltime = null - Object resourcesUsed = job["resources_used"] - String resourcedUsedMem = resourcesUsed["mem"] as String - String resourcesUsedWalltime = resourcesUsed["walltime"] as String - if (resourcedUsedMem) - withCaughtAndLoggedException { usedMem = new BufferValue(Integer.valueOf(resourcedUsedMem.find(/(\d+)/)), BufferUnit.valueOf(resourcedUsedMem[-2])) } - if (resourcesUsedWalltime) - withCaughtAndLoggedException { usedWalltime = new TimeUnit(resourcesUsedWalltime) } - - gj.setAskedResources(new ResourceSet(null, mem, cores, nodes, walltime, null, job["queue"] as String ?: null, additionalNodeFlag)) - gj.setUsedResources(new ResourceSet(null, usedMem, null, null, usedWalltime, null, job["queue"] as String ?: null, null)) - - gj.setLogFile(withCaughtAndLoggedException { getQstatFile(job["Output_Path"] as String, jobIdRaw) }) - gj.setErrorLogFile(withCaughtAndLoggedException { getQstatFile(job["Error_Path"] as String, jobIdRaw) }) - gj.setUser(job["euser"] as String ?: null) - gj.setExecutionHosts(withCaughtAndLoggedException { getExecutionHosts(job["exec_host"] as String) }) - gj.setSubmissionHost(job["submit_host"] as String ?: null) - gj.setPriority(job["Priority"] as String ?: null) - gj.setUserGroup(job["egroup"] as String ?: null) - gj.setResourceReq(job["submit_args"] as String ?: null) - gj.setRunTime(job["total_runtime"] ? withCaughtAndLoggedException { Duration.ofSeconds(Math.round(Double.parseDouble(job["total_runtime"] as String)), 0) } : null) - gj.setCpuTime(resourcesUsed["cput"] ? withCaughtAndLoggedException { parseColonSeparatedHHMMSSDuration(job["resources_used"]["cput"] as String) } : null) - gj.setServer(job["server"] as String ?: null) - gj.setUmask(job["umask"] as String ?: null) - gj.setJobState(parseJobState(job["job_state"] as String)) - gj.setExitCode(job["exit_status"] ? withCaughtAndLoggedException { Integer.valueOf(job["exit_status"] as String) }: null ) - gj.setAccount(job["Account_Name"] as String ?: null) - gj.setStartCount(job["start_count"] ? withCaughtAndLoggedException { Integer.valueOf(job["start_count"] as String) } : null) - - if (job["qtime"]) // The time that the job entered the current queue. - gj.setSubmitTime(parseTime(job["qtime"] as String)) - if (job["start_time"]) // The timepoint the job was started. - gj.setStartTime(parseTime(job["start_time"] as String)) - if (job["comp_time"]) // The timepoint the job was completed. - gj.setEndTime(parseTime(job["comp_time"] as String)) - if (job["etime"]) // The time that the job became eligible to run, i.e. in a queued state while residing in an execution queue. - gj.setEligibleTime(parseTime(job["etime"] as String)) - - queriedExtendedStates.put(jobID, gj) - } - return queriedExtendedStates + TimeUnit safelyCastToTimeUnit(Object value) { + String _value = value as String + if (_value) + return withCaughtAndLoggedException { new TimeUnit(_value) } + return null } - private static List getExecutionHosts(String s) { - if (!s) { - return null + static List getExecutionHosts(Object hosts) { + String _hosts = hosts as String + if (!_hosts) { + return [] + } + withCaughtAndLoggedException { + _hosts.split(/\+/) + .collect { String str -> str.split("/") } + .collect { it.first() } + .unique() } - s.split(/\+/) - .collect { String it -> it.split("/") } - .collect { it.first() } - .unique() } - private static List getJobDependencies(String s) { - if (!s) { + static List getJobDependencies(Object deps) { + String _deps = deps as String + if (!_deps) { return [] } - s.split(",") - .find { it.startsWith("afterok") } - ?.findAll(/(\d+)(\.\w+)?/) { fullMatch, String beforeDot, afterDot -> return beforeDot } ?: []as List + withCaughtAndLoggedException { + return _deps.split(",") + .find { it.startsWith("afterok") } + ?.findAll(/(\d+)(\.\w+)?/) { fullMatch, String beforeDot, afterDot -> return beforeDot } ?: [] as List + } } - private File getQstatFile(String s, String jobId) { - if (!s) { + File safelyGetQstatFile(Object s, String jobId) { + String _s = s as String + if (!_s) { return null } - String fileName - if (s.startsWith("/")) { - fileName = s - } else if (s =~ /^[\w-]+:\//) { - fileName = s.replaceAll(/^[\w-]+:/, "") - } else { - return null + withCaughtAndLoggedException { + String fileName + if (_s.startsWith("/")) { + fileName = _s + } else if (_s =~ /^[\w-]+:\//) { + fileName = _s.replaceAll(/^[\w-]+:/, "") + } else { + return null + } + new File(fileName.replace("\$${getJobIdVariable()}", jobId)) } - new File(fileName.replace("\$${getJobIdVariable()}", jobId)) } @Override diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFCommandParser.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFCommandParser.groovy index fbaec626..0f0db4ad 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFCommandParser.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFCommandParser.groovy @@ -3,18 +3,17 @@ package de.dkfz.roddy.execution.jobs.cluster.lsf import de.dkfz.roddy.BEException import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.jobs.BEJobID -import de.dkfz.roddy.execution.jobs.GenericJobInfo +import de.dkfz.roddy.execution.jobs.ExtendedJobInfo import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.ComplexLine import de.dkfz.roddy.tools.TimeUnit import groovy.transform.CompileStatic -import static de.dkfz.roddy.StringConstants.SPLIT_COLON -import static de.dkfz.roddy.StringConstants.SPLIT_COMMA -import static de.dkfz.roddy.StringConstants.SPLIT_EQUALS + +import static de.dkfz.roddy.StringConstants.* /** - * Used to convert commands from cli to e.g. GenericJobInfo + * Used to convert commands from cli to e.g. ExtendedJobInfo * Created by kaercher on 15.05.17. */ @CompileStatic @@ -53,11 +52,11 @@ class LSFCommandParser { if (!commandString.startsWith("bsub")) return // It is obviously not a PBS call - String[] splitted = line.splitBy(" ").findAll { it } + Collection splitted = line.splitBy(" ").findAll { it } script = splitted[-1] jobName = "not readable" - for (int i = 0; i < splitted.length - 1; i++) { + for (int i = 0; i < splitted.size() - 1; i++) { String option = splitted[i] if (!option.startsWith("-")) continue // It is not an option but a parameter or a text (e.g. bsub, script) @@ -123,12 +122,12 @@ class LSFCommandParser { } } - GenericJobInfo toGenericJobInfo() { - GenericJobInfo jInfo = new GenericJobInfo(jobName, new File(script), jobID, parameters, dependencies) + ExtendedJobInfo toGenericJobInfo() { + ExtendedJobInfo jInfo = new ExtendedJobInfo(jobName, script, jobID, parameters, dependencies) ResourceSet askedResources = new ResourceSet(null, memory ? new BufferValue(memory as Integer, bufferUnit) : null, cores ? cores as Integer : null, nodes ? nodes as Integer : null, walltime ? new TimeUnit(walltime) : null, null, null, null) - jInfo.setAskedResources(askedResources) + jInfo.setRequestedResources(askedResources) return jInfo } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy index 1be310af..3a05431c 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManager.groovy @@ -1,7 +1,7 @@ /* - * Copyright (c) 2017 eilslabs. + * Copyright (c) 2019 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). * - * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). + * Distributed under the MIT License (license terms are at https://www.github.com/TheRoddyWMS/BatchEuphoria/LICENSE.txt). */ package de.dkfz.roddy.execution.jobs.cluster.lsf @@ -14,12 +14,12 @@ import de.dkfz.roddy.execution.jobs.* import de.dkfz.roddy.tools.BashUtils import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.BufferValue +import de.dkfz.roddy.tools.DateTimeHelper import groovy.json.JsonSlurper import java.time.Duration import java.time.LocalDateTime import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter /** * Factory for the management of LSF cluster systems. @@ -29,35 +29,81 @@ import java.time.format.DateTimeFormatter @groovy.transform.CompileStatic class LSFJobManager extends AbstractLSFJobManager { - private static final String LSF_COMMAND_QUERY_STATES = "bjobs -a -o -hms -json \"jobid job_name stat user queue " + + private static final String LSF_COMMAND_QUERY_STATES = "bjobs -a -o -hms -json \"jobid job_name stat finish_time\"" + private static final String LSF_COMMAND_QUERY_EXTENDED_STATES = "bjobs -a -o -hms -json \"jobid job_name stat user queue " + "job_description proj_name job_group job_priority pids exit_code from_host exec_host submit_time start_time " + - "finish_time cpu_used run_time user_group swap max_mem runtimelimit sub_cwd " + + "finish_time cpu_used run_time user_group swap max_mem max_req_proc" + + "memlimit runtimelimit swaplimit sub_cwd " + "pend_reason exec_cwd output_file input_file effective_resreq exec_home slots error_file command dependency \"" private static final String LSF_COMMAND_DELETE_JOBS = "bkill" - private final DateTimeFormatter DATE_PATTERN + public static final String longReportedDateFormatWithoutStatus = "MMM ppd HH:mm yyyy" + static final DateTimeHelper dateTimeHelper = new DateTimeHelper(longReportedDateFormatWithoutStatus, Locale.ENGLISH) LSFJobManager(BEExecutionService executionService, JobManagerOptions parms) { super(executionService, parms) - DATE_PATTERN = DateTimeFormatter - .ofPattern("MMM ppd HH:mm yyyy") - .withLocale(Locale.ENGLISH) - .withZone(parms.timeZoneId) } - private String getQueryCommand() { - return LSF_COMMAND_QUERY_STATES + /** + * Time formats in LSF can be configured. + * E.g. to our knowledge, by default, reported values look like: + * "Jan 1 10:21 L" with status information or "Jan 1 10:21" without them. + * LSF can be configured to also report the year, so values would be reported as: + * "Jan 1 10:21 2010 L" with status information or "Jan 1 10:21 2010" without them. + * + * There might be different configurations, but we stick to those 4 versions. + * + * Furthermore, we do not know, if LSF will report in other languages than english. + * We assume, that english is used for month names. + * + * We also assume, that the method will not be misused. If its misused, it will throw + * an exception. + */ + @Override + ZonedDateTime parseTime(String str) { + // Prevent NullPointerException, will throw a DateTimeParserException later + if (str == null) str = "" + String dateForParser = str + if (str.size() == "Jan 01 01:00".size()) { + // Lets start and see, if the date is reported in its short version, if so, add the year. + dateForParser = "${str} ${LocalDateTime.now().year}" + } else if (str.size() == "Jan 01 01:00 L".size()) { + // Here we need to strip away the status first, then append the current year. + dateForParser = "${stripAwayStatusInfo(str)} ${LocalDateTime.now().year}" + } else if (str.size() == "Jan 01 01:00 1000".size()) { + // Easy enough, just keep it like it is. + dateForParser = str + } else if (str.size() == "Jan 01 01:00 1000 L".size()) { + // Again, strip away the status info. + dateForParser = stripAwayStatusInfo(str) + } + + // Finally we try to parse the date. Lets see if it works. If not, an exception is thrown. + ZonedDateTime date = dateTimeHelper.parseToZonedDateTime(dateForParser) + + // If LSF is not configured to report the date, the (kind of reasonable) assumption made here is that if + // the job's submission time (assuming the current year) is later than the current time, then the job was + // submitted last year. + + if (date > ZonedDateTime.now()) { + return date.minusYears(1) + } + return date } - @Override - Map queryExtendedJobStateById(List jobIds) { - Map queriedExtendedStates = [:] - for (BEJobID id : jobIds) { - Map jobDetails = runBjobs([id]).get(id) - queriedExtendedStates.put(id, queryJobInfo(jobDetails)) + /** + * Important here is, that LSF puts " L" or other status codes at the end of some dates, e.g. FINISH_DATE + * Thus said, " L" does not apply for all dates reported by LSF! This method just removes the last two characters + * of the time string. + */ + static String stripAwayStatusInfo(String time) { + String result = time + if (time && time.size() > 2) { + if (time[-2..-1] ==~ ~/[ ][a-zA-Z0-9]/) + result = time[0..-3] } - return queriedExtendedStates + return result } @Override @@ -83,20 +129,36 @@ class LSFJobManager extends AbstractLSFJobManager { } @Override - GenericJobInfo parseGenericJobInfo(String commandString) { + ExtendedJobInfo parseGenericJobInfo(String commandString) { return new LSFCommandParser(commandString).toGenericJobInfo(); } - protected Map queryJobStates(List jobIDs) { - runBjobs(jobIDs).collectEntries { BEJobID jobID, Object value -> + @Override + Map queryJobInfo(List jobIDs) { + def bjobs = runBjobs(jobIDs, false) + bjobs = bjobs.findAll { BEJobID k, Map v -> v } + + return bjobs.collectEntries { BEJobID jobID, Object value -> JobState js = parseJobState(value["STAT"] as String) - [(jobID): js] - } as Map + JobInfo jobInfo = new JobInfo(jobID) + jobInfo.jobState = js + [(jobID): jobInfo] + } as Map } - private Map> runBjobs(List jobIDs) { - StringBuilder queryCommand = new StringBuilder(getQueryCommand()) + @Override + Map queryExtendedJobInfo(List jobIds) { + Map queriedExtendedStates = [:] + for (BEJobID id : jobIds) { + Map jobDetails = runBjobs([id], true)[id] + queriedExtendedStates.put(id, convertJobDetailsMapToGenericJobInfoObject(jobDetails)) + } + return queriedExtendedStates + } + + Map> runBjobs(List jobIDs, boolean extended) { + StringBuilder queryCommand = new StringBuilder(extended ? LSF_COMMAND_QUERY_EXTENDED_STATES : LSF_COMMAND_QUERY_STATES) // user argument must be passed before the job IDs if (isTrackingOfUserJobsEnabled) @@ -109,117 +171,104 @@ class LSFJobManager extends AbstractLSFJobManager { ExecutionResult er = executionService.execute(queryCommand.toString()) List resultLines = er.resultLines - Map> result = [:] - - if (er.successful) { - if (resultLines.size() >= 1) { - String rawJson = resultLines.join("\n") - Object parsedJson = new JsonSlurper().parseText(rawJson) - List records = (List) parsedJson.getAt("RECORDS") - records.each { - BEJobID jobID = new BEJobID(it["JOBID"] as String) - result.put(jobID, it as Map) - } - } - - } else { + if (!er.successful) { String error = "Job status couldn't be updated. \n command: ${queryCommand} \n status code: ${er.exitCode} \n result: ${er.resultLines}" throw new BEException(error) } - return result + + return convertBJobsJsonOutputToResultMap(resultLines.join("\n")) } - /** parseTime() parses a zoned time as provided by LSF. Unfortunately, LFS does not return the submission year! The (kind of reasonable) - * assumption made here is that if the job's submission time (assuming the current year) is later than the current time, then the job was - * submitted last year. - * @param str - * @return - */ - ZonedDateTime parseTime(String str) { - ZonedDateTime date = ZonedDateTime.parse("${str} ${LocalDateTime.now().getYear()}", DATE_PATTERN) - if (date > ZonedDateTime.now()) { - return date.minusYears(1) + static Map> convertBJobsJsonOutputToResultMap(String rawJson) { + Map> result = [:] + + if (!rawJson) + return result + + Object parsedJson = new JsonSlurper().parseText(rawJson) + List records = (List) parsedJson["RECORDS"] + for (record in records) { + BEJobID jobID = new BEJobID(record["JOBID"] as String) + result[jobID] = record as Map } - return date + + result } /** * Used by @getJobDetails to set JobInfo */ - private GenericJobInfo queryJobInfo(Map jobResult) { - - GenericJobInfo jobInfo - BEJobID jobID - try{ - jobID = new BEJobID(jobResult["JOBID"] as String) - }catch (Exception exp){ - throw new BEException("Job ID '${jobResult["JOBID"]}' could not be transformed to BEJobID ") + ExtendedJobInfo convertJobDetailsMapToGenericJobInfoObject(Map _rawExtendedStates) { + // Remove empty entries first to keep the output clean (use null, where the value is null or empty.) + Map extendedStates = _rawExtendedStates.findAll { String k, String v -> v } + + BEJobID jobID = toJobID(extendedStates["JOBID"]) + + List dependIDs = extendedStates["DEPENDENCY"]?.tokenize(/&/)?.collect { it.find(/\d+/) } + ExtendedJobInfo jobInfo = new ExtendedJobInfo(extendedStates["JOB_NAME"], extendedStates["COMMAND"], jobID, null, dependIDs) + + /** Common */ + jobInfo.user = extendedStates["USER"] + jobInfo.userGroup = extendedStates["USER_GROUP"] + jobInfo.description = extendedStates["JOB_DESCRIPTION"] + jobInfo.projectName = extendedStates["PROJ_NAME"] + jobInfo.jobGroup = extendedStates["JOB_GROUP"] + jobInfo.priority = extendedStates["JOB_PRIORITY"] + jobInfo.processesInJob = extendedStates["PIDS"]?.split(",")?.toList() + jobInfo.submissionHost = extendedStates["FROM_HOST"] + + /** Resources */ + jobInfo.executionHosts = extendedStates["EXEC_HOST"]?.split(":")?.toList() + // Count hosts! The node count has no custom entry. However, we can calculate it from the host list. + Integer noOfExecutionHosts = jobInfo.executionHosts?.sort()?.unique()?.size() + String queue = extendedStates["QUEUE"] + Duration requestedWalltime = safelyParseColonSeparatedDuration(extendedStates["RUNTIMELIMIT"]) + Duration usedWalltime = safelyParseColonSeparatedDuration(extendedStates["RUN_TIME"]) + BufferValue usedMemory = safelyCastToBufferValue(extendedStates["MAX_MEM"]) + BufferValue requestedMemory = safelyCastToBufferValue(extendedStates["MEMLIMIT"]) + Integer requestedCores = extendedStates["MAX_REQ_PROC"] as Integer + BufferValue swap = withCaughtAndLoggedException { + String SWAP = extendedStates["SWAP"] + SWAP ? new BufferValue(SWAP.find("\\d+"), BufferUnit.m) : null } - List dependIDs = ((String) jobResult["DEPENDENCY"])? ((String) jobResult["DEPENDENCY"]).tokenize(/&/).collect { it.find(/\d+/) } : null - jobInfo = new GenericJobInfo(jobResult["JOB_NAME"] as String ?: null, jobResult["COMMAND"] as String ? new File(jobResult["COMMAND"] as String): null, jobID, null, dependIDs) + jobInfo.usedResources = new ResourceSet(usedMemory, null, noOfExecutionHosts, usedWalltime, null, queue, null) + jobInfo.requestedResources = new ResourceSet(requestedMemory, requestedCores, null, requestedWalltime, null, queue, null) + jobInfo.rawResourceRequest = extendedStates["EFFECTIVE_RESREQ"] + jobInfo.runTime = usedWalltime + jobInfo.cpuTime = safelyParseColonSeparatedDuration(extendedStates["CPU_USED"]) + + /** Status info */ + jobInfo.jobState = parseJobState(extendedStates["STAT"]) + jobInfo.exitCode = jobInfo.jobState == JobState.COMPLETED_SUCCESSFUL ? 0 : (extendedStates["EXIT_CODE"] as Integer) + jobInfo.pendReason = extendedStates["PEND_REASON"] + + /** Directories and files */ + jobInfo.cwd = extendedStates["SUB_CWD"] + jobInfo.execCwd = extendedStates["EXEC_CWD"] + jobInfo.logFile = getBjobsFile(extendedStates["OUTPUT_FILE"], jobID, "out") + jobInfo.errorLogFile = getBjobsFile(extendedStates["ERROR_FILE"], jobID, "err") + jobInfo.inputFile = extendedStates["INPUT_FILE"] ? new File(extendedStates["INPUT_FILE"]) : (File)null + jobInfo.execHome = extendedStates["EXEC_HOME"] + + /** Timestamps */ + jobInfo.submitTime = safelyParseTime(extendedStates["SUBMIT_TIME"]) + jobInfo.startTime = safelyParseTime(extendedStates["START_TIME"]) + jobInfo.endTime = safelyParseTime(extendedStates["FINISH_TIME"]) - String queue = jobResult["QUEUE"] ?: null - Duration runTime = withCaughtAndLoggedException { - jobResult["RUN_TIME"] ? parseColonSeparatedHHMMSSDuration(jobResult["RUN_TIME"] as String) : null - } - BufferValue swap = withCaughtAndLoggedException { - jobResult["SWAP"] ? new BufferValue((jobResult["SWAP"] as String).find("\\d+"), BufferUnit.m) : null - } - BufferValue memory = withCaughtAndLoggedException { - String unit = (jobResult["MAX_MEM"] as String).find("[a-zA-Z]+") - BufferUnit bufferUnit - if (unit == "Gbytes") - bufferUnit = BufferUnit.g - else - bufferUnit = BufferUnit.m - jobResult["MAX_MEM"] ? new BufferValue((jobResult["MAX_MEM"] as String).find("([0-9]*[.])?[0-9]+"), bufferUnit) : null - } - Duration runLimit = withCaughtAndLoggedException { - jobResult["RUNTIMELIMIT"] ? parseColonSeparatedHHMMSSDuration(jobResult["RUNTIMELIMIT"] as String) : null - } - Integer nodes = withCaughtAndLoggedException { jobResult["SLOTS"] ? jobResult["SLOTS"] as Integer : null } - - ResourceSet usedResources = new ResourceSet(memory, null, nodes, runTime, null, queue, null) - jobInfo.setUsedResources(usedResources) - - ResourceSet askedResources = new ResourceSet(null, null, null, runLimit, null, queue, null) - jobInfo.setAskedResources(askedResources) - - jobInfo.setUser(jobResult["USER"] as String ?: null) - jobInfo.setDescription(jobResult["JOB_DESCRIPTION"] as String ?: null) - jobInfo.setProjectName(jobResult["PROJ_NAME"] as String ?: null) - jobInfo.setJobGroup(jobResult["JOB_GROUP"] as String ?: null) - jobInfo.setPriority(jobResult["JOB_PRIORITY"] as String ?: null) - jobInfo.setPidStr(jobResult["PIDS"] as String ? (jobResult["PIDS"] as String).split(",").toList() : null) - jobInfo.setJobState(parseJobState(jobResult["STAT"] as String)) - jobInfo.setExitCode(jobInfo.jobState == JobState.COMPLETED_SUCCESSFUL ? 0 : (jobResult["EXIT_CODE"] ? Integer.valueOf(jobResult["EXIT_CODE"] as String) : null)) - jobInfo.setSubmissionHost(jobResult["FROM_HOST"] as String ?: null) - jobInfo.setExecutionHosts(jobResult["EXEC_HOST"] as String ? (jobResult["EXEC_HOST"] as String).split(":").toList() : null) + return jobInfo + } + + BufferValue safelyCastToBufferValue(String MAX_MEM) { withCaughtAndLoggedException { - jobInfo.setCpuTime(jobResult["CPU_USED"] ? parseColonSeparatedHHMMSSDuration(jobResult["CPU_USED"] as String) : null) - } - jobInfo.setRunTime(runTime) - jobInfo.setUserGroup(jobResult["USER_GROUP"] as String ?: null) - jobInfo.setCwd(jobResult["SUB_CWD"] as String ?: null) - jobInfo.setPendReason(jobResult["PEND_REASON"] as String ?: null) - jobInfo.setExecCwd(jobResult["EXEC_CWD"] as String ?: null) - jobInfo.setLogFile(getBjobsFile(jobResult["OUTPUT_FILE"] as String, jobID, "out")) - jobInfo.setErrorLogFile(getBjobsFile(jobResult["ERROR_FILE"] as String, jobID, "err")) - jobInfo.setInputFile(jobResult["INPUT_FILE"] ? new File(jobResult["INPUT_FILE"] as String) : null) - jobInfo.setResourceReq(jobResult["EFFECTIVE_RESREQ"] as String ?: null) - jobInfo.setExecHome(jobResult["EXEC_HOME"] as String ?: null) - - if (jobResult["SUBMIT_TIME"]) - withCaughtAndLoggedException { jobInfo.setSubmitTime(parseTime(jobResult["SUBMIT_TIME"] as String)) } - if (jobResult["START_TIME"]) - withCaughtAndLoggedException { jobInfo.setStartTime(parseTime(jobResult["START_TIME"] as String)) } - if (jobResult["FINISH_TIME"]) - withCaughtAndLoggedException { - jobInfo.setEndTime(parseTime((jobResult["FINISH_TIME"] as String).substring(0, (jobResult["FINISH_TIME"] as String).length() - 2))) + if (MAX_MEM) { + String bufferSize = MAX_MEM.find("([0-9]*[.])?[0-9]+") + String unit = MAX_MEM.find("[a-zA-Z]+") + BufferUnit bufferUnit = unit == "Gbytes" ? BufferUnit.g : BufferUnit.m + return new BufferValue(bufferSize, bufferUnit) } - - return jobInfo + return null + } } private File getBjobsFile(String s, BEJobID jobID, String type) { @@ -260,12 +309,12 @@ class LSFJobManager extends AbstractLSFJobManager { } @Override - String getQueryJobStatesCommand() { + String getQueryCommandForJobInfo() { return null } @Override - String getExtendedQueryJobStatesCommand() { + String getQueryCommandForExtendedJobInfo() { return null } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy index e8635fe8..865436d6 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy @@ -134,7 +134,7 @@ class LSFRestJobManager extends AbstractLSFJobManager { @Override - GenericJobInfo parseGenericJobInfo(String command) { + ExtendedJobInfo parseGenericJobInfo(String command) { return null } @@ -271,9 +271,9 @@ class LSFRestJobManager extends AbstractLSFJobManager { * Updates job information for given jobs * @param jobList */ - private Map getJobDetails(List jobList) { + private Map getJobDetails(List jobList) { List
headers = [] - Map jobDetailsResult = [:] + Map jobDetailsResult = [:] headers.add(new BasicHeader("Accept", "text/xml,application/xml;")) RestResult result = restExecutionService.execute(new RestCommand(URI_JOB_DETAILS + prepareURLWithParam(jobList), null, headers, RestCommand.HttpMethod.HTTPGET)) as RestResult @@ -296,14 +296,14 @@ class LSFRestJobManager extends AbstractLSFJobManager { * @param list of job ids */ @Override - Map queryJobStates(List jobIds) { + Map queryJobInfo(List jobIds) { List
headers = [] headers.add(new BasicHeader("Accept", "text/xml,application/xml;")) RestResult result = restExecutionService.execute(new RestCommand(URI_JOB_BASICS, null, headers, RestCommand.HttpMethod.HTTPGET)) as RestResult if (result.isSuccessful()) { GPathResult res = new XmlSlurper().parseText(result.body) - Map resultStates = [:] + Map resultStates = [:] res.getProperty("pseudoJob").each { NodeChild element -> String jobId = null if (jobIds) { @@ -315,7 +315,11 @@ class LSFRestJobManager extends AbstractLSFJobManager { } if (jobId) { - resultStates.put(new BEJobID(jobId), parseJobState(element.getProperty("jobStatus").toString())) + def jobID = new BEJobID(jobId) + def jobState = parseJobState(element.getProperty("jobStatus").toString()) + JobInfo jobInfo = new JobInfo(jobID) + jobInfo.jobState = jobState + resultStates[jobID] = jobInfo } } return resultStates @@ -324,14 +328,18 @@ class LSFRestJobManager extends AbstractLSFJobManager { } } - /** + @Override + Map queryExtendedJobInfo(List jobIDs) { + return null + } +/** * Used by @getJobDetails to set JobInfo * @param job * @param jobDetails - XML job details */ - private GenericJobInfo setJobInfoForJobDetails(NodeChild jobDetails) { + private ExtendedJobInfo setJobInfoForJobDetails(NodeChild jobDetails) { - GenericJobInfo jobInfo = new GenericJobInfo(jobDetails.getProperty("jobName").toString(), new File(jobDetails.getProperty("command").toString()), new BEJobID(jobDetails.getProperty("jobId").toString()), null, null) + ExtendedJobInfo jobInfo = new ExtendedJobInfo(jobDetails.getProperty("jobName").toString(), jobDetails.getProperty("command").toString(), new BEJobID(jobDetails.getProperty("jobId").toString()), null, null) String queue = jobDetails.getProperty("queue").toString() BufferValue swap = jobDetails.getProperty("swap") ? withCaughtAndLoggedException { new BufferValue(jobDetails.getProperty("swap").toString(), BufferUnit.m) } : null @@ -364,23 +372,23 @@ class LSFRestJobManager extends AbstractLSFJobManager { jobInfo.setRunTime(jobDetails.getProperty("runTime") ? withCaughtAndLoggedException { Duration.ofSeconds(Math.round(Double.parseDouble(jobDetails.getProperty("runTime").toString()))) } : null) jobInfo.setProjectName(jobDetails.getProperty("projectName").toString()) jobInfo.setExitCode(jobDetails.getProperty("exitStatus").toString() ? withCaughtAndLoggedException { Integer.valueOf(jobDetails.getProperty("exitStatus").toString()) } : null) - jobInfo.setPidStr(jobDetails.getProperty("pidStr") as String ? (jobDetails.getProperty("pidStr") as String).split(",").toList() : null) - jobInfo.setPgidStr(jobDetails.getProperty("pgidStr").toString()) + jobInfo.setProcessesInJob(jobDetails.getProperty("pidStr") as String ? (jobDetails.getProperty("pidStr") as String).split(",").toList() : null) + jobInfo.setProcessGroupID(jobDetails.getProperty("pgidStr").toString()) jobInfo.setCwd(jobDetails.getProperty("cwd").toString()) jobInfo.setPendReason(jobDetails.getProperty("pendReason").toString()) jobInfo.setExecCwd(jobDetails.getProperty("execCwd").toString()) jobInfo.setPriority(jobDetails.getProperty("priority").toString()) jobInfo.setLogFile(new File(jobDetails.getProperty("outFile").toString())) jobInfo.setInputFile(new File(jobDetails.getProperty("inFile").toString())) - jobInfo.setResourceReq(jobDetails.getProperty("resReq").toString()) + jobInfo.setRawResourceRequest(jobDetails.getProperty("resReq").toString()) jobInfo.setExecHome(jobDetails.getProperty("execHome").toString()) jobInfo.setExecUserName(jobDetails.getProperty("execUserName").toString()) - jobInfo.setAskedHostsStr(jobDetails.getProperty("askedHostsStr").toString()) + jobInfo.setRawHostQueryString(jobDetails.getProperty("askedHostsStr").toString()) return jobInfo } - private ZonedDateTime parseTime(String str) { + ZonedDateTime parseTime(String str) { if (!str) { return null } @@ -391,7 +399,7 @@ class LSFRestJobManager extends AbstractLSFJobManager { * Get the time history for each given job * @param jobList */ - private void updateJobStatistics(Map jobList) { + private void updateJobStatistics(Map jobList) { List
headers = [] headers.add(new BasicHeader("Accept", "text/xml,application/xml;")) @@ -415,7 +423,7 @@ class LSFRestJobManager extends AbstractLSFJobManager { * @param job * @param jobHistory - xml job history */ - private void setJobInfoFromJobHistory(GenericJobInfo jobInfo, NodeChild jobHistory) { + private void setJobInfoFromJobHistory(ExtendedJobInfo jobInfo, NodeChild jobHistory) { GPathResult timeSummary = jobHistory.getProperty("timeSummary") as GPathResult DateTimeFormatter lsfDatePattern = DateTimeFormatter.ofPattern("EEE MMM ppd HH:mm:ss yyyy").withLocale(Locale.ENGLISH) @@ -430,8 +438,8 @@ class LSFRestJobManager extends AbstractLSFJobManager { } @Override - Map queryExtendedJobStateById(List jobIds) { - Map jobDetailsResult = getJobDetails(jobIds) + Map queryExtendedJobStateById(List jobIds) { + Map jobDetailsResult = getJobDetails(jobIds) updateJobStatistics(jobDetailsResult) return jobDetailsResult } @@ -442,12 +450,12 @@ class LSFRestJobManager extends AbstractLSFJobManager { } @Override - String getQueryJobStatesCommand() { + String getQueryCommandForJobInfo() { return null } @Override - String getExtendedQueryJobStatesCommand() { + String getQueryCommandForExtendedJobInfo() { return null } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParser.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParser.groovy index f0119075..48e2ae9a 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParser.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParser.groovy @@ -9,19 +9,17 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs import de.dkfz.roddy.BEException import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.jobs.BEJobID -import de.dkfz.roddy.execution.jobs.GenericJobInfo +import de.dkfz.roddy.execution.jobs.ExtendedJobInfo import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.ComplexLine import de.dkfz.roddy.tools.TimeUnit import groovy.transform.CompileStatic -import static de.dkfz.roddy.StringConstants.SPLIT_COLON -import static de.dkfz.roddy.StringConstants.SPLIT_COMMA -import static de.dkfz.roddy.StringConstants.SPLIT_EQUALS +import static de.dkfz.roddy.StringConstants.* /** - * Used to convert commands from cli to e.g. GenericJobInfo + * Used to convert commands from cli to e.g. ExtendedJobInfo * Created by heinold on 04.04.17. */ @CompileStatic @@ -60,11 +58,11 @@ class PBSCommandParser { if (!commandString.startsWith("qsub")) return // It is obviously not a PBS call - String[] splitted = line.splitBy(" ").findAll { it } + Collection splitted = line.splitBy(" ").findAll { it } script = splitted[-1] jobName = "not readable" - for (int i = 0; i < splitted.length - 1; i++) { + for (int i = 0; i < splitted.size() - 1; i++) { String option = splitted[i] if (!option.startsWith("-")) continue // It is not an option but a parameter or a text (e.g. qsub, script) @@ -130,12 +128,12 @@ class PBSCommandParser { } } - GenericJobInfo toGenericJobInfo() { - GenericJobInfo jInfo = new GenericJobInfo(jobName, new File(script), jobID, parameters, dependencies) + ExtendedJobInfo toGenericJobInfo() { + ExtendedJobInfo jInfo = new ExtendedJobInfo(jobName, script, jobID, parameters, dependencies) ResourceSet askedResources = new ResourceSet(null, memory ? new BufferValue(memory as Integer, bufferUnit) : null, cores ? cores as Integer : null, nodes ? nodes as Integer : null, walltime ? new TimeUnit(walltime) : null, null, null, null) - jInfo.setAskedResources(askedResources) + jInfo.setRequestedResources(askedResources) return jInfo } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy index 5699decc..c9e9898d 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy @@ -7,16 +7,18 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs import com.google.common.collect.LinkedHashMultimap +import de.dkfz.roddy.BEException import de.dkfz.roddy.StringConstants import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService -import de.dkfz.roddy.execution.jobs.BEJob -import de.dkfz.roddy.execution.jobs.GenericJobInfo -import de.dkfz.roddy.execution.jobs.JobManagerOptions -import de.dkfz.roddy.execution.jobs.JobState +import de.dkfz.roddy.execution.io.ExecutionResult +import de.dkfz.roddy.execution.jobs.* import de.dkfz.roddy.execution.jobs.cluster.GridEngineBasedJobManager import de.dkfz.roddy.tools.BufferUnit +import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.TimeUnit +import groovy.transform.PackageScope +import groovy.util.slurpersupport.GPathResult /** * @author michael @@ -34,8 +36,8 @@ class PBSJobManager extends GridEngineBasedJobManager { } @Override - GenericJobInfo parseGenericJobInfo(String commandString) { - return new PBSCommandParser(commandString).toGenericJobInfo(); + ExtendedJobInfo parseGenericJobInfo(String commandString) { + return new PBSCommandParser(commandString).toGenericJobInfo() } /** @@ -79,12 +81,12 @@ class PBSJobManager extends GridEngineBasedJobManager { } @Override - String getQueryJobStatesCommand() { + String getQueryCommandForJobInfo() { return "qstat -t" } @Override - String getExtendedQueryJobStatesCommand() { + String getQueryCommandForExtendedJobInfo() { return "qstat -x -f" } @@ -157,4 +159,96 @@ class PBSJobManager extends GridEngineBasedJobManager { return "PBS_O_WORKDIR" } + + @Override + Map queryExtendedJobInfo(List jobIds) { + Map queriedExtendedStates = [:] + String qStatCommand = getQueryCommandForExtendedJobInfo() + qStatCommand += " " + jobIds.collect { it }.join(" ") + // For the reviewer: This does not make sense right? We have a list of given ids which we want to + // query anyway. + // if (isTrackingOfUserJobsEnabled) + // qStatCommand += " -u $userIDForQueries " + + ExecutionResult er = executionService.execute(qStatCommand.toString()) + if (er != null && er.successful) { + GPathResult parsedJobs = new XmlSlurper().parseText(er.resultLines.join("\n")) + for (BEJobID jobID : jobIds) { + def jobInfo = parsedJobs.children().find { it["Job_Id"].toString().startsWith("${jobID.id}.") } + queriedExtendedStates[jobID] = this.processQstatOutputFromXMLNodeChild(jobInfo) + } + } else { + throw new BEException("Extended job states couldn't be retrieved. \n Returned status code:${er.exitCode} \n ${qStatCommand.toString()} \n\t result:${er.resultLines.join("\n\t")}") + } + return queriedExtendedStates + } + + /** + * Reads the qstat output and creates ExtendedJobInfo objects + * @param resultLines - Input of ExecutionResult object + * @return map with jobid as key + */ + @PackageScope + ExtendedJobInfo processQstatOutputFromXMLNodeChild(GPathResult job) { + String jobIdRaw = job["Job_Id"] as String + BEJobID jobID = toJobID(jobIdRaw) + + List jobDependencies = getJobDependencies(job["depend"]) + ExtendedJobInfo jobInfo = new ExtendedJobInfo(job["Job_Name"] as String, null, jobID, null, jobDependencies) + + /** Common */ + jobInfo.user = (job["Job_Owner"] as String)?.split(StringConstants.SPLIT_AT)[0] + jobInfo.userGroup = job["egroup"] + jobInfo.priority = job["Priority"] + jobInfo.submissionHost = job["submit_host"] + jobInfo.server = job["server"] + jobInfo.umask = job["umask"] + jobInfo.account = job["Account_Name"] + jobInfo.startCount = safelyCastToInteger(job["start_count"]) + + /** Resources + * Note: Both nodes and nodect are deprecated values + * nodect is the number of requested nodes and of type integer + */ + jobInfo.executionHosts = getExecutionHosts(job["exec_host"]) + String queue = job["queue"] as String + + def requestedResources = job["Resource_List"] + String resourcesListNodes = requestedResources["nodes"] + Integer requestedNodes = safelyCastToInteger(requestedResources["nodect"]) + Integer requestedCores = safelyCastToInteger(resourcesListNodes?.find("ppn=.*")?.find(/(\d+)/)) + String requestedAdditionalNodeFlag = withCaughtAndLoggedException { resourcesListNodes?.find(/(\d+):(\.*)/) { fullMatch, nCores, feature -> return feature } } + TimeUnit requestedWalltime = safelyCastToTimeUnit(requestedResources["walltime"]) + BufferValue requestedMemory = safelyCastToBufferValue(requestedResources["mem"]) + + def usedResources = job["resources_used"] + TimeUnit usedWalltime = safelyCastToTimeUnit(usedResources["walltime"]) + BufferValue usedMemory = safelyCastToBufferValue(usedResources["mem"]) + + jobInfo.requestedResources = new ResourceSet(null, requestedMemory, requestedCores, requestedNodes, requestedWalltime, null, queue, requestedAdditionalNodeFlag) + jobInfo.usedResources = new ResourceSet(null, usedMemory, null, null, usedWalltime, null, queue, null) + jobInfo.rawResourceRequest = job["submit_args"] + jobInfo.runTime = safelyCastToDuration(job["total_runtime"]) + jobInfo.cpuTime = safelyParseColonSeparatedDuration(usedResources["cput"]) + + /** Status info */ + jobInfo.jobState = parseJobState(job["job_state"] as String) + jobInfo.exitCode = safelyCastToInteger(job["exit_status"]) + + /** Directories and files */ + jobInfo.logFile = safelyGetQstatFile(job["Output_Path"] as String, jobIdRaw) + jobInfo.errorLogFile = safelyGetQstatFile(job["Error_Path"] as String, jobIdRaw) + + /** Timestamps */ + jobInfo.submitTime = safelyParseTime(job["qtime"]) // The time that the job entered the current queue. + jobInfo.startTime = safelyParseTime(job["start_time"]) // The timepoint the job was started. + jobInfo.endTime = safelyParseTime(job["comp_time"]) // The timepoint the job was completed. + jobInfo.eligibleTime = safelyParseTime(job["etime"]) // The time that the job became eligible to run, i.e. in a queued state while residing in an execution queue. + // http://docs.adaptivecomputing.com/torque/4-2-7/Content/topics/9-accounting/accountingRecords.htm + // Left is ctime: The the time the job was created + + return jobInfo + } + + } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManager.groovy index 613eb0ad..d6c9c58a 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManager.groovy @@ -7,22 +7,19 @@ package de.dkfz.roddy.execution.jobs.cluster.sge import com.google.common.collect.LinkedHashMultimap -import de.dkfz.roddy.BEException import de.dkfz.roddy.StringConstants import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.execution.io.ExecutionResult import de.dkfz.roddy.execution.jobs.* -import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager import de.dkfz.roddy.execution.jobs.cluster.GridEngineBasedJobManager -import de.dkfz.roddy.tools.* +import de.dkfz.roddy.tools.BufferUnit +import de.dkfz.roddy.tools.BufferValue +import de.dkfz.roddy.tools.TimeUnit import groovy.util.slurpersupport.GPathResult import java.time.Duration -import java.time.Instant -import java.time.LocalDateTime -import java.time.ZoneId -import java.util.regex.Matcher +import java.time.ZonedDateTime /** * @author michael @@ -40,17 +37,17 @@ class SGEJobManager extends GridEngineBasedJobManager { } @Override - String getQueryJobStatesCommand() { - return "qstat -g d -j" + String getQueryCommandForJobInfo() { + return "qstat" } @Override - String getExtendedQueryJobStatesCommand() { + String getQueryCommandForExtendedJobInfo() { return "qstat -xml -ext -f -j" } @Override - GenericJobInfo parseGenericJobInfo(String command) { + ExtendedJobInfo parseGenericJobInfo(String command) { return null } @@ -134,4 +131,157 @@ class SGEJobManager extends GridEngineBasedJobManager { return null } + @Override + String assembleJobInfoQueryCommand(List jobIDs) { + + // SGE will output quite a bunch of information if you query by job. Thus said, just querying all jobs, + // maybe filtered by the user, will possible result in less lines per query. E.g. a query with around + // 8000 active jobs will result in approximately 830kByte, whereas a query with qstat -j ID will + // result in 30 lines per job multiplied by (guessed) 40Byte per line = 1,2kByte per Job. We could overcome this + // by putting in a filter with grep, but then again, the state is displayed differently than in the tabluar + // qstat (e.g. scheduling info: Job is in hold state), which makes it a little more difficult to use + // => We'll stick to a simple "qstat (-u)" for now. + String command = getQueryCommandForJobInfo() + + if (isTrackingOfUserJobsEnabled) + command += " -u $userIDForQueries " + + return command + } + + Map convertJobInfoMapToExtendedMap(Map jobInfoMap) { + return jobInfoMap.collectEntries { + BEJobID jobID, JobInfo jobInfo -> + [jobID, new ExtendedJobInfo(jobID, jobInfo.jobState)] + } as Map + } + + @Override + Map queryExtendedJobInfo(List jobIDs) { + // Unfortunately, SGEs qstat does not show some information in extended mode, e.g.: + // - The start time + // - The job state (ok it sometimes displays something. But its some sort of message and it is not directly + // attached to the job AND e.g. running is not shown at all + // - Finish time + // - Execution host + // To overcome at least the missing job state, we call queryJobInfo() for this information first. + + Map shortInfo = jobIDs == null ? queryAllJobInfo() : queryJobInfoByID(jobIDs) + + // If no jobs were found or all are of state unknown, we can actually just omit the rest. + // This will make things a lot easier because: + // - If there is NO job info in the result xml of the extended query, the result will be different and return + // invalid! (for my SGE tests, see the test files) xml code! + // - If there is at least one job found, you will not see info about unknown jobs! + // - So either preprocess the xml and correct it or check first, if we have info at + // all and omit the query then. + if (shortInfo.size() == 0 || shortInfo.values().every { it.jobState == JobState.UNKNOWN }) + return convertJobInfoMapToExtendedMap(shortInfo) + + // Now we at least have some states, regardless of their existence in our list of job ids + // Lets retrieve and parse our extended states. + ExecutionResult er = executionService.execute(getQueryCommandForExtendedJobInfo()) + + if (!er.successful) + return convertJobInfoMapToExtendedMap(shortInfo) + + Map result = [:] + + // Finally we can parse our hopefully correct xml info + def xml = new XmlSlurper().parseText(er.resultLines.join("\n")) + + xml["djob_info"]["element"].each { + GPathResult element -> + ExtendedJobInfo jobInfo = parseGPathJobInfoElement(element) + jobInfo.jobState = shortInfo[jobInfo.jobID]?.jobState ?: JobState.UNKNOWN + result[jobInfo.jobID] = jobInfo + } + + // If a result is missing, it will be filled in later by the methods in BatchEuphoriaJobManager. + return result + } + + /** + * The job state will NOT be parsed in this method! It is solely to be used as a helper for queryExtendedJobInfo + */ + ExtendedJobInfo parseGPathJobInfoElement(GPathResult job) { + ExtendedJobInfo jobInfo = new ExtendedJobInfo(new BEJobID(job["JB_job_number"] as String)) + + /** Common */ + jobInfo.user = job["JB_owner"] + jobInfo.userGroup = job["JB_group"] + jobInfo.priority = job["JB_priority"] + // jobInfo.submissionHost = job["JB_submit_host"] // Info is not available + // jobInfo.server = job["JB_server"] // Info is not available + // jobInfo.umask = job["JB_umask"] // Info is not available + jobInfo.account = job["JB_account"] + jobInfo.startCount = safelyCastToInteger(job["JB_restart"]) + + /** Resources */ + // jobInfo.executionHosts = getExecutionHosts(job["JB_exec_host"]) // Info is not available + // jobInfo.queue // Info is not available + + // Getting cores and nodes is a bit tricks. Nodes are not in the xml, cores via JB_pe_range + String requestedCoresRaw = job["JB_pe_range"]["ranges"]["RN_min"]?.toString() + Integer requestedCores = requestedCoresRaw ? requestedCoresRaw as Integer : null + + def requestedResourcesRaw = ((job["JB_hard_resource_list"]["qstat_l_requests"] as GPathResult).children() as List) + .findAll { it.name() == "qsat_l_requests" }.collectEntries { + [it["CE_name"], it["CE_doubleval"] as Integer] + } + Long requestedMemoryRaw = requestedResourcesRaw["h_rss"] as Long + BufferValue requestedMemory + if (requestedMemoryRaw != null) + requestedMemory = new BufferValue((Integer) (requestedMemoryRaw / 1024L), BufferUnit.k) + + Integer requestedWalltimeRaw = requestedResourcesRaw["h_rt"] as Integer + Duration requestedWalltime = requestedWalltimeRaw ? Duration.ofSeconds(requestedWalltimeRaw) : null + jobInfo.requestedResources = new ResourceSet(requestedMemory, requestedCores, null, requestedWalltime, null, null, null) + + Long usedWalltimeRaw = (extractStatisticsValue(job, "ru_wallclock") as Double) as Long + Duration usedWalltime = null + if (usedWalltime != null) + Duration.ofSeconds(usedWalltimeRaw) + Double usedMemoryRaw = extractStatisticsValue(job, "ru_maxrss") as Double + BufferValue usedMemory = null + if (usedMemoryRaw) + usedMemory = new BufferValue((usedMemoryRaw / 1024) as Integer, BufferUnit.k) + jobInfo.usedResources = new ResourceSet(usedMemory, requestedCores, null, usedWalltime, null, null, null) + + // jobInfo.cpuTime // acct_cpu?? + // jobInfo.runTime = safelyCastToDuration(job["JB_total_runtime"]) // ? + + /** Status info */ + // State is not set here! + jobInfo.exitCode = safelyCastToInteger(extractStatisticsValue(job, "exit_status")) + + /** Directories and files */ + boolean mergedLogs = job["JB_merge_stderr"] as Boolean + def outPath = job["JB_stdout_path_list"]["path_list"]["PN_path"] + jobInfo.logFile = outPath ? new File(outPath.toString()) : null + if (!mergedLogs) { + def errPath = job["JB_stderr_path_list"]["path_list"]["PN_path"] + jobInfo.errorLogFile = errPath ? new File(errPath.toString()) : null + } + + /** Timestamps */ + jobInfo.submitTime = extractAndParseTime(job, "submission_time") // The time the job entered the current queue. + jobInfo.startTime = extractAndParseTime(job, "start_time") // The time the job was started. + jobInfo.endTime = extractAndParseTime(job, "end_time") // The time the job was completed. + + return jobInfo + } + + ZonedDateTime extractAndParseTime(GPathResult job, String id) { + safelyParseTime((extractStatisticsValue(job, id) as Double) as Integer) + } + + Object extractStatisticsValue(GPathResult job, String id) { + GPathResult usageList = job["JB_ja_tasks"]["ulong_sublist"]["JAT_scaled_usage_list"] as GPathResult + GPathResult times = usageList.children().findAll { GPathResult it -> it.name() == "scaled" } + def entry = times.collectEntries { + [it["UA_name"]?.toString(), it["UA_value"].toString()] + }[id] + entry + } } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy index e71567c4..7808f099 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy @@ -9,14 +9,7 @@ package de.dkfz.roddy.execution.jobs.cluster.slurm import com.google.common.collect.LinkedHashMultimap import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService -import de.dkfz.roddy.execution.io.ExecutionResult -import de.dkfz.roddy.execution.jobs.BEJob -import de.dkfz.roddy.execution.jobs.BEJobID -import de.dkfz.roddy.execution.jobs.Command -import de.dkfz.roddy.execution.jobs.GenericJobInfo -import de.dkfz.roddy.execution.jobs.JobManagerOptions -import de.dkfz.roddy.execution.jobs.JobState -import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager +import de.dkfz.roddy.execution.jobs.* import de.dkfz.roddy.execution.jobs.cluster.GridEngineBasedJobManager import groovy.transform.CompileStatic @@ -58,17 +51,17 @@ class SlurmJobManager extends GridEngineBasedJobManager { } @Override - String getQueryJobStatesCommand() { + String getQueryCommandForJobInfo() { return null } @Override - String getExtendedQueryJobStatesCommand() { + String getQueryCommandForExtendedJobInfo() { return null } @Override - GenericJobInfo parseGenericJobInfo(String command) { + ExtendedJobInfo parseGenericJobInfo(String command) { return null } @@ -111,4 +104,9 @@ class SlurmJobManager extends GridEngineBasedJobManager { void createComputeParameter(ResourceSet resourceSet, LinkedHashMultimap parameters) { } + + @Override + Map queryExtendedJobInfo(List jobIDs) { + return null + } } \ No newline at end of file diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy index 8d907050..ae4607e9 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy @@ -24,22 +24,22 @@ class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManager queryJobStates(List jobIDs) { + Map queryJobInfo(List jobIDs) { + return [:] + } + + @Override + Map queryExtendedJobInfo(List jobIDs) { return [:] } @@ -158,17 +158,17 @@ class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManager queryExtendedJobStateById(List jobIds) { + Map queryExtendedJobStateById(List jobIds) { return [:] } } diff --git a/src/test/groovy/de/dkfz/roddy/batcheuphoria/BETestBaseSpec.groovy b/src/test/groovy/de/dkfz/roddy/batcheuphoria/BETestBaseSpec.groovy new file mode 100644 index 00000000..261442ff --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/batcheuphoria/BETestBaseSpec.groovy @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). + * + * Distributed under the MIT License (license terms are at https://www.github.com/TheRoddyWMS/BatchEuphoria/LICENSE.txt). + */ + +package de.dkfz.roddy.batcheuphoria + + +import spock.lang.Specification + +/** + * Base Spock spec for BE Spock specs + * Contains commonly used test methods + */ +class BETestBaseSpec extends Specification { + + static final File getResourceFile(Class _class, String file) { + String subDir = _class.package.name.replace(".", "/") + new File("src/test/resources/${subDir}/", file) + } + + def "test getResourceFile"() { + when: + File resourceFile = getResourceFile(BETestBaseSpec, "getResourceFileTest.txt") + String text = resourceFile.text + + then: + resourceFile.exists() + text == "abc" + } +} diff --git a/src/test/groovy/de/dkfz/roddy/batcheuphoria/jobs/JobManagerOptionsTest.groovy b/src/test/groovy/de/dkfz/roddy/batcheuphoria/jobs/JobManagerOptionsTest.groovy index f031b2b6..28ca2e02 100644 --- a/src/test/groovy/de/dkfz/roddy/batcheuphoria/jobs/JobManagerOptionsTest.groovy +++ b/src/test/groovy/de/dkfz/roddy/batcheuphoria/jobs/JobManagerOptionsTest.groovy @@ -23,7 +23,6 @@ class JobManagerOptionsTest { def parms = JobManagerOptions.create().build() assert parms.trackOnlyStartedJobs == false assert parms.updateInterval == Duration.ofMinutes(5) - assert parms.createDaemon == false assert !parms.passEnvironment } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/BEIntegrationTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/BEIntegrationTest.groovy index 0a6afc5d..15d7b26c 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/BEIntegrationTest.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/BEIntegrationTest.groovy @@ -18,7 +18,6 @@ import de.dkfz.roddy.tools.BufferValue import groovy.transform.CompileStatic import org.junit.BeforeClass import org.junit.Test -import org.xml.sax.SAXParseException import java.time.Duration @@ -71,7 +70,6 @@ class BEIntegrationTest { return system.loadClass().getDeclaredConstructor(BEExecutionService, JobManagerOptions) .newInstance(getExecutionServiceFor(system), JobManagerOptions.create() - .setCreateDaemon(false) .build() ) as BatchEuphoriaJobManager } @@ -148,14 +146,14 @@ class BEIntegrationTest { List lastStates = [] while (sleep > 0 && !allJobsInCorrectState) { lastStates.clear() - def status = jobManager.queryJobStatus(jobList, true) + def jobInfo = jobManager.queryJobInfoByJob(jobList) allJobsInCorrectState = true for (BEJob job in jobList) { - allJobsInCorrectState &= listOfStatesToCheck.contains(status[job]) - lastStates << status[job] + allJobsInCorrectState &= listOfStatesToCheck.contains(jobInfo[job]) + lastStates << jobInfo[job].jobState } if (!allJobsInCorrectState) { - assert status.values().join(" ").find(JobState.FAILED.name()) != JobState.FAILED.name() + assert jobInfo.values().join(" ").find(JobState.FAILED.name()) != JobState.FAILED.name() sleep-- } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManagerSpec.groovy new file mode 100644 index 00000000..0ecb7b5b --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManagerSpec.groovy @@ -0,0 +1,174 @@ +package de.dkfz.roddy.execution.jobs + +import de.dkfz.roddy.TestExecutionService +import de.dkfz.roddy.execution.jobs.cluster.JobManagerImplementationBaseSpec +import spock.lang.Shared + +import static de.dkfz.roddy.execution.jobs.JobState.* + +class BatchEuphoriaJobManagerSpec extends JobManagerImplementationBaseSpec { + + /** + * We will only test non abstract methods in the BatchEuphoriaJobManager class (which is abstract for reasons) + * For this, we will first create a custom test class, overriding some special methods necessary for our test + * cases. Afterwards, we create a Spy() of this class, which can then be utilized in our tests. + */ + abstract class TestBEJobManager extends BatchEuphoriaJobManager { + TestBEJobManager() { + super( + new TestExecutionService("test", "test"), + new JobManagerOptionsBuilder().build() + ) + } + + @Override + Map queryJobInfo(List jobIDs) { + // For the tests, it actually does not matter if the result contains JobInfo or ExtendedJobInfo objects + return queryExtendedJobInfo(jobIDs) as Map + } + + @Override + Map queryExtendedJobInfo(List jobIDs) { + [ + "1000": RUNNING, + "1001": SUSPENDED, + "1002": COMPLETED_SUCCESSFUL, + "1003": FAILED, + ].collectEntries { + String _id, JobState state -> + BEJobID id = new BEJobID(_id) + [id, new ExtendedJobInfo(id, state)] + } as Map + } + } + + def "test assertListIsValid"(list) { + when: + BatchEuphoriaJobManager.assertListIsValid(list) + + then: + thrown(AssertionError) + + where: + list | _ + null | _ + [] | _ + [null] | _ + } + + @Shared + TestBEJobManager jobManager = Spy() + + def "test queryJobInfoByJob (query for single object, also tests query by ID!)"(String id, JobState expectedState) { + when: + JobInfo result = jobManager.queryJobInfoByJob(new BEJob(new BEJobID(id), jobManager)) + + then: + result // There is ALWAYS a result! + result.jobState == expectedState + + where: + id | expectedState + "999" | UNKNOWN + "1000" | RUNNING + "1001" | SUSPENDED + "1002" | COMPLETED_SUCCESSFUL + "1003" | FAILED + } + + def "test queryJobInfoByJob"(String id, JobState expectedState) { + given: + def jobID = new BEJobID(id) + def job = new BEJob(jobID, jobManager) + + when: + def result = jobManager.queryJobInfoByJob([job]) + + then: + result.size() == 1 + result[job].jobState == expectedState + + where: + id | expectedState + "999" | UNKNOWN + "1000" | RUNNING + } + + def "test queryJobInfoByJob with null id or empty list"(List input) { + + when: + jobManager.queryJobInfoByJob(input) + + then: + thrown(AssertionError) + + where: + input | _ + null | _ + [] | _ + [null] | _ + } + + def "test queryAllJobInfo (same like above with null or [])"() { + expect: + jobManager.queryAllJobInfo().size() == 4 + } + + def "test queryExtendedJobInfoByJob (query for single object, also tests query by ID!)"() { + when: + JobInfo result = jobManager.queryExtendedJobInfoByJob(new BEJob(new BEJobID(id), jobManager)) + + then: + result // There is ALWAYS a result! + result.jobState == expectedState + + where: + id | expectedState + "999" | UNKNOWN + "1000" | RUNNING + "1001" | SUSPENDED + "1002" | COMPLETED_SUCCESSFUL + "1003" | FAILED + } + + def "test queryExtendedJobInfoByJob (will also test query by id"() { + given: + def jobID = new BEJobID(id) + def job = new BEJob(jobID, jobManager) + + when: + def result = jobManager.queryExtendedJobInfoByJob([job]) + + then: + result.size() == 1 + result[job].jobState == expectedState + + where: + id | expectedState + "999" | UNKNOWN + "1000" | RUNNING + "1001" | SUSPENDED + "1002" | COMPLETED_SUCCESSFUL + "1003" | FAILED + } + + def "test queryExtendedJobInfoByJob with null id or empty list"(List input) { + + when: + jobManager.queryExtendedJobInfoByJob(input) + + then: + thrown(AssertionError) + + where: + input | _ + null | _ + [] | _ + [null] | _ + } + + def "test queryAllExtendedJobInfo"() { + expect: + jobManager.queryAllExtendedJobInfo().size() == 4 + } +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfoTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfoTest.groovy new file mode 100644 index 00000000..7b2c0bc1 --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/ExtendedJobInfoTest.groovy @@ -0,0 +1,14 @@ +package de.dkfz.roddy.execution.jobs + +import spock.lang.Specification + +class ExtendedJobInfoTest extends Specification { + def "test @EqualsAndHashCode annotation with differing base class"() { + when: + def g1 = new ExtendedJobInfo(new BEJobID("1000")) + def g2 = new ExtendedJobInfo(new BEJobID("1001")) + + then: + g1 != g2 + } +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommandTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommandTest.groovy index 645ddf15..aaea9629 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommandTest.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/SubmissionCommandTest.groovy @@ -14,7 +14,7 @@ class SubmissionCommandTest extends Specification { JobManagerOptions.create().setPassEnvironment(passEnvironment).build()) { @Override - Map queryExtendedJobStateById(List jobIds) { + Map queryExtendedJobStateById(List jobIds) { return null } @@ -54,12 +54,12 @@ class SubmissionCommandTest extends Specification { } @Override - String getQueryJobStatesCommand() { + String getQueryCommandForJobInfo() { return null } @Override - String getExtendedQueryJobStatesCommand() { + String getQueryCommandForExtendedJobInfo() { return null } @@ -69,7 +69,7 @@ class SubmissionCommandTest extends Specification { } @Override - GenericJobInfo parseGenericJobInfo(String command) { + ExtendedJobInfo parseGenericJobInfo(String command) { return null } @@ -99,7 +99,12 @@ class SubmissionCommandTest extends Specification { } @Override - protected Map queryJobStates(List jobIDs) { + Map queryJobInfo(List jobIDs) { + return null + } + + @Override + Map queryExtendedJobInfo(List jobIDs) { return null } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManagerSpec.groovy index 02272a77..e556ddc0 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/ClusterJobManagerSpec.groovy @@ -6,36 +6,55 @@ package de.dkfz.roddy.execution.jobs.cluster +import de.dkfz.roddy.BEException import org.slf4j.Logger import spock.lang.Specification import java.lang.reflect.Field import java.lang.reflect.Method import java.lang.reflect.Modifier +import java.time.Duration class ClusterJobManagerSpec extends Specification { - private static Method prepareLogger(Logger log) { - Method method = ClusterJobManager.class.getDeclaredMethod("withCaughtAndLoggedException", Closure) - method.setAccessible(true) + void "test parseColonSeparatedHHMMSSDuration, parse duration"() { + expect: + ClusterJobManager.parseColonSeparatedHHMMSSDuration(input) == parsedDuration - Field f = ClusterJobManager.class.getDeclaredField("log") + where: + input | parsedDuration + "00:00:00" | Duration.ofSeconds(0) + "24:00:00" | Duration.ofHours(24) + "119:00:00" | Duration.ofHours(119) + } + + void "test parseColonSeparatedHHMMSSDuration, parse duration fails"() { + when: + ClusterJobManager.parseColonSeparatedHHMMSSDuration("02:42") + + then: + BEException e = thrown(BEException) + e.message == "Duration string is not of the format HH+:MM:SS: '02:42'" + } + + private static Method prepareClassLogger(Class _class, Logger log) { + Field f = _class.getDeclaredField("log") f.setAccessible(true) - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(f, f.getModifiers() & ~Modifier.FINAL); + + Field modifiersField = f.class.getDeclaredField("modifiers") + modifiersField.setAccessible(true) + modifiersField.setInt(f, f.getModifiers() & ~Modifier.FINAL) f.set(null, log) - return method } - def "test catchExceptionAndLog throws exception"() { + def "test withCaughtAndLoggedException throws exception"() { given: Logger log = Mock(Logger) - Method method = prepareLogger(log) + prepareClassLogger(ClusterJobManager, log) when: - Object result = method.invoke(null, { throw new Exception("123"); return "ABC" }) + Object result = ClusterJobManager.withCaughtAndLoggedException { throw new Exception("123") } then: result == null @@ -43,13 +62,13 @@ class ClusterJobManagerSpec extends Specification { 1 * log.warn(_) } - def "test catchExceptionAndLog returns value"() { + def "test withCaughtAndLoggedException returns value"() { given: Logger log = Mock(Logger) - Method method = prepareLogger(log) + prepareClassLogger(ClusterJobManager, log) when: - Object result = method.invoke(null, { return "ABC" }) + Object result = ClusterJobManager.withCaughtAndLoggedException { return "ABC" } then: result == "ABC" diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBaseJobManagerTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBaseJobManagerTest.groovy deleted file mode 100644 index 1a581c54..00000000 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBaseJobManagerTest.groovy +++ /dev/null @@ -1,200 +0,0 @@ -package de.dkfz.roddy.execution.jobs.cluster - -import com.google.common.collect.LinkedHashMultimap -import de.dkfz.roddy.config.JobLog -import de.dkfz.roddy.config.ResourceSet -import de.dkfz.roddy.config.ResourceSetSize -import de.dkfz.roddy.execution.jobs.BEJob -import de.dkfz.roddy.execution.jobs.Command -import de.dkfz.roddy.execution.jobs.GenericJobInfo -import de.dkfz.roddy.execution.jobs.JobManagerOptions -import de.dkfz.roddy.execution.jobs.JobState -import de.dkfz.roddy.execution.jobs.TestHelper -import de.dkfz.roddy.execution.jobs.cluster.pbs.PBSCommand -import de.dkfz.roddy.execution.jobs.cluster.pbs.PBSJobManager -import de.dkfz.roddy.tools.BufferUnit -import de.dkfz.roddy.tools.BufferValue -import de.dkfz.roddy.tools.TimeUnit -import groovy.transform.CompileStatic -import org.junit.Before -import org.junit.Test - -@CompileStatic -class GridEngineBaseJobManagerTest { - - GridEngineBasedJobManager jobManager - - @Before - void setUp() throws Exception { - jobManager = new GridEngineBasedJobManager(TestHelper.makeExecutionService(), JobManagerOptions.create().build()) { - @Override - void createComputeParameter(ResourceSet resourceSet, LinkedHashMultimap parameters) { - - } - - @Override - void createQueueParameter(LinkedHashMultimap parameters, String queue) { - - } - - @Override - void createWalltimeParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) { - - } - - @Override - void createMemoryParameter(LinkedHashMultimap parameters, ResourceSet resourceSet) { - - } - - @Override - void createStorageParameters(LinkedHashMultimap parameters, ResourceSet resourceSet) { - - } - - @Override - String getJobIdVariable() { - return null - } - - @Override - String getJobNameVariable() { - return null - } - - @Override - String getQueueVariable() { - return null - } - - @Override - String getNodeFileVariable() { - return null - } - - @Override - String getSubmitHostVariable() { - return null - } - - @Override - String getSubmitDirectoryVariable() { - return null - } - - @Override - String getQueryJobStatesCommand() { - return null - } - - @Override - String getExtendedQueryJobStatesCommand() { - return null - } - - @Override - GenericJobInfo parseGenericJobInfo(String command) { - return null - } - - @Override - protected Command createCommand(BEJob job) { - return null - } - - @Override - protected String parseJobID(String commandOutput) { - return null - } - - @Override - protected JobState parseJobState(String stateString) { - return null - } - } - } - - private BEJob makeJob(Map mapOfParameters) { - BEJob job = new BEJob(null, "Test", new File("/tmp/test.sh"), null, null, new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 4, 1, new TimeUnit("1h"), null, null, null), [], mapOfParameters, jobManager, JobLog.none(), null) - job - } - - @Test - void testAssembleDependencyStringWithoutDependencies() throws Exception { - def mapOfVars = ["a": "a", "b": "b"] - GridEngineBasedCommand cmd = new GridEngineBasedCommand(jobManager, makeJob(mapOfVars), - "jobName", null, mapOfVars, null, "/tmp/test.sh") { - @Override - protected String getDependsSuperParameter() { - return null - } - - @Override - protected String getDependencyParameterName() { - return null - } - - @Override - protected String getDependencyOptionSeparator() { - return null - } - - @Override - protected String getDependencyIDSeparator() { - return null - } - - @Override - protected String getJobNameParameter() { - return null - } - - @Override - protected String getHoldParameter() { - return null - } - - @Override - protected String getAccountParameter(String account) { - return null - } - - @Override - protected String getWorkingDirectory() { - return null - } - - @Override - protected String getLoggingParameter(JobLog jobLog) { - return null - } - - @Override - protected String getEmailParameter(String address) { - return null - } - - @Override - protected String getGroupListParameter(String groupList) { - return null - } - - @Override - protected String getUmaskString(String umask) { - return null - } - - @Override - protected String getAdditionalCommandParameters() { - return null - } - - @Override - protected String assembleVariableExportParameters() { - return null - } - } - assert cmd.assembleDependencyString([]) == "" - } - -} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManagerSpec.groovy index d714efdc..c3a75f14 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineBasedJobManagerSpec.groovy @@ -9,17 +9,18 @@ class GridEngineBasedJobManagerSpec extends Specification { output == GridEngineBasedJobManager.getExecutionHosts(input) where: - input || output - null || null - "" || null - "asdf" || ["asdf"] - "asdf+asdf" || ["asdf"] - "asdf+qwertz" || ["asdf", "qwertz"] - "asdf/5+asdf" || ["asdf"] - "asdf/10+asdf/1" || ["asdf"] - "asdf/3+qwertz/7" || ["asdf", "qwertz"] - "asdf+qwertz/2" || ["asdf", "qwertz"] - "asdf+qwertz/2+yxcv/6" || ["asdf", "qwertz", "yxcv"] + input || output + null || [] + "" || [] + "asdf" || ["asdf"] + "asdf+asdf" || ["asdf"] + "asdf+qwertz" || ["asdf", "qwertz"] + "asdf/5+asdf" || ["asdf"] + "asdf/10+asdf/1" || ["asdf"] + "asdf/3+qwertz/7" || ["asdf", "qwertz"] + "asdf+qwertz/2" || ["asdf", "qwertz"] + "asdf+qwertz/2+yxcv/6" || ["asdf", "qwertz", "yxcv"] + "exec-host/0+exec-host/1+exec-host/2" || ["exec-host"] } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineQstatParserTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineQstatParserTest.groovy deleted file mode 100644 index 4b628a96..00000000 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/GridEngineQstatParserTest.groovy +++ /dev/null @@ -1,429 +0,0 @@ -/* - * Copyright (c) 2017 eilslabs. - * - * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). - */ - -package de.dkfz.roddy.execution.jobs.cluster - -import de.dkfz.roddy.TestExecutionService -import de.dkfz.roddy.execution.jobs.JobManagerOptions -import de.dkfz.roddy.execution.jobs.cluster.pbs.PBSJobManager -import groovy.transform.CompileDynamic -import groovy.transform.CompileStatic -import org.junit.Test - -/** - * Created by heinold on 26.03.17. - */ -@CompileStatic -class GridEngineQstatParserTest { - - final static String output1 = """\ -Job Id: 14973441.tbi-pbs-ng.inet.dkfz-heidelberg.de - Job_Name = r170623_063520503_A056-XQH5TD_createControlBafPlots - Job_Owner = otproddy@tbi-pbs4.inet.dkfz-heidelberg.de - resources_used.cput = 00:02:38 - resources_used.energy_used = 0 - resources_used.mem = 472772kb - resources_used.vmem = 756656kb - resources_used.walltime = 00:03:44 - job_state = C - queue = medium - server = tbi-pbs-ng.inet.dkfz-heidelberg.de - Checkpoint = u - ctime = Fri Jun 23 06:35:59 2017 - Error_Path = tbi-pbs4:/icgc/dkfzlsdf/project/hipo/hipo_A056/sequencing/who -\tle_genome_sequencing/view-by-pid/A056-XQH5TD/cnv_results/paired/metast -\tasis_blood/results_ACEseqWorkflow-1.2.8-1_v1_0_2017-06-23_06h33_+0200/ -\troddyExecutionStore/exec_170623_063520503_otproddy_WGS/r170623_0635205 -\t03_A056-XQH5TD_createControlBafPlots.e14973441 - exec_host = tbi-dsx50/22 - group_list = B080 - Hold_Types = n - Join_Path = oe - Keep_Files = n - Mail_Points = a - mtime = Fri Jun 23 10:59:30 2017 - Output_Path = tbi-pbs4:/icgc/dkfzlsdf/project/hipo/hipo_A056/sequencing/wh -\tole_genome_sequencing/view-by-pid/A056-XQH5TD/cnv_results/paired/metas -\ttasis_blood/results_ACEseqWorkflow-1.2.8-1_v1_0_2017-06-23_06h33_+0200 -\t/roddyExecutionStore/exec_170623_063520503_otproddy_WGS/r170623_063520 -\t503_A056-XQH5TD_createControlBafPlots.o14973441 - Priority = 0 - qtime = Sat Jun 3 06:35:59 2017 - Rerunable = True - Resource_List.mem = 5120mb - Resource_List.walltime = 01:00:00 - session_id = 34432 - euser = otproddy - egroup = B080 - queue_type = E - etime = Fri Jun 23 10:54:49 2017 - exit_status = 0 - submit_args = -N r170623_063520503_A056-XQH5TD_createControlBafPlots -o /i -\tcgc/dkfzlsdf/project/hipo/hipo_A056/sequencing/whole_genome_sequencing -\t/view-by-pid/A056-XQH5TD/cnv_results/paired/metastasis_blood/results_A -\tCEseqWorkflow-1.2.8-1_v1_0_2017-06-23_06h33_+0200/roddyExecutionStore/ -\texec_170623_063520503_otproddy_WGS -j oe -W group_list=B080 -W umask=0 -\t07 -l mem=5120M -l walltime=00:01:00:00 -W depend=afterok:14973440.tbi -\t-pbs-ng.inet.dkfz-heidelberg.de:14973412.tbi-pbs-ng.inet.dkfz-heidelbe -\trg.de -v WRAPPED_SCRIPT=/icgc/dkfzlsdf/project/hipo/hipo_A056/sequenci -\tng/whole_genome_sequencing/view-by-pid/A056-XQH5TD/cnv_results/paired/ -\tmetastasis_blood/results_ACEseqWorkflow-1.2.8-1_v1_0_2017-06-23_06h33_ -\t+0200/roddyExecutionStore/exec_170623_063520503_otproddy_WGS/analysisT -\tools/copyNumberEstimationWorkflow/createControlBafPlots.sh, -\tPARAMETER_FILE=/icgc/dkfzlsdf/project/hipo/hipo_A056/sequencing/whole -\t_genome_sequencing/view-by-pid/A056-XQH5TD/cnv_results/paired/metastas -\tis_blood/results_ACEseqWorkflow-1.2.8-1_v1_0_2017-06-23_06h33_+0200/ro -\tddyExecutionStore/exec_170623_063520503_otproddy_WGS/r170623_063520503 -\t_A056-XQH5TD_createControlBafPlots_120.parameters /icgc/dkfzlsdf/proje -\tct/hipo/hipo_A056/sequencing/whole_genome_sequencing/view-by-pid/A056- -\tXQH5TD/cnv_results/paired/metastasis_blood/results_ACEseqWorkflow-1.2. -\t8-1_v1_0_2017-06-23_06h33_+0200/roddyExecutionStore/exec_170623_063520 -\t503_otproddy_WGS/analysisTools/roddyTools/wrapInScript.sh - umask = 7 - start_time = Fri Jun 23 10:55:46 2017 - start_count = 1 - fault_tolerant = False - comp_time = Fri Jun 23 10:59:30 2017 - job_radix = 0 - total_runtime = 279.826458 - submit_host = tbi-pbs4.inet.dkfz-heidelberg.de - request_version = 1 - -Job Id: 14973792.tbi-pbs-ng.inet.dkfz-heidelberg.de - Job_Name = yapima - Job_Owner = pastor@tbi-worker.inet.dkfz-heidelberg.de - job_state = Q - queue = long - server = tbi-pbs-ng.inet.dkfz-heidelberg.de - Checkpoint = u - ctime = Fri Jun 23 11:02:42 2017 - Error_Path = tbi-worker:/ibios/co02/xavier/eo/yapima/yapima.e14973792 - Hold_Types = n - Join_Path = oe - Keep_Files = n - Mail_Points = a - Mail_Users = output2.pastorhostench@dkfz-heidelberg.de - mtime = Fri Jun 23 11:02:42 2017 - Output_Path = tbi-worker:/ibios/co02/xavier/eo/yapima/yapima.o14973792 - Priority = 0 - qtime = Fri Jun 23 11:02:42 2017 - Rerunable = True - Resource_List.mem = 10gb - Resource_List.nodect = 1 - Resource_List.nodes = 1:ppn=1 - Resource_List.walltime = 03:00:00 - euser = pastor - egroup = B080 - queue_type = E - etime = Fri Jun 23 11:02:42 2017 - submit_args = -o /ibios/co02/xavier/eo/yapima -j oe -M output2.pastorhostench@dk -\tfz-heidelberg.de -N yapima -l walltime=3:00:00,nodes=1:ppn=1, -\tmem=10g -v CONFIG_FILE=/icgc/dkfzlsdf/analysis/hipo/hipo_043/methylat -\tion_data_H043/config_yapima.sh /home/pastor/pipelines/run/yapima/proce -\tss450k.sh - fault_tolerant = False - job_radix = 0 - submit_host = tbi-worker.inet.dkfz-heidelberg.de - request_version = 1 - -Job Id: 14973766.tbi-pbs-ng.inet.dkfz-heidelberg.de - Job_Name = r170623_105203487_A017-WQL2_snvCalling - Job_Owner = otproddy@tbi-pbs4.inet.dkfz-heidelberg.de - resources_used.cput = 00:10:06 - resources_used.energy_used = 0 - resources_used.mem = 146892kb - resources_used.vmem = 261756kb - resources_used.walltime = 00:10:03 - job_state = R - queue = verylong - server = tbi-pbs-ng.inet.dkfz-heidelberg.de - Checkpoint = u - ctime = Fri Jun 23 10:52:20 2017 - depend = beforeok:14973775.tbi-pbs-ng.inet.dkfz-heidelberg.de - Error_Path = tbi-pbs4:/icgc/dkfzlsdf/project/hipo/hipo_A017/sequencing/who -\tle_genome_sequencing/view-by-pid/A017-WQL2/snv_results/paired/tumor_co -\tntrol02/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_10h50_+02 -\t00/roddyExecutionStore/exec_170623_105203487_otproddy_WGS/r170623_1052 -\t03487_A017-WQL2_snvCalling.e14973766 - exec_host = tbi-dsx55/25 - group_list = B080 - Hold_Types = n - Join_Path = oe - Keep_Files = n - Mail_Points = a - mtime = Fri Jun 23 10:53:47 2017 - Output_Path = tbi-pbs4:/icgc/dkfzlsdf/project/hipo/hipo_A017/sequencing/wh -\tole_genome_sequencing/view-by-pid/A017-WQL2/snv_results/paired/tumor_c -\tontrol02/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_10h50_+0 -\t200/roddyExecutionStore/exec_170623_105203487_otproddy_WGS/r170623_105 -\t203487_A017-WQL2_snvCalling.o14973766 - Priority = 0 - qtime = Fri Jun 23 10:52:20 2017 - Rerunable = True - Resource_List.mem = 4096mb - Resource_List.nodect = 1 - Resource_List.nodes = 1:ppn=1 - Resource_List.walltime = 48:00:00 - session_id = 45434 - euser = otproddy - egroup = B080 - queue_type = E - etime = Fri Jun 23 10:52:20 2017 - submit_args = -N r170623_105203487_A017-WQL2_snvCalling -o /icgc/dkfzlsdf/ -\tproject/hipo/hipo_A017/sequencing/whole_genome_sequencing/view-by-pid/ -\tA017-WQL2/snv_results/paired/tumor_control02/results_SNVCallingWorkflo -\tw-1.0.166-1_v1_0_2017-06-23_10h50_+0200/roddyExecutionStore/exec_17062 -\t3_105203487_otproddy_WGS -j oe -W group_list=B080 -W umask=007 -l mem= -\t4096m -l nodes=1:ppn=1 -l walltime=48:00:00 -v WRAPPED_SCRIPT=/icgc/dk -\tfzlsdf/project/hipo/hipo_A017/sequencing/whole_genome_sequencing/view- -\tby-pid/A017-WQL2/snv_results/paired/tumor_control02/results_SNVCalling -\tWorkflow-1.0.166-1_v1_0_2017-06-23_10h50_+0200/roddyExecutionStore/exe -\tc_170623_105203487_otproddy_WGS/analysisTools/snvPipeline/snvCalling.output1 -\th, -\tPARAMETER_FILE=/icgc/dkfzlsdf/project/hipo/hipo_A017/sequencing/whole -\t_genome_sequencing/view-by-pid/A017-WQL2/snv_results/paired/tumor_cont -\trol02/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_10h50_+0200 -\t/roddyExecutionStore/exec_170623_105203487_otproddy_WGS/r170623_105203 -\t487_A017-WQL2_snvCalling_50.parameters /icgc/dkfzlsdf/project/hipo/hip -\to_A017/sequencing/whole_genome_sequencing/view-by-pid/A017-WQL2/snv_re -\tsults/paired/tumor_control02/results_SNVCallingWorkflow-1.0.166-1_v1_0 -\t_2017-06-23_10h50_+0200/roddyExecutionStore/exec_170623_105203487_otpr -\toddy_WGS/analysisTools/roddyTools/wrapInScript.sh - umask = 7 - start_time = Fri Jun 23 10:53:47 2017 - Walltime.Remaining = 172137 - start_count = 1 - fault_tolerant = False - job_radix = 0 - submit_host = tbi-pbs4.inet.dkfz-heidelberg.de - request_version = 1 - -Job Id: 14973745.tbi-pbs-ng.inet.dkfz-heidelberg.de - Job_Name = r170623_10495311_A017-39867J_snvAnnotation - Job_Owner = otproddy@tbi-pbs4.inet.dkfz-heidelberg.de - job_state = H - queue = verylong - server = tbi-pbs-ng.inet.dkfz-heidelberg.de - Checkpoint = u - ctime = Fri Jun 23 10:50:14 2017 - depend = afterok:14973744.tbi-pbs-ng.inet.dkfz-heidelberg.de, -\tbeforeok:14973746.tbi-pbs-ng.inet.dkfz-heidelberg.de - Error_Path = tbi-pbs4:/icgc/dkfzlsdf/project/hipo/hipo_A017/sequencing/who -\tle_genome_sequencing/view-by-pid/A017-39867J/snv_results/paired/metast -\tasis_control02/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_10 -\th48_+0200/roddyExecutionStore/exec_170623_10495311_otproddy_WGS/r17062 -\t3_10495311_A017-39867J_snvAnnotation.e14973745 - group_list = B080 - Hold_Types = output1 - Join_Path = oe - Keep_Files = n - Mail_Points = a - mtime = Fri Jun 23 10:50:15 2017 - Output_Path = tbi-pbs4:/icgc/dkfzlsdf/project/hipo/hipo_A017/sequencing/wh -\tole_genome_sequencing/view-by-pid/A017-39867J/snv_results/paired/metas -\ttasis_control02/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_1 -\t0h48_+0200/roddyExecutionStore/exec_170623_10495311_otproddy_WGS/r1706 -\t23_10495311_A017-39867J_snvAnnotation.o14973745 - Priority = 0 - qtime = Fri Jun 23 10:50:14 2017 - Rerunable = True - Resource_List.mem = 4096mb - Resource_List.nodect = 1 - Resource_List.nodes = 1:ppn=2 - Resource_List.walltime = 180:00:00 - euser = otproddy - egroup = B080 - queue_type = E - submit_args = -N r170623_10495311_A017-39867J_snvAnnotation -o /icgc/dkfzl -\tsdf/project/hipo/hipo_A017/sequencing/whole_genome_sequencing/view-by- -\tpid/A017-39867J/snv_results/paired/metastasis_control02/results_SNVCal -\tlingWorkflow-1.0.166-1_v1_0_2017-06-23_10h48_+0200/roddyExecutionStore -\t/exec_170623_10495311_otproddy_WGS -j oe -W group_list=B080 -W umask=0 -\t07 -l mem=4096m -l nodes=1:ppn=2 -l walltime=180:00:00 -W depend=after -\tok:14973744.tbi-pbs-ng.inet.dkfz-heidelberg.de -v WRAPPED_SCRIPT=/icgc -\t/dkfzlsdf/project/hipo/hipo_A017/sequencing/whole_genome_sequencing/vi -\tew-by-pid/A017-39867J/snv_results/paired/metastasis_control02/results_ -\tSNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_10h48_+0200/roddyExecutio -\tnStore/exec_170623_10495311_otproddy_WGS/analysisTools/snvPipeline/snv -\tAnnotation.sh, -\tPARAMETER_FILE=/icgc/dkfzlsdf/project/hipo/hipo_A017/sequencing/whole -\t_genome_sequencing/view-by-pid/A017-39867J/snv_results/paired/metastas -\tis_control02/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_10h4 -\t8_+0200/roddyExecutionStore/exec_170623_10495311_otproddy_WGS/r170623_ -\t10495311_A017-39867J_snvAnnotation_60.parameters /icgc/dkfzlsdf/projec -\tt/hipo/hipo_A017/sequencing/whole_genome_sequencing/view-by-pid/A017-3 -\t9867J/snv_results/paired/metastasis_control02/results_SNVCallingWorkfl -\tow-1.0.166-1_v1_0_2017-06-23_10h48_+0200/roddyExecutionStore/exec_1706 -\t23_10495311_otproddy_WGS/analysisTools/roddyTools/wrapInScript.sh - umask = 7 - fault_tolerant = False - job_radix = 0 - submit_host = tbi-pbs4.inet.dkfz-heidelberg.de - request_version = 1 - -""" - - - final static String output2 = """ -Job Id: 14973826.tbi-pbs-ng.inet.dkfz-heidelberg.de - Job_Name = r170623_111000536_XI061_9EP29_snvDeepAnnotation - Job_Owner = otproddy@tbi-pbs4.inet.dkfz-heidelberg.de - job_state = H - queue = long - server = tbi-pbs-ng.inet.dkfz-heidelberg.de - Checkpoint = u - ctime = Fri Jun 23 11:10:25 2017 - depend = afterok:14973825.tbi-pbs-ng.inet.dkfz-heidelberg.de, - beforeok:14973827.tbi-pbs-ng.inet.dkfz-heidelberg.de - Error_Path = tbi-pbs4:/icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/sequ - encing/whole_genome_sequencing/view-by-pid/XI061_9EP29/snv_results/pai - red/tumor_blood/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_1 - 1h08_+0200/roddyExecutionStore/exec_170623_111000536_otproddy_WGS/r170 - 623_111000536_XI061_9EP29_snvDeepAnnotation.e14973826 - group_list = B080 - Hold_Types = output1 - Join_Path = oe - Keep_Files = n - Mail_Points = a - mtime = Fri Jun 23 11:10:25 2017 - Output_Path = tbi-pbs4:/icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/seq - uencing/whole_genome_sequencing/view-by-pid/XI061_9EP29/snv_results/pa - ired/tumor_blood/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_ - 11h08_+0200/roddyExecutionStore/exec_170623_111000536_otproddy_WGS/r17 - 0623_111000536_XI061_9EP29_snvDeepAnnotation.o14973826 - Priority = 0 - qtime = Fri Jun 23 11:10:25 2017 - Rerunable = True - Resource_List.mem = 4096mb - Resource_List.nodect = 1 - Resource_List.nodes = 1:ppn=3 - Resource_List.walltime = 04:00:00 - euser = otproddy - egroup = B080 - queue_type = E - submit_args = -N r170623_111000536_XI061_9EP29_snvDeepAnnotation -o /icgc/ - dkfzlsdf/project/Xintern/XI061_ependymoma/sequencing/whole_genome_sequ - encing/view-by-pid/XI061_9EP29/snv_results/paired/tumor_blood/results_ - SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_11h08_+0200/roddyExecutio - nStore/exec_170623_111000536_otproddy_WGS -j oe -W group_list=B080 -W - umask=007 -l mem=4096m -l nodes=1:ppn=3 -l walltime=4:00:00 -W depend= - afterok:14973825.tbi-pbs-ng.inet.dkfz-heidelberg.de -v WRAPPED_SCRIPT= - /icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/sequencing/whole_genom - e_sequencing/view-by-pid/XI061_9EP29/snv_results/paired/tumor_blood/re - sults_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_11h08_+0200/roddyEx - ecutionStore/exec_170623_111000536_otproddy_WGS/analysisTools/tools/vc - f_pipeAnnotator.sh, - PARAMETER_FILE=/icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/sequen - cing/whole_genome_sequencing/view-by-pid/XI061_9EP29/snv_results/paire - d/tumor_blood/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_11h - 08_+0200/roddyExecutionStore/exec_170623_111000536_otproddy_WGS/r17062 - 3_111000536_XI061_9EP29_snvDeepAnnotation_61.parameters /icgc/dkfzlsdf - /project/Xintern/XI061_ependymoma/sequencing/whole_genome_sequencing/v - iew-by-pid/XI061_9EP29/snv_results/paired/tumor_blood/results_SNVCalli - ngWorkflow-1.0.166-1_v1_0_2017-06-23_11h08_+0200/roddyExecutionStore/e - xec_170623_111000536_otproddy_WGS/analysisTools/roddyTools/wrapInScrip - t.sh - umask = 7 - fault_tolerant = False - job_radix = 0 - submit_host = tbi-pbs4.inet.dkfz-heidelberg.de - request_version = 1 - -Job Id: 14973827.tbi-pbs-ng.inet.dkfz-heidelberg.de - Job_Name = r170623_111000536_XI061_9EP29_snvFilter - Job_Owner = otproddy@tbi-pbs4.inet.dkfz-heidelberg.de - job_state = H - queue = long - server = tbi-pbs-ng.inet.dkfz-heidelberg.de - Checkpoint = u - ctime = Fri Jun 23 11:10:25 2017 - depend = afterok:14973826.tbi-pbs-ng.inet.dkfz-heidelberg.de:14973824.tbi- - pbs-ng.inet.dkfz-heidelberg.de - Error_Path = tbi-pbs4:/icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/sequ - encing/whole_genome_sequencing/view-by-pid/XI061_9EP29/snv_results/pai - red/tumor_blood/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_1 - 1h08_+0200/roddyExecutionStore/exec_170623_111000536_otproddy_WGS/r170 - 623_111000536_XI061_9EP29_snvFilter.e14973827 - group_list = B080 - Hold_Types = output1 - Join_Path = oe - Keep_Files = n - Mail_Points = a - mtime = Fri Jun 23 11:10:25 2017 - Output_Path = tbi-pbs4:/icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/seq - uencing/whole_genome_sequencing/view-by-pid/XI061_9EP29/snv_results/pa - ired/tumor_blood/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_ - 11h08_+0200/roddyExecutionStore/exec_170623_111000536_otproddy_WGS/r17 - 0623_111000536_XI061_9EP29_snvFilter.o14973827 - Priority = 0 - qtime = Fri Jun 23 11:10:25 2017 - Rerunable = True - Resource_List.mem = 4096mb - Resource_List.nodect = 1 - Resource_List.nodes = 1:ppn=1 - Resource_List.walltime = 04:00:00 - euser = otproddy - egroup = B080 - queue_type = E - submit_args = -N r170623_111000536_XI061_9EP29_snvFilter -o /icgc/dkfzlsdf - /project/Xintern/XI061_ependymoma/sequencing/whole_genome_sequencing/v - iew-by-pid/XI061_9EP29/snv_results/paired/tumor_blood/results_SNVCalli - ngWorkflow-1.0.166-1_v1_0_2017-06-23_11h08_+0200/roddyExecutionStore/e - xec_170623_111000536_otproddy_WGS -j oe -W group_list=B080 -W umask=00 - 7 -l mem=4096m -l nodes=1:ppn=1 -l walltime=4:00:00 -W depend=afterok: - 14973826.tbi-pbs-ng.inet.dkfz-heidelberg.de:14973824.tbi-pbs-ng.inet.d - kfz-heidelberg.de -v WRAPPED_SCRIPT=/icgc/dkfzlsdf/project/Xintern/XI0 - 61_ependymoma/sequencing/whole_genome_sequencing/view-by-pid/XI061_9EP - 29/snv_results/paired/tumor_blood/results_SNVCallingWorkflow-1.0.166-1 - _v1_0_2017-06-23_11h08_+0200/roddyExecutionStore/exec_170623_111000536 - _otproddy_WGS/analysisTools/snvPipeline/filter_vcf.sh, - PARAMETER_FILE=/icgc/dkfzlsdf/project/Xintern/XI061_ependymoma/sequen - cing/whole_genome_sequencing/view-by-pid/XI061_9EP29/snv_results/paire - d/tumor_blood/results_SNVCallingWorkflow-1.0.166-1_v1_0_2017-06-23_11h - 08_+0200/roddyExecutionStore/exec_170623_111000536_otproddy_WGS/r17062 - 3_111000536_XI061_9EP29_snvFilter_62.parameters /icgc/dkfzlsdf/project - /Xintern/XI061_ependymoma/sequencing/whole_genome_sequencing/view-by-p - id/XI061_9EP29/snv_results/paired/tumor_blood/results_SNVCallingWorkfl - ow-1.0.166-1_v1_0_2017-06-23_11h08_+0200/roddyExecutionStore/exec_1706 - 23_111000536_otproddy_WGS/analysisTools/roddyTools/wrapInScript.sh - umask = 7 - fault_tolerant = False - job_radix = 0 - submit_host = tbi-pbs4.inet.dkfz-heidelberg.de - request_version = 1 - -""" - - protected List getTestQstat() { - return Arrays.asList( - "job - ID prior name user jobState submit / start at queue slots ja -task - ID", - "---------------------------------------------------------------------------------------------------------------- -", - " 1187 0.75000 r140710_09 seqware r 07 / 10 / 2014 09:51:55 main.q @worker3 1", - " 1188 0.41406 r140710_09 seqware r 07 / 10 / 2014 09:51:40 main.q @worker1 1", - " 1190 0.25000 r140710_09 seqware r 07 / 10 / 2014 09:51:55 main.q @worker2 1", - " 1189 0.00000 r140710_09 seqware hqw 07 / 10 / 2014 09:51:27 1", - " 1191 0.00000 r140710_09 seqware hqw 07 / 10 / 2014 09:51:48 1", - " 1192 0.00000 r140710_09 seqware hqw 07 / 10 / 2014 09:51:48 1") - } - - @CompileDynamic - @Test - void testProcessQstatOutputFromPlainText() throws Exception { - TestExecutionService executionService = new TestExecutionService("", "") - PBSJobManager jm = new PBSJobManager(executionService, JobManagerOptions.create() - .setCreateDaemon(false) - .setUserIdForJobQueries("asdf") - .build()) - - Map> qstatReaderResultOutput1 = (Map>) GridEngineBasedJobManager.processQstatOutputFromPlainText(output1) - Map> qstatReaderResultOutput2 = (Map>) GridEngineBasedJobManager.processQstatOutputFromPlainText(output2) - - assert qstatReaderResultOutput1.size() == 4 - assert qstatReaderResultOutput1.keySet() as List == [ "14973441", "14973792", "14973766", "14973745"] - assert qstatReaderResultOutput2.size() == 2 - } - -} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpec.groovy new file mode 100644 index 00000000..fab02f28 --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpec.groovy @@ -0,0 +1,123 @@ +package de.dkfz.roddy.execution.jobs.cluster + +import com.google.common.reflect.TypeToken +import de.dkfz.roddy.TestExecutionService +import de.dkfz.roddy.batcheuphoria.BETestBaseSpec +import de.dkfz.roddy.execution.BEExecutionService +import de.dkfz.roddy.execution.io.ExecutionResult +import de.dkfz.roddy.execution.jobs.BEJobID +import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager +import de.dkfz.roddy.execution.jobs.JobManagerOptions + +import java.lang.reflect.Constructor + +/** + * Base test specification for job manager implementations. + * + * Used to enforce a common test structure for all subclasses. + */ +abstract class JobManagerImplementationBaseSpec extends BETestBaseSpec { + + protected static final BEJobID testJobID = new BEJobID("22005") + + /** + * https://stackoverflow.com/questions/3403909/get-generic-type-of-class-at-runtime + */ + private final TypeToken typeToken = new TypeToken(getClass()) {} + + private final Class jobManagerClass = typeToken.getRawType() + + T _newJobManager(BEExecutionService testExecutionService, JobManagerOptions parms) { + Constructor c = jobManagerClass.getConstructor(BEExecutionService, JobManagerOptions) + return c.newInstance(testExecutionService, parms) + } + + /** + * Create a job manager instance of type T. + * The job manager will feature a modified instance of BEExecutionService which will return + * the contents of the resource file jobStateQueryResultFile when execute() is called. + * + * For this feature to work, your JobManager implementation must have a constructor with + * BEExecutionService and JobManagerOptions as parameters. + * + * @param maxTrackingTime Duration used for job tracking filter tests + * @param resourceFileForExecuteResult The file whose content will be returned for execute() calls + * @return + */ + final T createJobManagerWithModifiedExecutionService(String resourceFileForExecuteResult) { + JobManagerOptions parms = JobManagerOptions.create().build() + final Class _class = jobManagerClass + BEExecutionService testExecutionService = [ + execute: { + String s -> new ExecutionResult(true, 0, getResourceFile(_class, resourceFileForExecuteResult).readLines(), null) + } + ] as BEExecutionService + return _newJobManager(testExecutionService, parms) + } + + final T createJobManager() { + JobManagerOptions parms = JobManagerOptions.create().build() + TestExecutionService testExecutionService = new TestExecutionService("test", "test") + return _newJobManager(testExecutionService, parms) + } + + /////////////////////////////////////////////////////////////////////////// + // Unfortunately it is not possible to override Spock test features. + // Or at least, I was not able to find a suitable way. What I propose here + // is, that we just list all necessary tests and hope for the best in the + // future. Maybe the override support comes at some point. Until then, + // we have a list in form of a big comment. + /////////////////////////////////////////////////////////////////////////// + + /* + abstract def "test submitJob"() + + abstract def "test addToListOfStartedJobs"() + + abstract def "test startHeldJobs"() + + abstract def "test killJobs"() + + // Most of the logic for the query(Extended)JobInfo methods is already done in the BatchEuphoriaJobManager spec + // Concentrate on only a couple tests for query... in your JobManager test spec. + abstract def "test queryJobInfoByJob (also tests multiple requests and by ID)"(String id, expectedState) + + abstract def "test queryAllJobInfo"() + + abstract def "test queryExtendedJobInfoByJob (also tests multiple requests and by ID)"(String id, expectedState) + + abstract def "test queryAllExtendedJobInfo"() + + abstract def "test getEnvironmentVariableGlobs"() + + abstract def "test getDefaultForHoldJobsEnabled"() + + abstract def "test isHoldJobsEnabled"() + + abstract def "test getUserEmail"() + + abstract def "test getUserMask"() + + abstract def "test getUserGroup"() + + abstract def "test getUserAccount"() + + abstract def "test executesWithoutJobSystem"() + + abstract def "test convertResourceSet"() + + abstract def "test collectJobIDsFromJobs"() + + abstract def "test extractAndSetJobResultFromExecutionResult"() + + abstract def "test createCommand"() + + abstract def "test parseJobID"() + + abstract def "test parseJobState"() + + abstract def "test executeStartHeldJobs"() + + abstract def "test executeKillJobs"() + */ +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpecSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpecSpec.groovy new file mode 100644 index 00000000..6c449c15 --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/JobManagerImplementationBaseSpecSpec.groovy @@ -0,0 +1,150 @@ +package de.dkfz.roddy.execution.jobs.cluster + +import de.dkfz.roddy.TestExecutionService +import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager +import de.dkfz.roddy.execution.jobs.cluster.pbs.PBSJobManager + +/** + * Test spec for JobManagerImplementationBaseSpecSpec + */ +class JobManagerImplementationBaseSpecSpec extends JobManagerImplementationBaseSpec { + + def "test createJobManagerWithModifiedExecutionService"() { + when: + BatchEuphoriaJobManager jm = createJobManagerWithModifiedExecutionService("createJobManagerWithModifiedExecutionServiceTest.txt") + def expected = ["Line1", "Line2", "Line3"] + def res0 = jm.executionService.execute("blabla").resultLines + def res1 = jm.executionService.execute("something else").resultLines + def res2 = jm.executionService.execute("just a test").resultLines + + then: + res0 == expected + res1 == expected + res2 == expected + } + + def "test createJobManager"() { + when: + BatchEuphoriaJobManager jm = createJobManager() + + then: + jm instanceof PBSJobManager + jm.getExecutionService() instanceof TestExecutionService + } + + def "test submitJob"() { + return null + } + + def "test addToListOfStartedJobs"() { + return null + } + + def "test startHeldJobs"() { + return null + } + + def "test killJobs"() { + return null + } + + def "test queryJobStateByJob"() { + return null + } + + def "test queryJobStateByID"() { + return null + } + + def "test queryJobStatesByJob"() { + return null + } + + def "test queryJobStatesByID"() { + return null + } + + def "test queryAllJobStates"() { + return null + } + + def "test queryJobStates with overdue finish date"() { + return null + } + + def "test queryExtendedJobStatesByJob"() { + return null + } + + def "test queryExtendedJobStatesById"() { + return null + } + + def "test queryExtendedJobStatesById with overdue date"() { + return null + } + + def "test getEnvironmentVariableGlobs"() { + return null + } + + def "test getDefaultForHoldJobsEnabled"() { + return null + } + + def "test isHoldJobsEnabled"() { + return null + } + + def "test getUserEmail"() { + return null + } + + def "test getUserMask"() { + return null + } + + def "test getUserGroup"() { + return null + } + + def "test getUserAccount"() { + return null + } + + def "test executesWithoutJobSystem"() { + return null + } + + def "test convertResourceSet"() { + return null + } + + def "test collectJobIDsFromJobs"() { + return null + } + + def "test extractAndSetJobResultFromExecutionResult"() { + return null + } + + def "test createCommand"() { + return null + } + + def "test parseJobID"() { + return null + } + + def "test parseJobState"() { + return null + } + + def "test executeStartHeldJobs"() { + return null + } + + def "test executeKillJobs"() { + return null + } +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy index 8d29a622..5b51abb1 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/LSFJobManagerSpec.groovy @@ -1,293 +1,368 @@ /* - * Copyright (c) 2017 eilslabs. + * Copyright (c) 2019 German Cancer Research Center (Deutsches Krebsforschungszentrum, DKFZ). * - * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). + * Distributed under the MIT License (license terms are at https://www.github.com/TheRoddyWMS/BatchEuphoria/LICENSE.txt). */ package de.dkfz.roddy.execution.jobs.cluster.lsf import de.dkfz.roddy.TestExecutionService -import de.dkfz.roddy.execution.BEExecutionService -import de.dkfz.roddy.execution.io.ExecutionResult -import de.dkfz.roddy.execution.jobs.BEJobID -import de.dkfz.roddy.execution.jobs.GenericJobInfo -import de.dkfz.roddy.execution.jobs.JobManagerOptions -import de.dkfz.roddy.execution.jobs.JobState +import de.dkfz.roddy.config.ResourceSet +import de.dkfz.roddy.execution.jobs.* +import de.dkfz.roddy.execution.jobs.cluster.JobManagerImplementationBaseSpec import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.BufferValue import groovy.json.JsonSlurper -import spock.lang.Specification +import groovy.transform.CompileStatic +import spock.lang.Ignore -import java.lang.reflect.Method import java.time.Duration +import java.time.LocalDateTime import java.time.ZoneId import java.time.ZonedDateTime import java.time.format.DateTimeFormatter -import java.time.temporal.ChronoUnit - -class LSFJobManagerSpec extends Specification { - - - final static String RAW_JSON_OUTPUT = ''' -{ - "COMMAND":"bjobs", - "JOBS":1, - "RECORDS":[ - { - "JOBID":"22005", - "JOB_NAME":"ls -l", - "STAT":"DONE", - "USER":"otptest", - "QUEUE":"short-dmg", - "JOB_DESCRIPTION":"", - "PROJ_NAME":"default", - "JOB_GROUP":"", - "JOB_PRIORITY":"", - "PIDS":"46782,46796,46798,46915,47458,47643", - "EXIT_CODE":"", - "FROM_HOST":"from-host", - "EXEC_HOST":"exec-host:exec-host", - "SUBMIT_TIME":"Dec 28 19:56", - "START_TIME":"Dec 28 19:56", - "FINISH_TIME":"Dec 28 19:56 L", - "CPU_USED":"00:00:01", - "RUN_TIME":"00:00:01", - "USER_GROUP":"", - "SWAP":"", - "MAX_MEM":"5.2 Gbytes", - "RUNTIMELIMIT":"00:10:00", - "SUB_CWD":"$HOME", - "PEND_REASON":"", - "EXEC_CWD":"\\/some\\/test", - "OUTPUT_FILE":"", - "INPUT_FILE":"", - "EFFECTIVE_RESREQ":"select[type == local] order[r15s:pg] ", - "EXEC_HOME":"\\/some\\/test", - "SLOTS":"1", - "ERROR_FILE":"", - "COMMAND":"ls -l", - "DEPENDENCY":"done(22004)" - } - ] -} -''' - - final static String RAW_JSON_OUTPUT_WITHOUT_LISTS = ''' -{ - "COMMAND":"bjobs", - "JOBS":1, - "RECORDS":[ - { - "JOBID":"22005", - "JOB_NAME":"ls -l", - "STAT":"DONE", - "USER":"otptest", - "QUEUE":"short-dmg", - "JOB_DESCRIPTION":"", - "PROJ_NAME":"default", - "JOB_GROUP":"", - "JOB_PRIORITY":"", - "PIDS":"51904", - "EXIT_CODE":"1", - "FROM_HOST":"from-host", - "EXEC_HOST":"exec-host", - "SUBMIT_TIME":"Dec 28 19:56", - "START_TIME":"Dec 28 19:56", - "FINISH_TIME":"Dec 28 19:56 L", - "CPU_USED":"00:00:01", - "RUN_TIME":"00:00:01", - "USER_GROUP":"", - "SWAP":"0 Mbytes", - "MAX_MEM":"522 MBytes", - "RUNTIMELIMIT":"00:10:00", - "SUB_CWD":"$HOME", - "PEND_REASON":"Job dependency condition not satisfied;", - "EXEC_CWD":"\\/some/\\/test", - "OUTPUT_FILE":"\\/sequencing\\/whole_genome_sequencing\\/coveragePlotSingle.o30060", - "INPUT_FILE":"", - "EFFECTIVE_RESREQ":"select[type == local] order[r15s:pg] ", - "EXEC_HOME":"\\/some\\/test", - "SLOTS":"1", - "ERROR_FILE":"", - "COMMAND":"ls -l", - "DEPENDENCY":"done(22004)" - } - ] -} -''' +import java.time.format.DateTimeParseException - void "queryJobInfo, bjobs JSON output with lists "() { +class LSFJobManagerSpec extends JobManagerImplementationBaseSpec { - given: - def parms = JobManagerOptions.create().build() - TestExecutionService testExecutionService = new TestExecutionService("test", "test") - LSFJobManager jm = new LSFJobManager(testExecutionService, parms) - Method method = LSFJobManager.class.getDeclaredMethod("queryJobInfo", Map) - method.setAccessible(true) - Object parsedJson = new JsonSlurper().parseText(RAW_JSON_OUTPUT) - List records = (List) parsedJson.getAt("RECORDS") + ////////////////////////////////////// + // Time zone helpers and tests. + ////////////////////////////////////// + + @CompileStatic + String zonedDateTimeToString(ZonedDateTime date) { + DateTimeFormatter.ofPattern('MMM ppd HH:mm').withLocale(Locale.ENGLISH).format(date) + } + + @CompileStatic + String localDateTimeToLSFString(LocalDateTime date) { + // Important here is, that LSF puts " L" or other status codes at the end of some dates, e.g. FINISH_DATE + // Thus said, " L" does not apply for all dates reported by LSF! + DateTimeFormatter.ofPattern('MMM ppd HH:mm').withLocale(Locale.ENGLISH).format(date) + " L" + } + + void testZonedDateTimeToString(input, expected) { + expect: + zonedDateTimeToString(input) == expected + + where: + input | expected + ZonedDateTime.of(2000, 1, 1, 10, 21, 0, 0, ZoneId.systemDefault()) | "Jan 1 10:21" + ZonedDateTime.of(2000, 5, 7, 10, 21, 0, 0, ZoneId.systemDefault()) | "May 7 10:21" + } + + void testLocalDateTimeToLSFString(input, expected) { + expect: + localDateTimeToLSFString(input) == expected + where: + input | expected + LocalDateTime.of(2000, 1, 1, 10, 21) | "Jan 1 10:21 L" + LocalDateTime.of(2000, 5, 7, 10, 21) | "May 7 10:21 L" + } + + ////////////////////////////////////// + // Class tests + ////////////////////////////////////// + + def "test parseTime"(String time, ZonedDateTime expected) { + expect: + createJobManager().parseTime(time) == expected + + where: + time | expected + "Jan 1 10:21" | ZonedDateTime.of(LocalDateTime.now().year, 1, 1, 10, 21, 0, 0, ZoneId.systemDefault()) + "Jan 2 10:21 L" | ZonedDateTime.of(LocalDateTime.now().year, 1, 2, 10, 21, 0, 0, ZoneId.systemDefault()) + "May 3 10:21 2016" | ZonedDateTime.of(2016, 5, 3, 10, 21, 0, 0, ZoneId.systemDefault()) + "May 4 10:21 2016 L" | ZonedDateTime.of(2016, 5, 4, 10, 21, 0, 0, ZoneId.systemDefault()) + } + + def "test parseTime with malformatted string"(String time, expected) { when: - GenericJobInfo jobInfo = method.invoke(jm, records.get(0)) + createJobManager().parseTime(time) then: - jobInfo != null - jobInfo.tool == new File("ls -l") + thrown(expected) + + where: + time | expected + null | DateTimeParseException + "" | DateTimeParseException + "Hfjfj" | DateTimeParseException + "Mai 1 10:21" | DateTimeParseException + } + + def "test stripAwayStatusInfo"() { + expect: + LSFJobManager.stripAwayStatusInfo(time) == expected + + where: + time | expected + null | null + "" | "" + "Jan 1 10:21" | "Jan 1 10:21" + "Jan 1 10:21 1" | "Jan 1 10:21" + "Jan 1 10:21 E" | "Jan 1 10:21" + "Jan 1 10:21 F" | "Jan 1 10:21" + } + + def "test submitJob"() { + return null + } + + def "test addToListOfStartedJobs"() { + return null } - void "queryJobInfo, bjobs JSON output without lists "() { + def "test startHeldJobs"() { + return null + } + + def "test killJobs"() { + return null + } + + def "test queryJobStateByJob"() { + return null + } + def "test queryJobStateByID"() { given: - def parms = JobManagerOptions.create().build() - TestExecutionService testExecutionService = new TestExecutionService("test", "test") - LSFJobManager jm = new LSFJobManager(testExecutionService, parms) - Method method = LSFJobManager.class.getDeclaredMethod("queryJobInfo", Map) - method.setAccessible(true) - Object parsedJson = new JsonSlurper().parseText(RAW_JSON_OUTPUT_WITHOUT_LISTS) - List records = (List) parsedJson.getAt("RECORDS") + def jm = createJobManagerWithModifiedExecutionService("queryJobStateByIdTest.json") when: - GenericJobInfo jobInfo = method.invoke(jm, records.get(0)) + def result = jm.queryJobInfoByID(testJobID) then: - jobInfo != null - jobInfo.tool == new File("ls -l") + result.jobState == JobState.COMPLETED_SUCCESSFUL } - void "queryJobInfo, bjobs JSON output empty "() { + def "test queryJobStatesByJob"() { + return null + } + def "test queryJobStatesByID"() { given: - String emptyRawJsonOutput= ''' - { - "COMMAND":"bjobs", - "JOBS":1, - "RECORDS":[ - { - "JOBID":"22005", - } - ] - } - ''' - def parms = JobManagerOptions.create().build() - TestExecutionService testExecutionService = new TestExecutionService("test", "test") - LSFJobManager jm = new LSFJobManager(testExecutionService, parms) - Method method = LSFJobManager.class.getDeclaredMethod("queryJobInfo", Map) - method.setAccessible(true) - Object parsedJson = new JsonSlurper().parseText(emptyRawJsonOutput) - List records = (List) parsedJson.getAt("RECORDS") + def jm = createJobManagerWithModifiedExecutionService("queryJobStateByIdTest.json") + when: - GenericJobInfo jobInfo = method.invoke(jm, records.get(0)) + def result = jm.queryJobInfoByID([testJobID]) then: - jobInfo.jobID.toString() == "22005" + result.size() == 1 + result[testJobID].jobState == JobState.COMPLETED_SUCCESSFUL } - String dateToString(ZonedDateTime date) { - DateTimeFormatter.ofPattern('MMM ppd HH:mm').format(date) + def "test queryAllJobStates"() { + return null } - void "test parseTime"() { + def "test queryExtendedJobStatesByJob"() { given: - JobManagerOptions parms = JobManagerOptions.create().build() - BEExecutionService testExecutionService = [ - execute: { String s -> new ExecutionResult(true, 0, RAW_JSON_OUTPUT.split("\n") as List, null) } - ] as BEExecutionService - LSFJobManager manager = new LSFJobManager(testExecutionService, parms) + def jobManager = createJobManagerWithModifiedExecutionService("queryExtendedJobStateByIdTest.json") + BEJob job = new BEJob(testJobID, jobManager) when: - ZonedDateTime refTime = ZonedDateTime.now() - ZonedDateTime earlierTime = refTime.minusDays(1) - ZonedDateTime laterTime = refTime.plusDays(1) - ZonedDateTime laterLastYear = laterTime.minusYears(1) + def result = jobManager.queryExtendedJobInfoByJob([job]) then: - manager.parseTime(dateToString(earlierTime)).truncatedTo(ChronoUnit.MINUTES).equals(earlierTime.truncatedTo(ChronoUnit.MINUTES)) - manager.parseTime(dateToString(laterTime)).truncatedTo(ChronoUnit.MINUTES).equals(laterLastYear.truncatedTo(ChronoUnit.MINUTES)) + result.size() == 1 + result[job] } - void "test queryExtendedJobStateById"() { + def "test queryExtendedJobStatesById"() { + given: - JobManagerOptions parms = JobManagerOptions.create().build() - BEExecutionService testExecutionService = [ - execute: { String s -> new ExecutionResult(true, 0, RAW_JSON_OUTPUT.split("\n") as List, null) } - ] as BEExecutionService - LSFJobManager manager = new LSFJobManager(testExecutionService, parms) + def jobManager = createJobManagerWithModifiedExecutionService("queryExtendedJobStateByIdTest.json") + + ExtendedJobInfo expected = new ExtendedJobInfo(testJobID, JobState.COMPLETED_SUCCESSFUL) + expected.requestedResources = new ResourceSet(null, null, null, null, Duration.ofMinutes(10), null, "short-dmg", null) + expected.usedResources = new ResourceSet(null, new BufferValue(5452595, BufferUnit.k), null, 1, Duration.ofSeconds(1), null, "short-dmg", null) + expected.jobName = "ls -l" + expected.command = "/home/testuser/somescript.sh" + expected.submissionHost = "from-host" + expected.executionHosts = ["exec-host"] + expected.user = "otptest" + expected.rawResourceRequest = 'select[type == local] order[r15s:pg] ' + expected.parentJobIDs = ["22004"] + expected.execHome = "/some/test" + expected.processesInJob = ["46782", "46796", "46798", "46915", "47458", "47643"] + expected.exitCode = 0 + expected.execCwd = "/some/test" + expected.cwd = '$HOME' + expected.projectName = "default" + expected.cpuTime = Duration.ofSeconds(1) + expected.runTime = Duration.ofSeconds(1) when: - Map result = manager.queryExtendedJobStateById([new BEJobID("22005")]) + Map result = jobManager.queryExtendedJobInfoByID([testJobID]) + ExtendedJobInfo jobInfo = result[testJobID] then: result.size() == 1 - GenericJobInfo jobInfo = result.get(new BEJobID("22005")) - jobInfo - jobInfo.askedResources.size == null - jobInfo.askedResources.mem == null - jobInfo.askedResources.cores == null - jobInfo.askedResources.nodes == null - jobInfo.askedResources.walltime == Duration.ofMinutes(10) - jobInfo.askedResources.storage == null - jobInfo.askedResources.queue == "short-dmg" - jobInfo.askedResources.nthreads == null - jobInfo.askedResources.swap == null - - jobInfo.usedResources.size == null - jobInfo.usedResources.mem == new BufferValue(5452595, BufferUnit.k) - jobInfo.usedResources.cores == null - jobInfo.usedResources.nodes == 1 - jobInfo.usedResources.walltime == Duration.ofSeconds(1) - jobInfo.usedResources.storage == null - jobInfo.usedResources.queue == "short-dmg" - jobInfo.usedResources.nthreads == null - jobInfo.usedResources.swap == null - - jobInfo.jobName == "ls -l" - jobInfo.tool == new File("ls -l") - jobInfo.jobID == new BEJobID("22005") - - // The year-parsing/inferrence is checked in another test. Here just take the parsed value. - jobInfo.submitTime == ZonedDateTime.of(jobInfo.submitTime.year, 12, 28, 19, 56, 0, 0, ZoneId.systemDefault()) - jobInfo.eligibleTime == null - jobInfo.startTime == ZonedDateTime.of(jobInfo.submitTime.year, 12, 28, 19, 56, 0, 0, ZoneId.systemDefault()) - jobInfo.endTime == ZonedDateTime.of(jobInfo.submitTime.year, 12, 28, 19, 56, 0, 0, ZoneId.systemDefault()) - jobInfo.executionHosts == ["exec-host", "exec-host"] - jobInfo.submissionHost == "from-host" - jobInfo.priority == null - jobInfo.logFile == null - jobInfo.errorLogFile == null - jobInfo.inputFile == null - jobInfo.user == "otptest" - jobInfo.userGroup == null - jobInfo.resourceReq == 'select[type == local] order[r15s:pg] ' - jobInfo.startCount == null - jobInfo.account == null - jobInfo.server == null - jobInfo.umask == null - jobInfo.parameters == null - jobInfo.parentJobIDs == ["22004"] - jobInfo.otherSettings == null - jobInfo.jobState == JobState.COMPLETED_SUCCESSFUL - jobInfo.userTime == null - jobInfo.systemTime == null - jobInfo.pendReason == null - jobInfo.execHome == "/some/test" - jobInfo.execUserName == null - jobInfo.pidStr == ["46782", "46796", "46798", "46915", "47458", "47643"] - jobInfo.pgidStr == null - jobInfo.exitCode == 0 - jobInfo.jobGroup == null - jobInfo.description == null - jobInfo.execCwd == "/some/test" - jobInfo.askedHostsStr == null - jobInfo.cwd == '$HOME' - jobInfo.projectName == "default" - jobInfo.cpuTime == Duration.ofSeconds(1) - jobInfo.runTime == Duration.ofSeconds(1) - jobInfo.timeUserSuspState == null - jobInfo.timePendState == null - jobInfo.timePendSuspState == null - jobInfo.timeSystemSuspState == null - jobInfo.timeUnknownState == null - jobInfo.timeOfCalculation == null + + when: + // Small hack to get the right year. LSF won't always report years in its various date fields. + // This needs to be configured by the LSF cluster administrators. + ZonedDateTime testTime = ZonedDateTime.of(jobInfo.submitTime.year, 12, 28, 19, 56, 0, 0, ZoneId.systemDefault()) + expected.submitTime = testTime + expected.startTime = testTime + expected.endTime = testTime + + then: + jobInfo == expected + } + + def "test convertBJobsResultLinesToResultMap"() { + given: + def jsonFile = getResourceFile(LSFJobManagerSpec,"convertBJobsResultLinesToResultMapTest.json") + def json = jsonFile.text + + when: + Map> map = LSFJobManager.convertBJobsJsonOutputToResultMap(json) + def jobId = map.keySet()[0] + + then: + map.size() == 6 + jobId.id == "487641" + map[jobId]["JOBID"] == "487641" + map[jobId]["JOB_NAME"] == "RoddyTest_testScript" + map[jobId]["STAT"] == "EXIT" + map[jobId]["FINISH_TIME"] == "Jan 7 09:59 L" + } + + /** + * This test should not be run by default, as it runs quite a while (on purpose). + * Reenable it, if you run into memory leaks. + */ + @Ignore + def "test massive ConvertBJobsResultLinesToResultMap"(def _entries, def value) { + when: + int entries = _entries[0] + String template1 = getResourceFile("bjobsJobTemplatePart1.txt").text + String template2 = getResourceFile("bjobsJobTemplatePart2.txt").text + List lines = new LinkedList<>() + + lines << "{" + lines << ' "COMMAND":"bjobs",' + lines << ' "JOBS":"' + entries + '",' + lines << ' "RECORDS":[' + + int maximum = 1000000 + entries - 1 + for (int i = 1000000; i <= maximum; i++) { + lines += template1.readLines() + lines << ' "JOBID":"' + i + '",' + lines << ' "JOB_NAME":"r181217_003553288_Strand_T_150_aTestJob",' + lines += template2.readLines() + if (i < maximum) + lines << " ," + } + lines << " ]" + lines << "}" + println("Entries ${entries}") + def result = LSFJobManager.convertBJobsJsonOutputToResultMap(lines.join("\n")) + + then: + result.size() == entries + + where: + _entries | value + [1] | true + [10] | true + [100] | true + [1000] | true + [2000] | true + [4000] | true + [8000] | true + [16000] | true + } + + def "test convertJobDetailsMapToGenericJobInfoObject"() { + given: + def parms = JobManagerOptions.create().build() + TestExecutionService testExecutionService = new TestExecutionService("test", "test") + LSFJobManager jm = new LSFJobManager(testExecutionService, parms) + + Object parsedJson = new JsonSlurper().parseText(getResourceFile(LSFJobManagerSpec, resourceFile).text) + List records = (List) parsedJson.getAt("RECORDS") + + when: + ExtendedJobInfo jobInfo = jm.convertJobDetailsMapToGenericJobInfoObject(records.get(0)) + + then: + jobInfo != null + jobInfo.jobID.toString() == "22005" + jobInfo.command == expectedCommand + + where: + resourceFile | expectedJobId | expectedCommand + "queryExtendedJobStateByIdTest.json" | "22005" | "/home/testuser/somescript.sh" + "queryExtendedJobStateByIdWithoutListsTest.json" | "22005" | "/home/testuser/somescript.sh" + "queryExtendedJobStateByIdEmptyTest.json" | "22005" | null + } + + def "test getEnvironmentVariableGlobs"() { + return null + } + + def "test getDefaultForHoldJobsEnabled"() { + return null + } + + def "test isHoldJobsEnabled"() { + return null + } + + def "test getUserEmail"() { + return null + } + + def "test getUserMask"() { + return null + } + + def "test getUserGroup"() { + return null + } + + def "test getUserAccount"() { + return null + } + + def "test executesWithoutJobSystem"() { + return null + } + + def "test convertResourceSet"() { + return null + } + + def "test collectJobIDsFromJobs"() { + return null + } + + def "test extractAndSetJobResultFromExecutionResult"() { + return null + } + + def "test createCommand"() { + return null + } + + def "test parseJobID"() { + return null + } + + def "test parseJobState"() { + return null + } + + def "test executeStartHeldJobs"() { + return null + } + + def "test executeKillJobs"() { + return null } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManagerSpec.groovy new file mode 100644 index 00000000..acd1f2c0 --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManagerSpec.groovy @@ -0,0 +1,121 @@ +package de.dkfz.roddy.execution.jobs.cluster.lsf.rest + +import de.dkfz.roddy.execution.jobs.cluster.JobManagerImplementationBaseSpec + +class LSFRestJobManagerSpec extends JobManagerImplementationBaseSpec { + def "test submitJob"() { + return null + } + + def "test addToListOfStartedJobs"() { + return null + } + + def "test startHeldJobs"() { + return null + } + + def "test killJobs"() { + return null + } + + def "test queryJobStateByJob"() { + return null + } + + def "test queryJobStateByID"() { + return null + } + + def "test queryJobStatesByJob"() { + return null + } + + def "test queryJobStatesByID"() { + return null + } + + def "test queryAllJobStates"() { + return null + } + + def "test queryJobStates with overdue finish date"() { + return null + } + + def "test queryExtendedJobStatesByJob"() { + return null + } + + def "test queryExtendedJobStatesByID"() { + return null + } + + def "test queryExtendedJobStatesByID with overdue date"() { + return null + } + + def "test getEnvironmentVariableGlobs"() { + return null + } + + def "test getDefaultForHoldJobsEnabled"() { + return null + } + + def "test isHoldJobsEnabled"() { + return null + } + + def "test getUserEmail"() { + return null + } + + def "test getUserMask"() { + return null + } + + def "test getUserGroup"() { + return null + } + + def "test getUserAccount"() { + return null + } + + def "test executesWithoutJobSystem"() { + return null + } + + def "test convertResourceSet"() { + return null + } + + def "test collectJobIDsFromJobs"() { + return null + } + + def "test extractAndSetJobResultFromExecutionResult"() { + return null + } + + def "test createCommand"() { + return null + } + + def "test parseJobID"() { + return null + } + + def "test parseJobState"() { + return null + } + + def "test executeStartHeldJobs"() { + return null + } + + def "test executeKillJobs"() { + return null + } +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParserTest.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParserTest.groovy index cfd8308e..30ae01c5 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParserTest.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommandParserTest.groovy @@ -6,10 +6,8 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs -import de.dkfz.roddy.execution.BEExecutionService -import de.dkfz.roddy.execution.io.ExecutionResult +import de.dkfz.roddy.TestExecutionService import de.dkfz.roddy.execution.jobs.BatchEuphoriaJobManager -import de.dkfz.roddy.execution.jobs.Command import de.dkfz.roddy.execution.jobs.JobManagerOptions import groovy.transform.CompileStatic import org.junit.BeforeClass @@ -27,42 +25,7 @@ class PBSCommandParserTest { @BeforeClass static void setup() { - testJobManager = new PBSJobManager(new BEExecutionService() { - @Override - ExecutionResult execute(Command command) { - return null - } - - @Override - ExecutionResult execute(Command command, boolean waitFor) { - return null - } - - @Override - ExecutionResult execute(String command) { - return null - } - - @Override - ExecutionResult execute(String command, boolean waitFor) { - return null - } - - @Override - ExecutionResult execute(String command, boolean waitForIncompatibleClassChangeError, OutputStream outputStream) { - return null - } - - @Override - boolean isAvailable() { - return false - } - - @Override - File queryWorkingDirectory() { - return null - } - }, JobManagerOptions.create().setCreateDaemon(false).build()) + testJobManager = new PBSJobManager(new TestExecutionService("testuser", "localhost"), JobManagerOptions.create().build()) } @Test @@ -86,18 +49,16 @@ class PBSCommandParserTest { assert commandParser.memory == "16384" assert commandParser.nodes == "1" assert commandParser.cores == "8" - assert commandParser.parameters == ["PARAMETER_FILE":"/data/michael/temp/roddyLocalTest/testproject/rpp/A100/roddyExecutionStore/exec_170402_171935425_heinold_indelCalling/r170402_171935425_A100_indelCalling_1.parameters"] + assert commandParser.parameters == ["PARAMETER_FILE": "/data/michael/temp/roddyLocalTest/testproject/rpp/A100/roddyExecutionStore/exec_170402_171935425_heinold_indelCalling/r170402_171935425_A100_indelCalling_1.parameters"] assert commandParser.dependencies == ["120015"] def gji = testJobManager.parseGenericJobInfo(commandString) assert gji.jobName == "r170402_171935425_A100_indelCalling" - assert gji.askedResources.getCores() == 8 - assert gji.askedResources.getNodes() == 1 - assert gji.askedResources.getWalltime() == Duration.ofDays(2).plusHours(2) - assert gji.askedResources.getMem().toLong() == 16384 - assert gji.parameters == ["PARAMETER_FILE":"/data/michael/temp/roddyLocalTest/testproject/rpp/A100/roddyExecutionStore/exec_170402_171935425_heinold_indelCalling/r170402_171935425_A100_indelCalling_1.parameters"] + assert gji.requestedResources.getCores() == 8 + assert gji.requestedResources.getNodes() == 1 + assert gji.requestedResources.getWalltime() == Duration.ofDays(2).plusHours(2) + assert gji.requestedResources.getMem().toLong() == 16384 + assert gji.parameters == ["PARAMETER_FILE": "/data/michael/temp/roddyLocalTest/testproject/rpp/A100/roddyExecutionStore/exec_170402_171935425_heinold_indelCalling/r170402_171935425_A100_indelCalling_1.parameters"] assert gji.parentJobIDs == ["120015"] } - - } \ No newline at end of file diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManagerSpec.groovy index e4ae5e8c..b9b069af 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManagerSpec.groovy @@ -6,326 +6,245 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs -import de.dkfz.roddy.TestExecutionService + +import de.dkfz.roddy.config.ResourceSet +import de.dkfz.roddy.config.ResourceSetSize import de.dkfz.roddy.execution.jobs.BEJobID -import de.dkfz.roddy.execution.jobs.GenericJobInfo -import de.dkfz.roddy.execution.jobs.JobManagerOptions +import de.dkfz.roddy.execution.jobs.ExtendedJobInfo import de.dkfz.roddy.execution.jobs.JobState -import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager +import de.dkfz.roddy.execution.jobs.cluster.JobManagerImplementationBaseSpec import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.BufferValue -import spock.lang.Specification +import de.dkfz.roddy.tools.TimeUnit -import java.lang.reflect.InvocationTargetException -import java.lang.reflect.Method import java.time.Duration +import java.time.ZoneId import java.time.ZoneOffset import java.time.ZonedDateTime -class PBSJobManagerSpec extends Specification { - PBSJobManager manager - - String rawXmlExample = ''' - - - 15020227.testServer - workflow_test - testOwner - H - fast - afterok:15624736.testServer - testServer - u - 1499432861 - logging_root_path/clusterLog/2017-07-07/workflow_test - u - oe - n - a - 1499432861 - 1514572025 - logging_root_path/clusterLog/2017-07-07/workflow_test.o15020227 - 0 - 1499432861 - True - 71497 - - 5120mb - 1 - 1:ppn=1 - 00:20:00 - - - 04:50:19 - 0 - 2449672kb - 2524268kb - 03:07:41 - - PBS_O_QUEUE=default,PBS_O_HOME=/home/test,PBS_O_LOGNAME=test,PBS_O_SHELL=/bin/bash,PBS_O_LANG=en_GB.UTF-8 - test - testGroup - E - -N workflow_test -h -o /clusterLog/2017-07-07 -j oe -W umask=027 -l mem=5120M -l walltime=00:00:20:00 -l nodes=1:ppn=1 - 23 - False - 0 - sub.testServer - 1 - 1 - - - ''' - - String outputQueued = '''4499334.pbsserverr180328_183957634_pid_4_starAlignmentotp-data@subm.example.comQotppbsserveru1522255162subm:/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment.o$PBS_JOBIDnoena1522255162subm:/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment.o$PBS_JOBID01522255162True168:00:0037888mb1511:ppn=405:00:00PBS_O_QUEUE=otp,PBS_O_HOME=/home/otp-data,PBS_O_LOGNAME=otp-data,PBS_O_PATH=/usr/local/bin:/usr/bin:/bin:/usr/games,PBS_O_MAIL=/var/mail/otp-data,PBS_O_SHELL=/bin/bash,PBS_O_LANG=en_US.utf8,TOOL_ID=starAlignment,PARAMETER_FILE=/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment_3.parameters,CONFIG_FILE=/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment_3.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme,PBS_O_WORKDIR=/home/otp-data,PBS_O_HOST=subm.example.com,PBS_O_SERVER=pbsserver1522255162-v TOOL_ID=starAlignment,PARAMETER_FILE=/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment_3.parameters,CONFIG_FILE=/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment_3.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N r180328_183957634_pid_4_starAlignment -h -w /home/otp-data -j oe -o /net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment.o$PBS_JOBID -l mem=37888M -l walltime=00:05:00:00 -l nodes=1:ppn=4 -q otp /net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/analysisTools/roddyTools/wrapInScript.shFalse0subm.example.com/home/otp-data - -''' - - String outputFinished = '''4564045.pbsserverr180405_163953553_stds_snvJoinVcfFilesotp-data@subm.example.com00:00:006064kb454232kb00:00:06Cotppbsserveru1522939158afterok:6059780.pbsserver@pbsserver:6059781.pbsserver@pbsserver,beforeok:6059788.pbsserver@pbsserversubm:/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o$PBS_JOBIDdenbi5-int/0+denbi5-int/1+denbi5-int/215003noena1522939194subm:/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o$PBS_JOBID01522939158True168:00:001024mb1511:ppn=304:00:0014506PBS_O_QUEUE=otp,PBS_O_HOME=/home/otp-data,PBS_O_LOGNAME=otp-data,PBS_O_PATH=/usr/local/bin:/usr/bin:/bin:/usr/games,PBS_O_MAIL=/var/mail/otp-data,PBS_O_SHELL=/bin/bash,PBS_O_LANG=en_US.utf8,TOOL_ID=snvJoinVcfFiles,PARAMETER_FILE=/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles_9.parameters,CONFIG_FILE=/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles_9.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme,PBS_O_WORKDIR=/home/otp-data,PBS_O_HOST=subm.example.com,PBS_O_SERVER=pbsserverPost job file processing error; job 4564045.pbsserver on host denbi5-int/1 - -Unable to copy file /var/spool/torque/spool/4564045.pbsserver.OU to otp-data@subm:/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o4564045.pbsserver -*** error from copy -Host key verification failed. -lost connection -*** end error output -Output retained on that host in: /var/spool/torque/undelivered/4564045.pbsserver.OU15229391820-v TOOL_ID=snvJoinVcfFiles,PARAMETER_FILE=/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles_9.parameters,CONFIG_FILE=/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles_9.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N r180405_163953553_stds_snvJoinVcfFiles -h -w /home/otp-data -j oe -o /net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o$PBS_JOBID -l mem=1024M -l walltime=00:04:00:00 -l nodes=1:ppn=3 -q otp -W depend=afterok:6059780:6059781 /net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/analysisTools/roddyTools/wrapInScript.sh15229391831False1522939194010.431524subm.example.com/home/otp-data -''' - - void setup() { - JobManagerOptions parms = JobManagerOptions.create().build() - TestExecutionService testExecutionService = new TestExecutionService("test", "test") - manager = new PBSJobManager(testExecutionService, parms) - } - - void "test processQstatOutputFromXML with queued job"() { +class PBSJobManagerSpec extends JobManagerImplementationBaseSpec { + + /** + * convertResourceSet is implemented in ClusterJobManager but calls methods in the actual JobManager instance + */ + void "test convertResourceSet"(String expected, Integer mem, Integer cores, Integer nodes, String walltime) { + given: + BufferValue _mem = mem ? new BufferValue(mem, BufferUnit.m) : null + TimeUnit _walltime = walltime ? new TimeUnit(walltime) : null + ResourceSet test = new ResourceSet(ResourceSetSize.l, _mem, cores, nodes, (TimeUnit) _walltime, null, null, null) + + expect: + createJobManager().convertResourceSet(null, test).processingCommandString == expected + + where: + expected | mem | cores | nodes | walltime + "-l mem=1024M -l walltime=00:01:00:00 -l nodes=1:ppn=2" | 1024 | 2 | 1 | "h" + "-l walltime=00:01:00:00 -l nodes=1:ppn=1" | null | null | 1 | "h" + "-l mem=1024M -l nodes=1:ppn=2" | 1024 | 2 | null | null + } + + ExtendedJobInfo getGenericJobInfoObjectForTests() { + ExtendedJobInfo expected = new ExtendedJobInfo(testJobID, JobState.QUEUED) + expected.requestedResources = new ResourceSet(null, new BufferValue(37888, BufferUnit.M), 4, 1, Duration.ofHours(5), null, "debug", null) + expected.usedResources = new ResourceSet(null, null, null, null, (Duration) null, null, "debug", null) + expected.jobName = "ATestJob" + expected.user = "testuser" + expected.userGroup = "" + expected.account = "" + expected.umask = "" + expected.submissionHost = "subm-host.example.com" + expected.executionHosts = [] + expected.submitTime = ZonedDateTime.of(2018, 03, 28, 18, 39, 22, 0, ZoneOffset.systemDefault()) + expected.eligibleTime = ZonedDateTime.of(2018, 03, 28, 18, 39, 22, 0, ZoneOffset.systemDefault()) + expected.priority = "0" + expected.logFile = new File("/somefolder/job.o22005.pbsserver") + expected.errorLogFile = new File("/somefolder/job.o22005.pbsserver") + expected.rawResourceRequest = '-v TOOL_ID=starAlignment,PARAMETER_FILE=/somefolder/exec/ATestJob.parameters,CONFIG_FILE=/somefolder/exec/ATestJob.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N ATestJob -h -w /home/otp-data -j oe -o /somefolder/job.o$PBS_JOBID -l mem=37888M -l walltime=00:05:00:00 -l nodes=1:ppn=4 -q otp /somefolder/wrapInScript.sh' + expected.server = "pbsserver" + expected.parentJobIDs = [] + expected + } + + def "test submitJob"() { + return null + } + + def "test addToListOfStartedJobs"() { + return null + } + + def "test startHeldJobs"() { + return null + } + + def "test killJobs"() { + return null + } + + def "test queryJobStateByJob"() { + return null + } + + def "test queryJobStateByID"() { + return null + } + + def "test queryJobInfByJob"() { + return null + } + + def "test queryJobInfByID"() { + return null + } + + def "test queryAllJobInf"() { + return null + } + + def "test queryExtendedJobInfByJob"() { + return null + } + + def "test queryExtendedJobInfByID"() { + return null + } + + void "test queryExtendedJobInfByID with queued job"() { + + given: + + def jobManager = createJobManagerWithModifiedExecutionService("processExtendedQstatXMLOutputWithQueuedJob.xml") + + ExtendedJobInfo expected = getGenericJobInfoObjectForTests() + when: - Map result = manager.processQstatOutputFromXML(outputQueued) + Map result = jobManager.queryExtendedJobInfo([testJobID]) + ExtendedJobInfo jobInfo = result.get(testJobID) then: result.size() == 1 - GenericJobInfo jobInfo = result.get(new BEJobID("4499334")) - jobInfo - jobInfo.askedResources.size == null - jobInfo.askedResources.mem == new BufferValue(37888, BufferUnit.M) - jobInfo.askedResources.cores == 4 - jobInfo.askedResources.nodes == 1 - jobInfo.askedResources.walltime == Duration.ofHours(5) - jobInfo.askedResources.storage == null - jobInfo.askedResources.queue == "otp" - jobInfo.askedResources.nthreads == null - jobInfo.askedResources.swap == null - - jobInfo.usedResources.size == null - jobInfo.usedResources.mem == null - jobInfo.usedResources.cores == null - jobInfo.usedResources.nodes == null - jobInfo.usedResources.walltime == null - jobInfo.usedResources.storage == null - jobInfo.usedResources.queue == "otp" - jobInfo.usedResources.nthreads == null - jobInfo.usedResources.swap == null - - jobInfo.jobName == "r180328_183957634_pid_4_starAlignment" - jobInfo.tool == null - jobInfo.jobID == new BEJobID("4499334") - jobInfo.submitTime.isEqual ZonedDateTime.of(2018, 03, 28, 16, 39, 22, 0, ZoneOffset.UTC) - jobInfo.eligibleTime.isEqual ZonedDateTime.of(2018, 03, 28, 16, 39, 22, 0, ZoneOffset.UTC) - jobInfo.startTime == null - jobInfo.endTime == null - jobInfo.executionHosts == null - jobInfo.submissionHost == "subm.example.com" - jobInfo.priority == "0" - jobInfo.logFile == new File("/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment.o4499334.pbsserver") - jobInfo.errorLogFile == new File("/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment.o4499334.pbsserver") - jobInfo.inputFile == null - jobInfo.user == null - jobInfo.userGroup == null - jobInfo.resourceReq == '-v TOOL_ID=starAlignment,PARAMETER_FILE=/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment_3.parameters,CONFIG_FILE=/net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment_3.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N r180328_183957634_pid_4_starAlignment -h -w /home/otp-data -j oe -o /net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/r180328_183957634_pid_4_starAlignment.o$PBS_JOBID -l mem=37888M -l walltime=00:05:00:00 -l nodes=1:ppn=4 -q otp /net/isilon/otp/workflow-tests/tmp/RnaPairedAlignmentWorkflow-otp-web-2018-03-28-18-36-57-153+0200-mFrFsCEXnGMdFEGC/root_path/projectDirName_20/sequencing/rna_sequencing/view-by-pid/pid_4/control/paired/merged-alignment/.merging_0/roddyExecutionStore/exec_180328_183957634_otp-data_RNA/analysisTools/roddyTools/wrapInScript.sh' - jobInfo.startCount == null - jobInfo.account == null - jobInfo.server == "pbsserver" - jobInfo.umask == null - jobInfo.parameters == null - jobInfo.parentJobIDs == [] - jobInfo.otherSettings == null - jobInfo.jobState == JobState.QUEUED - jobInfo.userTime == null - jobInfo.systemTime == null - jobInfo.pendReason == null - jobInfo.execHome == null - jobInfo.execUserName == null - jobInfo.pidStr == null - jobInfo.pgidStr == null - jobInfo.exitCode == null - jobInfo.jobGroup == null - jobInfo.description == null - jobInfo.execCwd == null //??? - jobInfo.askedHostsStr == null - jobInfo.cwd == null - jobInfo.projectName == null - jobInfo.cpuTime == null - jobInfo.runTime == null - jobInfo.timeUserSuspState == null - jobInfo.timePendState == null - jobInfo.timePendSuspState == null - jobInfo.timeSystemSuspState == null - jobInfo.timeUnknownState == null - jobInfo.timeOfCalculation == null - } - - void "test processQstatOutputFromXML with finished job, with newline"() { + jobInfo.requestedResources == expected.requestedResources + jobInfo.usedResources == expected.usedResources + jobInfo.submitTime == expected.submitTime + jobInfo == expected + } + + void "test queryExtendedJobInfByID with finished job"() { + + given: + + def jobManager = createJobManagerWithModifiedExecutionService("processExtendedQstatXMLOutputWithFinishedJob.xml") + + ExtendedJobInfo expected = getGenericJobInfoObjectForTests() + + expected.usedResources = new ResourceSet(null, new BufferValue(6064, BufferUnit.k), null, null, Duration.ofSeconds(6), null, "otp", null) + expected.startTime = ZonedDateTime.of(2018, 4, 5, 16, 39, 43, 0, ZoneId.systemDefault()) + expected.endTime = ZonedDateTime.of(2018, 4, 5, 16, 39, 54, 0, ZoneId.systemDefault()) + expected.executionHosts = ["exec-host"] + expected.startCount = 1 + expected.parentJobIDs = ["6059780", "6059781"] + expected.jobState = JobState.COMPLETED_UNKNOWN + expected.exitCode = 0 + expected.cpuTime = Duration.ZERO + expected.runTime = Duration.ofSeconds(10) + when: - Map result = manager.processQstatOutputFromXML(outputFinished) + Map result = jobManager.queryExtendedJobInfo([testJobID]) + ExtendedJobInfo jobInfo = result.get(testJobID) then: result.size() == 1 - GenericJobInfo jobInfo = result.get(new BEJobID("4564045")) - jobInfo - jobInfo.askedResources.size == null - jobInfo.askedResources.mem == new BufferValue(1024, BufferUnit.M) - jobInfo.askedResources.cores == 3 - jobInfo.askedResources.nodes == 1 - jobInfo.askedResources.walltime == Duration.ofHours(4) - jobInfo.askedResources.storage == null - jobInfo.askedResources.queue == "otp" - jobInfo.askedResources.nthreads == null - jobInfo.askedResources.swap == null - - jobInfo.usedResources.size == null - jobInfo.usedResources.mem == new BufferValue(6064, BufferUnit.k) - jobInfo.usedResources.cores == null - jobInfo.usedResources.nodes == null - jobInfo.usedResources.walltime == Duration.ofSeconds(6) - jobInfo.usedResources.storage == null - jobInfo.usedResources.queue == "otp" - jobInfo.usedResources.nthreads == null - jobInfo.usedResources.swap == null - - jobInfo.jobName == "r180405_163953553_stds_snvJoinVcfFiles" - jobInfo.tool == null - jobInfo.jobID == new BEJobID("4564045") - jobInfo.submitTime.isEqual ZonedDateTime.of(2018, 4, 5, 14, 39, 18, 0, ZoneOffset.UTC) - jobInfo.eligibleTime.isEqual ZonedDateTime.of(2018, 4, 5, 14, 39, 42, 0, ZoneOffset.UTC) - jobInfo.startTime.isEqual ZonedDateTime.of(2018, 4, 5, 14, 39, 43, 0, ZoneOffset.UTC) - jobInfo.endTime.isEqual ZonedDateTime.of(2018, 4, 5, 14, 39, 54, 0, ZoneOffset.UTC) - jobInfo.executionHosts == ["denbi5-int"] - jobInfo.submissionHost == "subm.example.com" - jobInfo.priority == "0" - jobInfo.logFile == new File("/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o4564045.pbsserver") - jobInfo.errorLogFile == new File("/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o4564045.pbsserver") - jobInfo.inputFile == null - jobInfo.user == null - jobInfo.userGroup == null - jobInfo.resourceReq == '-v TOOL_ID=snvJoinVcfFiles,PARAMETER_FILE=/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles_9.parameters,CONFIG_FILE=/net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles_9.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N r180405_163953553_stds_snvJoinVcfFiles -h -w /home/otp-data -j oe -o /net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/r180405_163953553_stds_snvJoinVcfFiles.o$PBS_JOBID -l mem=1024M -l walltime=00:04:00:00 -l nodes=1:ppn=3 -q otp -W depend=afterok:6059780:6059781 /net/isilon/otp/workflow-tests/tmp/RoddyBamFileWgsRoddySnvWorkflow-otp-web-2018-04-05-16-38-05-094+0200-k4FXdCscdyo3vAxl/root_path/projectDirName_22/sequencing/whole_genome_sequencing/view-by-pid/stds/snv_results/paired/sampletypename-24_sampletypename-43/results_SNVCallingWorkflow-1.2.166-1_v1_0_2018-04-05_16h39_+0200/roddyExecutionStore/exec_180405_163953553_otp-data_WGS/analysisTools/roddyTools/wrapInScript.sh' - jobInfo.startCount == 1 - jobInfo.account == null - jobInfo.server == "pbsserver" - jobInfo.umask == null - jobInfo.parameters == null - jobInfo.parentJobIDs == ["6059780", "6059781"] - jobInfo.otherSettings == null - jobInfo.jobState == JobState.COMPLETED_UNKNOWN - jobInfo.userTime == null - jobInfo.systemTime == null - jobInfo.pendReason == null - jobInfo.execHome == null - jobInfo.execUserName == null - jobInfo.pidStr == null - jobInfo.pgidStr == null - jobInfo.exitCode == 0 - jobInfo.jobGroup == null - jobInfo.description == null - jobInfo.execCwd == null - jobInfo.askedHostsStr == null - jobInfo.cwd == null - jobInfo.projectName == null - jobInfo.cpuTime == Duration.ZERO - jobInfo.runTime == Duration.ofSeconds(10) - jobInfo.timeUserSuspState == null - jobInfo.timePendState == null - jobInfo.timePendSuspState == null - jobInfo.timeSystemSuspState == null - jobInfo.timeUnknownState == null - jobInfo.timeOfCalculation == null - } - - void "processQstatOutput, qstat with empty XML output"() { + jobInfo == expected + } + + void "test queryExtendedJobInfoByID with empty XML output"() { given: - String rawXMLOutput =''' - - - 15020227.testServer - - - ''' + def jm = createJobManagerWithModifiedExecutionService("queryExtendedJobInfoWithEmptyXML.xml") when: - Map jobInfo = manager.processQstatOutputFromXML(rawXMLOutput) + Map result = jm.queryExtendedJobInfo([testJobID]) + def jobInfo = result.get(testJobID) then: - jobInfo.size() == 1 + result.size() == 1 + jobInfo } - void "processQstatOutput, qstat with XML output"() { + void "test queryExtendedJobInfoByID, replace placeholder PBS_JOBID in logFile and errorLogFile with job id "() { + given: + def jm = createJobManagerWithModifiedExecutionService("queryExtendedJobInfoWithPlaceholderReplacement.xml") + when: - Map jobInfo = manager.processQstatOutputFromXML(rawXmlExample) + Map result = jm.queryExtendedJobInfo([testJobID]) + def jobInfo = result.get(testJobID) then: - jobInfo.size() == 1 - jobInfo.get(new BEJobID("15020227")).jobName == "workflow_test" + result.size() == 1 + jobInfo.logFile.absolutePath == "/logging_root_path/logfile.o22005.testServer" + jobInfo.errorLogFile.absolutePath == "/logging_root_path/logfile.e22005.testServer" } + def "test getEnvironmentVariableGlobs"() { + return null + } - void "processQstatOutput, replace placeholder PBS_JOBID in logFile and errorLogFile with job id "() { - given: - String rawXMLOutput=''' - - - 15976927.testServer - /logging_root_path/clusterLog/2017-07-07/workflow_test/snvFilter.o$PBS_JOBID - tbi-pbs:/logging_root_path/clusterLog/2017-07-07/workflow_test/snvFilter.e$PBS_JOBID - - - ''' + def "test getDefaultForHoldJobsEnabled"() { + return null + } - when: - Map jobInfo = manager.processQstatOutputFromXML(rawXMLOutput) + def "test isHoldJobsEnabled"() { + return null + } - then: - jobInfo.size() == 1 - jobInfo.get(new BEJobID("15976927")).logFile.toString() == "/logging_root_path/clusterLog/2017-07-07/workflow_test/snvFilter.o15976927.testServer" - jobInfo.get(new BEJobID("15976927")).errorLogFile.toString() == "/logging_root_path/clusterLog/2017-07-07/workflow_test/snvFilter.e15976927.testServer" + def "test getUserEmail"() { + return null } + def "test getUserMask"() { + return null + } - void "parseColonSeparatedHHMMSSDuration, parse duration"() { + def "test getUserGroup"() { + return null + } - given: - Method method = ClusterJobManager.class.getDeclaredMethod("parseColonSeparatedHHMMSSDuration", String) - method.setAccessible(true) + def "test getUserAccount"() { + return null + } - expect: - parsedDuration == method.invoke(null, input) + def "test executesWithoutJobSystem"() { + return null + } - where: - input || parsedDuration - "00:00:00" || Duration.ofSeconds(0) - "24:00:00" || Duration.ofHours(24) - "119:00:00" || Duration.ofHours(119) + def "test convertResourceSet"() { + return null } - - void "parseColonSeparatedHHMMSSDuration, parse duration fails"() { + def "test collectJobIDsFromJobs"() { + return null + } - given: - Method method = ClusterJobManager.class.getDeclaredMethod("parseColonSeparatedHHMMSSDuration", String) - method.setAccessible(true) + def "test extractAndSetJobResultFromExecutionResult"() { + return null + } - when: - method.invoke(null, "02:42") + def "test createCommand"() { + return null + } - then: - InvocationTargetException e = thrown(InvocationTargetException) - e.targetException.message == "Duration string is not of the format HH+:MM:SS: '02:42'" + def "test parseJobID"() { + return null + } + + def "test parseJobState"() { + return null + } + + def "test executeStartHeldJobs"() { + return null + } + + def "test executeKillJobs"() { + return null } } diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerSpec.groovy new file mode 100644 index 00000000..986c549c --- /dev/null +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerSpec.groovy @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2017 eilslabs. + * + * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). + */ + +package de.dkfz.roddy.execution.jobs.cluster.sge + +import de.dkfz.roddy.execution.BEExecutionService +import de.dkfz.roddy.execution.io.ExecutionResult +import de.dkfz.roddy.execution.jobs.* +import de.dkfz.roddy.execution.jobs.cluster.JobManagerImplementationBaseSpec + +import static de.dkfz.roddy.execution.jobs.JobState.* + +/** + */ +class SGEJobManagerSpec extends JobManagerImplementationBaseSpec { + + + def "test submitJob"() { + return null + } + + def "test addToListOfStartedJobs"() { + return null + } + + def "test startHeldJobs"() { + return null + } + + def "test killJobs"() { + return null + } + + def "test queryJobInfoByJob (also tests multiple requests and by ID)"(String id, expectedState) { + given: + def jobManager = createJobManagerWithModifiedExecutionService("simpleQStatOutput.txt") + def jobID = new BEJobID(id) + def job = new BEJob(jobID, jobManager) + + when: + JobInfo info = jobManager.queryJobInfoByJob(job) + + then: + info.jobID == jobID + info.jobState == expectedState + + where: + id | expectedState + "22000" | UNKNOWN + "22003" | RUNNING + "22005" | HOLD + } + + def "test queryAllJobInfo"() { + given: + def jobManager = createJobManagerWithModifiedExecutionService("simpleQStatOutput.txt") + def jobIDs = ["22003", "22005", "22006", "22007"].collect { new BEJobID(it) } + when: + def result = jobManager.queryAllJobInfo() + + then: + result.size() == 4 + result.keySet() as List == jobIDs + result[jobIDs[0]].jobState == RUNNING + result[jobIDs[1]].jobState == HOLD + result[jobIDs[2]].jobState == HOLD + result[jobIDs[3]].jobState == HOLD + } + + SGEJobManager createSGEJobManagerForExtendedInfoTests() { + // This one is special, as query extended states in SGE utilizes queryJobInfo + BEExecutionService testExecutionService = [ + execute: { + String s -> + if (s.startsWith("qstat -xml")) + new ExecutionResult(true, 0, getResourceFile(SGEJobManagerSpec, "queryExtendedJobStatesWithQueuedJobs.xml").readLines(), null) + else + new ExecutionResult(true, 0, getResourceFile(SGEJobManagerSpec, "simpleQStatOutput.txt").readLines(), null) + } + ] as BEExecutionService + def jobManager = new SGEJobManager(testExecutionService, new JobManagerOptionsBuilder().build()) + return jobManager + } + + def "test queryExtendedJobInfoByJob (also tests multiple requests and by ID)"(String id, expectedState) { + given: + def jobManager = createSGEJobManagerForExtendedInfoTests() + def jobID = new BEJobID(id) + def job = new BEJob(jobID, jobManager) + + when: + JobInfo info = jobManager.queryExtendedJobInfoByJob(job) + + then: + info.jobID == jobID && info.jobState == expectedState + + where: + id | expectedState + "22000" | UNKNOWN + "22003" | RUNNING + "22005" | HOLD + } + + def "test queryAllExtendedJobInfo"() { + given: + def jobManager = createSGEJobManagerForExtendedInfoTests() + def jobIDs = ["22003", "22005", "22006", "22007"].collect { new BEJobID(it) } + + when: + Map result = jobManager.queryAllExtendedJobInfo() + + then: + result.size() == 4 + result[jobIDs[0]].jobID == jobIDs[0] + result[jobIDs[0]].jobState == RUNNING + + result[jobIDs[1]].jobID == jobIDs[1] + result[jobIDs[1]].jobState == HOLD + + result[jobIDs[2]].jobID == jobIDs[2] + result[jobIDs[2]].jobState == HOLD + + result[jobIDs[3]].jobID == jobIDs[3] + result[jobIDs[3]].jobState == HOLD + } + + def "test getEnvironmentVariableGlobs"() { + return null + } + + def "test getDefaultForHoldJobsEnabled"() { + return null + } + + def "test isHoldJobsEnabled"() { + return null + } + + def "test getUserEmail"() { + return null + } + + def "test getUserMask"() { + return null + } + + def "test getUserGroup"() { + return null + } + + def "test getUserAccount"() { + return null + } + + def "test executesWithoutJobSystem"() { + return null + } + + def "test convertResourceSet"() { + return null + } +// @Test +// public void testConvertToolEntryToPBSCommandParameters() { + // Roddy specific tests! +// ResourceSet rset1 = new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 2, 1, new TimeUnit("h"), null, null, null); +// ResourceSet rset2 = new ResourceSet(ResourceSetSize.l, null, null, 1, new TimeUnit("h"), null, null, null); +// ResourceSet rset3 = new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 2, null, null, null, null, null); +// +// Configuration cfg = new Configuration(new InformationalConfigurationContent(null, Configuration.ConfigurationType.OTHER, "test", "", "", null, "", ResourceSetSize.l, null, null, null, null)); +// +// BatchEuphoriaJobManager cFactory = new SGEJobManager(false); +// PBSResourceProcessingParameters test = (PBSResourceProcessingParameters) cFactory.convertResourceSet(cfg, rset1); +// assert test.getProcessingCommandString().trim().equals("-V -l s_data=1024M"); +// +// test = (PBSResourceProcessingParameters) cFactory.convertResourceSet(cfg, rset2); +// assert test.getProcessingCommandString().equals(" -V"); +// +// test = (PBSResourceProcessingParameters) cFactory.convertResourceSet(cfg, rset3); +// assert test.getProcessingCommandString().equals(" -V -l s_data=1024M"); +// } + + def "test collectJobIDsFromJobs"() { + return null + } + + def "test extractAndSetJobResultFromExecutionResult"() { + return null + } + + def "test createCommand"() { + return null + } + + def "test parseJobID"() { + return null + } + + def "test parseJobState"() { + return null + } + + def "test executeStartHeldJobs"() { + return null + } + + def "test executeKillJobs"() { + return null + } +} diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerTest.java b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerTest.java deleted file mode 100644 index 438ed5c9..00000000 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/sge/SGEJobManagerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2017 eilslabs. - * - * Distributed under the MIT License (license terms are at https://www.github.com/eilslabs/Roddy/LICENSE.txt). - */ - -package de.dkfz.roddy.execution.jobs.cluster.sge; - -import org.junit.Test; - -/** - */ -public class SGEJobManagerTest { - - @Test - public void testConvertToolEntryToPBSCommandParameters() { - // Roddy specific tests! -// ResourceSet rset1 = new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 2, 1, new TimeUnit("h"), null, null, null); -// ResourceSet rset2 = new ResourceSet(ResourceSetSize.l, null, null, 1, new TimeUnit("h"), null, null, null); -// ResourceSet rset3 = new ResourceSet(ResourceSetSize.l, new BufferValue(1, BufferUnit.G), 2, null, null, null, null, null); -// -// Configuration cfg = new Configuration(new InformationalConfigurationContent(null, Configuration.ConfigurationType.OTHER, "test", "", "", null, "", ResourceSetSize.l, null, null, null, null)); -// -// BatchEuphoriaJobManager cFactory = new SGEJobManager(false); -// PBSResourceProcessingParameters test = (PBSResourceProcessingParameters) cFactory.convertResourceSet(cfg, rset1); -// assert test.getProcessingCommandString().trim().equals("-V -l s_data=1024M"); -// -// test = (PBSResourceProcessingParameters) cFactory.convertResourceSet(cfg, rset2); -// assert test.getProcessingCommandString().equals(" -V"); -// -// test = (PBSResourceProcessingParameters) cFactory.convertResourceSet(cfg, rset3); -// assert test.getProcessingCommandString().equals(" -V -l s_data=1024M"); - } - - -} diff --git a/src/test/resources/de/dkfz/roddy/batcheuphoria/getResourceFileTest.txt b/src/test/resources/de/dkfz/roddy/batcheuphoria/getResourceFileTest.txt new file mode 100644 index 00000000..f2ba8f84 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/batcheuphoria/getResourceFileTest.txt @@ -0,0 +1 @@ +abc \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/bjobsJobTemplatePart1.txt b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/bjobsJobTemplatePart1.txt new file mode 100644 index 00000000..7e9163bd --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/bjobsJobTemplatePart1.txt @@ -0,0 +1 @@ + { diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/bjobsJobTemplatePart2.txt b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/bjobsJobTemplatePart2.txt new file mode 100644 index 00000000..f76286ac --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/bjobsJobTemplatePart2.txt @@ -0,0 +1,32 @@ + "STAT":"RUN", + "USER":"icgcdata", + "QUEUE":"verylong", + "JOB_DESCRIPTION":"", + "PROJ_NAME":"default", + "JOB_GROUP":"", + "JOB_PRIORITY":"", + "PIDS":"15197,15238,15242,16672,16750,16751,16754,16756,16767,16769,16771,16774,16775,16776,16777,16778,16779,16781,16783,16784,16786,16787,16789,16801,16802,16803,16804,16805,16806,16807,16808,16809,16810,16811,16812,16814,16815,16833", + "EXIT_CODE":"", + "FROM_HOST":"host-submission", + "EXEC_HOST":"host-cn037:host-cn037:host-cn037:host-cn037:host-cn037:host-cn037:host-cn037:host-cn037", + "SUBMIT_TIME":"Dec 17 00:35", + "START_TIME":"Dec 17 00:36", + "FINISH_TIME":"Dec 27 00:36 L", + "CPU_USED":"59:32:51", + "RUN_TIME":"08:38:26", + "USER_GROUP":"", + "SWAP":"0 Mbytes", + "MAX_MEM":"40.8 Gbytes", + "RUNTIMELIMIT":"240:00:00", + "SUB_CWD":"\/tmp\/some\/cwd\/folder", + "PEND_REASON":"", + "EXEC_CWD":"/home/a/testfolder", + "OUTPUT_FILE":"\/tmp\/some\/annoyinglylong\/folder\/for\/a\/textfile\/andhereisthelogfile.o471773", + "INPUT_FILE":"", + "EFFECTIVE_RESREQ":"select[type == local] order[r15s:pg] rusage[mem=61440.00] span[hosts=1] ", + "EXEC_HOME":"\/home\/icgcdata", + "SLOTS":"8", + "ERROR_FILE":"", + "COMMAND":"\/tmp\/a\/project\/folder\/with\/results\/roddyExecutionStore\/exec_181217_003553288_testuser_test\/analysisTools\/roddyTools\/wrapInScript.sh", + "DEPENDENCY":"" + } diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/convertBJobsResultLinesToResultMapTest.json b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/convertBJobsResultLinesToResultMapTest.json new file mode 100644 index 00000000..03504df4 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/convertBJobsResultLinesToResultMapTest.json @@ -0,0 +1,42 @@ +{ + "COMMAND":"bjobs", + "JOBS":6, + "RECORDS":[ + { + "JOBID":"487641", + "JOB_NAME":"RoddyTest_testScript", + "STAT":"EXIT", + "FINISH_TIME":"Jan 7 09:59 L" + }, + { + "JOBID":"487642", + "JOB_NAME":"RoddyTest_testScript", + "STAT":"EXIT", + "FINISH_TIME":"Jan 7 09:59 L" + }, + { + "JOBID":"488155", + "JOB_NAME":"RoddyTest_testScript", + "STAT":"EXIT", + "FINISH_TIME":"Jan 7 09:59 L" + }, + { + "JOBID":"491861", + "JOB_NAME":"RoddyTest_testScript", + "STAT":"DONE", + "FINISH_TIME":"Jan 8 14:17 L" + }, + { + "JOBID":"491864", + "JOB_NAME":"RoddyTest_testScript", + "STAT":"DONE", + "FINISH_TIME":"Jan 8 14:20 L" + }, + { + "JOBID":"491867", + "JOB_NAME":"RoddyTest_testScriptMultiInMixedParameters", + "STAT":"DONE", + "FINISH_TIME":"Jan 8 14:21 L" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/createJobManagerWithModifiedExecutionServiceTest.txt b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/createJobManagerWithModifiedExecutionServiceTest.txt new file mode 100644 index 00000000..0b008121 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/createJobManagerWithModifiedExecutionServiceTest.txt @@ -0,0 +1,3 @@ +Line1 +Line2 +Line3 \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdEmptyTest.json b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdEmptyTest.json new file mode 100644 index 00000000..114f9481 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdEmptyTest.json @@ -0,0 +1,9 @@ +{ + "COMMAND": "bjobs", + "JOBS": 1, + "RECORDS": [ + { + "JOBID": "22005" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdTest.json b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdTest.json new file mode 100644 index 00000000..295eabef --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdTest.json @@ -0,0 +1,41 @@ +{ + "COMMAND": "bjobs", + "JOBS": 1, + "RECORDS": [ + { + "JOBID": "22005", + "JOB_NAME": "ls -l", + "STAT": "DONE", + "USER": "otptest", + "QUEUE": "short-dmg", + "JOB_DESCRIPTION": "", + "PROJ_NAME": "default", + "JOB_GROUP": "", + "JOB_PRIORITY": "", + "PIDS": "46782,46796,46798,46915,47458,47643", + "EXIT_CODE": "", + "FROM_HOST": "from-host", + "EXEC_HOST": "exec-host:exec-host", + "SUBMIT_TIME": "Dec 28 19:56", + "START_TIME": "Dec 28 19:56", + "FINISH_TIME": "Dec 28 19:56 L", + "CPU_USED": "00:00:01", + "RUN_TIME": "00:00:01", + "USER_GROUP": "", + "SWAP": "", + "MAX_MEM": "5.2 Gbytes", + "RUNTIMELIMIT": "00:10:00", + "SUB_CWD": "$HOME", + "PEND_REASON": "", + "EXEC_CWD": "/some/test", + "OUTPUT_FILE": "", + "INPUT_FILE": "", + "EFFECTIVE_RESREQ": "select[type == local] order[r15s:pg] ", + "EXEC_HOME": "/some/test", + "SLOTS": "1", + "ERROR_FILE": "", + "COMMAND": "/home/testuser/somescript.sh", + "DEPENDENCY": "done(22004)" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdWithoutListsTest.json b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdWithoutListsTest.json new file mode 100644 index 00000000..1b24940a --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryExtendedJobStateByIdWithoutListsTest.json @@ -0,0 +1,41 @@ +{ + "COMMAND": "bjobs", + "JOBS": 1, + "RECORDS": [ + { + "JOBID": "22005", + "JOB_NAME": "ATestJob", + "STAT": "DONE", + "USER": "testuser", + "QUEUE": "debug", + "JOB_DESCRIPTION": "", + "PROJ_NAME": "default", + "JOB_GROUP": "", + "JOB_PRIORITY": "", + "PIDS": "51904", + "EXIT_CODE": "1", + "FROM_HOST": "subm-host", + "EXEC_HOST": "exec-host", + "SUBMIT_TIME": "Dec 28 19:56", + "START_TIME": "Dec 28 19:56", + "FINISH_TIME": "Dec 28 19:56 L", + "CPU_USED": "00:00:01", + "RUN_TIME": "00:00:01", + "USER_GROUP": "", + "SWAP": "0 Mbytes", + "MAX_MEM": "522 MBytes", + "RUNTIMELIMIT": "00:10:00", + "SUB_CWD": "$HOME", + "PEND_REASON": "Job dependency condition not satisfied;", + "EXEC_CWD": "/some//test", + "OUTPUT_FILE": "/sequencing/whole_genome_sequencing/coveragePlotSingle.o30060", + "INPUT_FILE": "", + "EFFECTIVE_RESREQ": "select[type == local] order[r15s:pg] ", + "EXEC_HOME": "/home/testuser", + "SLOTS": "1", + "ERROR_FILE": "", + "COMMAND": "/home/testuser/somescript.sh", + "DEPENDENCY": "done(22004)" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryJobStateByIdTest.json b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryJobStateByIdTest.json new file mode 100644 index 00000000..80f55881 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/lsf/queryJobStateByIdTest.json @@ -0,0 +1,12 @@ +{ + "COMMAND": "bjobs", + "JOBS": 1, + "RECORDS": [ + { + "JOBID": "22005", + "JOB_NAME": "ls -l", + "STAT": "DONE", + "FINISH_TIME": "Dec 28 19:56 L" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/createJobManagerWithModifiedExecutionServiceTest.txt b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/createJobManagerWithModifiedExecutionServiceTest.txt new file mode 100644 index 00000000..0b008121 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/createJobManagerWithModifiedExecutionServiceTest.txt @@ -0,0 +1,3 @@ +Line1 +Line2 +Line3 \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputTests.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputTests.xml new file mode 100644 index 00000000..657e6102 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputTests.xml @@ -0,0 +1,52 @@ + + + + 15020227.testServer + workflow_test + testOwner + H + fast + afterok:15624736.testServer + testServer + u + 1499432861 + logging_root_path/clusterLog/2017-07-07/workflow_test + u + oe + n + a + 1499432861 + 1514572025 + logging_root_path/clusterLog/2017-07-07/workflow_test.o15020227 + 0 + 1499432861 + True + + 71497 + + + 5120mb + 1 + 1:ppn=1 + 00:20:00 + + + 04:50:19 + 0 + 2449672kb + 2524268kb + 03:07:41 + + PBS_O_QUEUE=default,PBS_O_HOME=/home/test,PBS_O_LOGNAME=test,PBS_O_SHELL=/bin/bash,PBS_O_LANG=en_GB.UTF-8 + test + testGroup + E + -N workflow_test -h -o /clusterLog/2017-07-07 -j oe -W umask=027 -l mem=5120M -l walltime=00:00:20:00 -l nodes=1:ppn=1 + 23 + False + 0 + sub.testServer + 1 + 1 + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithFinishedJob.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithFinishedJob.xml new file mode 100644 index 00000000..b53b4dd9 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithFinishedJob.xml @@ -0,0 +1,65 @@ + + + + 22005.pbsserver + ATestJob + testuser@subm-host.example.com + + 00:00:00 + 6064kb + 454232kb + 00:00:06 + + C + debug + pbsserver + u + 1522255162 + afterok:6059780.pbsserver@pbsserver:6059781.pbsserver@pbsserver,beforeok:6059788.pbsserver@pbsserver + subm:/somefolder/job.o$PBS_JOBID + exec-host/0+exec-host/1+exec-host/2 + 15003 + n + oe + n + a + 1522255162 + subm:/somefolder/job.o$PBS_JOBID + 0 + 1522255162 + True + + 168:00:00 + 37888mb + 1 + 5 + 1 + 1:ppn=4 + 05:00:00 + + 51904 + PBS_O_QUEUE=otp,PBS_O_HOME=/home/otp-data,PBS_O_LOGNAME=otp-data,PBS_O_PATH=/usr/local/bin:/usr/bin:/bin:/usr/games,PBS_O_MAIL=/var/mail/otp-data,PBS_O_SHELL=/bin/bash,PBS_O_LANG=en_US.utf8,TOOL_ID=starAlignment,PARAMETER_FILE=/somefolder/exec/ATestJob.parameters,CONFIG_FILE=/somefolder/exec/ATestJob.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme,PBS_O_WORKDIR=/home/otp-data,PBS_O_HOST=subm.example.com,PBS_O_SERVER=pbsserver + Post job file processing error; job 4564045.pbsserver on host exec-host/1 + + Unable to copy file /var/spool/torque/spool/4564045.pbsserver.OU to + otp-data@subm:/somefolder/results/r180405_163953553_stds_snvJoinVcfFiles.o4564045.pbsserver + *** error from copy + Host key verification failed. + lost connection + *** end error output + Output retained on that host in: /var/spool/torque/undelivered/4564045.pbsserver.OU + + 1522255162 + 0 + -v TOOL_ID=starAlignment,PARAMETER_FILE=/somefolder/exec/ATestJob.parameters,CONFIG_FILE=/somefolder/exec/ATestJob.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N ATestJob -h -w /home/otp-data -j oe -o /somefolder/job.o$PBS_JOBID -l mem=37888M -l walltime=00:05:00:00 -l nodes=1:ppn=4 -q otp /somefolder/wrapInScript.sh + 1522939183 + 1 + False + 1522939194 + 0 + 10.431524 + subm-host.example.com + /home/testuser + + diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithQueuedJob.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithQueuedJob.xml new file mode 100644 index 00000000..6ca8491a --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/processExtendedQstatXMLOutputWithQueuedJob.xml @@ -0,0 +1,40 @@ + + + + 22005.pbsserver + ATestJob + testuser@subm-host.example.com + Q + debug + pbsserver + u + 1522255162 + subm:/somefolder/job.o$PBS_JOBID + n + oe + n + a + 1522255162 + subm:/somefolder/job.o$PBS_JOBID + 0 + 1522255162 + True + + 168:00:00 + 37888mb + 1 + 5 + 1 + 1:ppn=4 + 05:00:00 + + PBS_O_QUEUE=otp,PBS_O_HOME=/home/otp-data,PBS_O_LOGNAME=otp-data,PBS_O_PATH=/usr/local/bin:/usr/bin:/bin:/usr/games,PBS_O_MAIL=/var/mail/otp-data,PBS_O_SHELL=/bin/bash,PBS_O_LANG=en_US.utf8,TOOL_ID=starAlignment,PARAMETER_FILE=/somefolder/exec/ATestJob.parameters,CONFIG_FILE=/somefolder/exec/ATestJob.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme,PBS_O_WORKDIR=/home/otp-data,PBS_O_HOST=subm.example.com,PBS_O_SERVER=pbsserver + 1522255162 + -v TOOL_ID=starAlignment,PARAMETER_FILE=/somefolder/exec/ATestJob.parameters,CONFIG_FILE=/somefolder/exec/ATestJob.parameters,debugWrapInScript=false,baseEnvironmentScript=/tbi/software/sourceme -N ATestJob -h -w /home/otp-data -j oe -o /somefolder/job.o$PBS_JOBID -l mem=37888M -l walltime=00:05:00:00 -l nodes=1:ppn=4 -q otp /somefolder/wrapInScript.sh + False + 0 + subm-host.example.com + /home/testuser + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithEmptyXML.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithEmptyXML.xml new file mode 100644 index 00000000..3c688c0c --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithEmptyXML.xml @@ -0,0 +1,5 @@ + + + 22005.testServer + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithPlaceholderReplacement.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithPlaceholderReplacement.xml new file mode 100644 index 00000000..1ddf87b4 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/pbs/queryExtendedJobInfoWithPlaceholderReplacement.xml @@ -0,0 +1,7 @@ + + + 22005.testServer + testServer:/logging_root_path/logfile.o$PBS_JOBID + testServer:/logging_root_path/logfile.e$PBS_JOBID + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/createJobManagerWithModifiedExecutionServiceTest.txt b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/createJobManagerWithModifiedExecutionServiceTest.txt new file mode 100644 index 00000000..0b008121 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/createJobManagerWithModifiedExecutionServiceTest.txt @@ -0,0 +1,3 @@ +Line1 +Line2 +Line3 \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithFinishedJob.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithFinishedJob.xml new file mode 100644 index 00000000..d68ed0bb --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithFinishedJob.xml @@ -0,0 +1,317 @@ + + + + + 22005 + 0 + 1548935890 + testuser + 1000 + testuser + 1000 + sge + true + + + testuser + localhost + + + false + RoddyTest_testScript + + + /somefolder/job.o$JOB_ID + + + false + + + 0 + + + h_rss + 4 + 37888M + 39728447488.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + h_rt + 3 + 18000 + 18000.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + + + /bin/bash + + + false + + + + + __SGE_PREFIX__O_HOME + /home/testuser + + + __SGE_PREFIX__O_LOGNAME + testuser + + + __SGE_PREFIX__O_PATH + /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + + + __SGE_PREFIX__O_SHELL + /bin/bash + + + __SGE_PREFIX__O_MAIL + /var/mail/testuser + + + __SGE_PREFIX__O_HOST + localhost + + + __SGE_PREFIX__O_WORKDIR + /home/testuser + + + PARAMETER_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript_1.parameters + + + TOOL_ID + testScript + + + CONFIG_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript_1.parameters + + + debugWrapInScript + false + + + baseEnvironmentScript + /etc/profile + + + + + 65536 + 1 + + + io + 0.000000 + + + iow + 0.000000 + + + mem + 0.000000 + + + cpu + 0.016000 + + + vmem + 0.000000 + + + maxvmem + 0.000000 + + + submission_time + 1548935890.000000 + + + priority + 0.000000 + + + exit_status + 1.000000 + + + signal + 0.000000 + + + start_time + 1548936457.000000 + + + end_time + 1548936457.000000 + + + ru_wallclock + 0.000000 + + + ru_utime + 0.016000 + + + ru_stime + 0.000000 + + + ru_maxrss + 3384.000000 + + + ru_ixrss + 0.000000 + + + ru_ismrss + 0.000000 + + + ru_idrss + 0.000000 + + + ru_isrss + 0.000000 + + + ru_minflt + 5188.000000 + + + ru_majflt + 0.000000 + + + ru_nswap + 0.000000 + + + ru_inblock + 0.000000 + + + ru_oublock + 32.000000 + + + ru_msgsnd + 0.000000 + + + ru_msgrcv + 0.000000 + + + ru_nsignals + 0.000000 + + + ru_nvcsw + 170.000000 + + + ru_nivcsw + 113.000000 + + + acct_cpu + 0.016000 + + + acct_mem + 0.000000 + + + acct_io + 0.000000 + + + acct_iow + 0.000000 + + + acct_maxvmem + 0.000000 + + + + + /home/testuser + + + 22006 + + + 22007 + + + 0 + 0 + 0 + 0 + false + 0 + 1024 + 0 + 0 + 0 + serial + + + 4 + 4 + 1 + + + 0 + 0 + 0 + 0 + 1 + + + 1 + 1 + 1 + + + 0 + + + + + + + 37 + queue instance "main.q@localhost" dropped because it is full + + + + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithJobWithPipedScript.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithJobWithPipedScript.xml new file mode 100644 index 00000000..00fe0a5c --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithJobWithPipedScript.xml @@ -0,0 +1,111 @@ + + + + + 22005 + 0 + job_scripts/145 + 1548934527 + testuser + 1000 + testuser + 1000 + sge + true + + + testuser + localhost + + + false + ATestJob + 0 + + + h_rss + 4 + 2G + 2147483648.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + + + __SGE_PREFIX__O_HOME + /home/testuser + + + __SGE_PREFIX__O_LOGNAME + testuser + + + __SGE_PREFIX__O_PATH + /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + + + __SGE_PREFIX__O_SHELL + /bin/bash + + + __SGE_PREFIX__O_MAIL + /var/mail/testuser + + + __SGE_PREFIX__O_HOST + localhost + + + __SGE_PREFIX__O_WORKDIR + /home/testuser + + + STDIN + + + 128 + 1 + + + 0 + 0 + 0 + 0 + false + 0 + 1024 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 1 + 1 + 1 + + + 0 + + + + + + + 37 + queue instance "main.q@localhost" dropped because it is full + + + + + diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithQueuedJobs.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithQueuedJobs.xml new file mode 100644 index 00000000..1e2ec829 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithQueuedJobs.xml @@ -0,0 +1,632 @@ + + + + + 22003 + 0 + job_scripts/22003 + 1548935890 + testuser + 1000 + testuser + 1000 + sge + true + + + testuser + localhost + + + false + ATestJob + + + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript.o$JOB_ID + + + false + + + 0 + + + h_rss + 4 + 1024M + 1073741824.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + h_rt + 3 + 18000 + 18000.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + + + /bin/bash + + + false + + + + + __SGE_PREFIX__O_HOME + /home/testuser + + + __SGE_PREFIX__O_LOGNAME + testuser + + + __SGE_PREFIX__O_PATH + /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + + + __SGE_PREFIX__O_SHELL + /bin/bash + + + __SGE_PREFIX__O_MAIL + /var/mail/testuser + + + __SGE_PREFIX__O_HOST + localhost + + + __SGE_PREFIX__O_WORKDIR + /home/testuser + + + PARAMETER_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript_1.parameters + + + TOOL_ID + testScript + + + CONFIG_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript_1.parameters + + + debugWrapInScript + false + + + baseEnvironmentScript + /etc/profile + + + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/analysisTools/roddyTools/wrapInScript.sh + /home/testuser + + + 22006 + + + 22007 + + + 0 + 0 + 0 + 0 + false + 0 + 1024 + 0 + 0 + 0 + serial + + + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + + + 1 + 1 + 1 + + + 0 + + + 22005 + 0 + job_scripts/22005 + 1549272907 + heinold + 1000 + heinold + 1000 + sge + true + + + heinold + localhost + + + false + RoddyTest_testScript + + + /home/heinold/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190204_103439242_heinold_testCorrect2/RoddyTest_testScript.o$JOB_ID + + + false + + + 0 + + + h_rss + 4 + 3788M + 3972005888.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + h_rt + 3 + 18000 + 18000.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + + + /bin/bash + + + false + + + + + __SGE_PREFIX__O_HOME + /home/heinold + + + __SGE_PREFIX__O_LOGNAME + heinold + + + __SGE_PREFIX__O_PATH + /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + + + __SGE_PREFIX__O_SHELL + /bin/bash + + + __SGE_PREFIX__O_MAIL + /var/mail/heinold + + + __SGE_PREFIX__O_HOST + localhost + + + __SGE_PREFIX__O_WORKDIR + /home/heinold + + + PARAMETER_FILE + /home/heinold/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190204_103439242_heinold_testCorrect2/RoddyTest_testScript_1.parameters + + + TOOL_ID + testScript + + + CONFIG_FILE + /home/heinold/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190204_103439242_heinold_testCorrect2/RoddyTest_testScript_1.parameters + + + debugWrapInScript + false + + + baseEnvironmentScript + /etc/profile + + + /home/heinold/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190204_103439242_heinold_testCorrect2/analysisTools/roddyTools/wrapInScript.sh + /home/heinold + + + 162 + + + 163 + + + 0 + 0 + 0 + 0 + false + 0 + 1024 + 0 + 0 + 0 + serial + + + 4 + 4 + 1 + + + 0 + 0 + 0 + 0 + 0 + + + 1 + 1 + 1 + + + 0 + + + 22006 + 0 + job_scripts/147 + 1548935902 + testuser + 1000 + testuser + 1000 + sge + true + + + testuser + localhost + + + false + ATestJob + + + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript.o$JOB_ID + + + false + + + 0 + + + h_rss + 4 + 1024M + 1073741824.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + h_rt + 3 + 18000 + 18000.000000 + 0 + 0 + 0 + 0.000000 + 0 + 0 + 0 + + + + + /bin/bash + + + false + + + + + __SGE_PREFIX__O_HOME + /home/testuser + + + __SGE_PREFIX__O_LOGNAME + testuser + + + __SGE_PREFIX__O_PATH + /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + + + __SGE_PREFIX__O_SHELL + /bin/bash + + + __SGE_PREFIX__O_MAIL + /var/mail/testuser + + + __SGE_PREFIX__O_HOST + localhost + + + __SGE_PREFIX__O_WORKDIR + /home/testuser + + + PARAMETER_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript_2.parameters + + + TOOL_ID + testScript + + + CONFIG_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScript_2.parameters + + + debugWrapInScript + false + + + baseEnvironmentScript + /etc/profile + + + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/analysisTools/roddyTools/wrapInScript.sh + /home/testuser + + + 146 + + + + + 148 + + + 0 + 0 + 0 + 0 + false + 0 + 1024 + 0 + 0 + 0 + serial + + + 1 + 1 + 1 + + + + + 0 + 146 + + + 0 + 0 + 0 + 0 + 0 + + + 1 + 1 + 1 + + + 0 + + + 22007 + 0 + job_scripts/148 + 1548935913 + testuser + 1000 + testuser + 1000 + sge + true + + + testuser + localhost + + + false + ATestJob + + + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScriptMultiInMixedParameters.o$JOB_ID + + + false + + + 0 + + + /bin/bash + + + false + + + + + __SGE_PREFIX__O_HOME + /home/testuser + + + __SGE_PREFIX__O_LOGNAME + testuser + + + __SGE_PREFIX__O_PATH + /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games + + + __SGE_PREFIX__O_SHELL + /bin/bash + + + __SGE_PREFIX__O_MAIL + /var/mail/testuser + + + __SGE_PREFIX__O_HOST + localhost + + + __SGE_PREFIX__O_WORKDIR + /home/testuser + + + PARAMETER_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScriptMultiInMixedParameters_3.parameters + + + TOOL_ID + testScriptMultiInMixedParameters + + + CONFIG_FILE + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/RoddyTest_testScriptMultiInMixedParameters_3.parameters + + + debugWrapInScript + false + + + baseEnvironmentScript + /etc/profile + + + /home/testuser/temp/roddyLocalTest/testproject/rpp/stds/roddyExecutionStore/exec_190131_125744358_testuser_testCorrect2/analysisTools/roddyTools/wrapInScript.sh + /home/testuser + + + 146 + + + 147 + + + 0 + 0 + 0 + 0 + false + 0 + 1024 + 0 + 0 + 0 + + + 0 + 146 + + + 0 + 147 + + + 0 + 0 + 0 + 0 + 0 + + + 1 + 1 + 1 + + + 0 + + + + + + + + + 146 + + + 147 + + + 148 + + + 33 + Job is in hold state + + + + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithUnknownJobs.xml b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithUnknownJobs.xml new file mode 100644 index 00000000..11130f92 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/queryExtendedJobStatesWithUnknownJobs.xml @@ -0,0 +1,9 @@ + + + <> + 153 + + <> + 159 + + \ No newline at end of file diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/simpleQStatOutput.txt b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/simpleQStatOutput.txt new file mode 100644 index 00000000..7997162d --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/sge/simpleQStatOutput.txt @@ -0,0 +1,6 @@ +job-ID prior name user state submit/start at queue slots ja-task-ID +----------------------------------------------------------------------------------------------------------------- + 22003 0.50000 RoddyTest_ testuser r 01/31/2019 13:07:37 main.q@localhost 1 + 22005 0.50000 RoddyTest_ testuser hqw 01/31/2019 13:07:37 1 + 22006 0.00000 RoddyTest_ testuser hqw 01/31/2019 12:58:22 1 + 22007 0.00000 RoddyTest_ testuser hqw 01/31/2019 12:58:33 1