From f456e983652ce87a78d43804e0a5e994719df78f Mon Sep 17 00:00:00 2001 From: Philipp Strubel Date: Wed, 5 Jul 2023 11:30:10 +0200 Subject: [PATCH] Changed the conversion of EpochSeconds to ZonedDateTime for the value "0". For eligibleTime it is set to null For startTime it is set to endTime --- README.md | 5 + .../jobs/cluster/slurm/SlurmJobManager.groovy | 12 +- .../cluster/slurm/SlurmJobManagerSpec.groovy | 25 ++- .../jobs/cluster/slurm/sacct_not_started.json | 146 ++++++++++++++++++ 4 files changed, 178 insertions(+), 10 deletions(-) create mode 100644 src/test/resources/de/dkfz/roddy/execution/jobs/cluster/slurm/sacct_not_started.json diff --git a/README.md b/README.md index 85f72106..ed7acaf3 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,11 @@ To start the integration tests, please fill in host and user settings (password # Change Logs +* 0.1.4 + - patch: Changed the conversion of EpochSeconds to ZonedDateTime for the value "0". + * For eligibleTime it is set to null + * For startTime it is set to endTime + * 0.1.3 - patch: Added getter for JobState enum. - patch: Fix SLURM API. diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy index 7ae79a6c..6a5048cb 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManager.groovy @@ -160,7 +160,7 @@ class SlurmJobManager extends GridEngineBasedJobManager { if (!supplement) { return primary } - primary.properties.findAll { k, v-> + primary.properties.findAll { k, v -> if (!v && supplement."${k}") { primary."${k}" = supplement."${k}" } @@ -337,9 +337,15 @@ class SlurmJobManager extends GridEngineBasedJobManager { /** Timestamps */ jobInfo.submitTime = parseTimeOfEpochSecond(jsonEntry["time"]["submission"] as String) - jobInfo.eligibleTime = parseTimeOfEpochSecond(jsonEntry["time"]["eligible"] as String) - jobInfo.startTime = parseTimeOfEpochSecond(jsonEntry["time"]["start"] as String) + String eligibleTime = jsonEntry["time"]["eligible"] as String + jobInfo.eligibleTime = eligibleTime != "0" ? parseTimeOfEpochSecond(eligibleTime) : null // convert "0" to null jobInfo.endTime = parseTimeOfEpochSecond(jsonEntry["time"]["end"] as String) + /** + * When startTime is "0", endTime should be used. Otherwise parseTimeOfEpochSecond will return 1970. + * Null here is not an option with how this value is currently used. + **/ + String startTime = jsonEntry["time"]["start"] as String + jobInfo.startTime = startTime != "0" ? parseTimeOfEpochSecond(startTime) : jobInfo.endTime return jobInfo diff --git a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManagerSpec.groovy b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManagerSpec.groovy index c1c66296..5c5e9b49 100644 --- a/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManagerSpec.groovy +++ b/src/test/groovy/de/dkfz/roddy/execution/jobs/cluster/slurm/SlurmJobManagerSpec.groovy @@ -44,9 +44,9 @@ class SlurmJobManagerSpec extends Specification { jobInfo.logFile == new File("/path/to/outputFile") jobInfo.user == "user(192456)" jobInfo.submissionHost == "compute038" - jobInfo.executionHosts == ["compute038"] - jobInfo.errorLogFile == new File("/path/to/errorFile") - jobInfo.execHome == "/home/user" + jobInfo.executionHosts == ["compute038"] + jobInfo.errorLogFile == new File("/path/to/errorFile") + jobInfo.execHome == "/home/user" /** Status info */ jobInfo.jobState == JobState.RUNNING @@ -59,12 +59,12 @@ class SlurmJobManagerSpec extends Specification { jobInfo.askedResources.cores == 76 jobInfo.askedResources.nodes == 1 jobInfo.askedResources.walltime == Duration.parse("PT20H") - jobInfo.askedResources.queue == "compute" + jobInfo.askedResources.queue == "compute" jobInfo.usedResources.mem == new BufferValue(300, BufferUnit.G, BufferUnit.K) jobInfo.usedResources.cores == 76 jobInfo.usedResources.nodes == 1 jobInfo.usedResources.walltime == Duration.parse("PT3H21M58S") - jobInfo.usedResources.queue == "compute" + jobInfo.usedResources.queue == "compute" jobInfo.submitTime.toString().startsWith("2023-06-20T07:07:33") jobInfo.eligibleTime.toString().startsWith("2023-06-20T07:07:33") jobInfo.startTime.toString().startsWith("2023-06-20T07:07:34") @@ -93,12 +93,12 @@ class SlurmJobManagerSpec extends Specification { jobInfo.askedResources.cores == 1 jobInfo.askedResources.nodes == 1 jobInfo.askedResources.walltime == Duration.parse("PT5H") - jobInfo.askedResources.queue == "compute" + jobInfo.askedResources.queue == "compute" jobInfo.usedResources.mem == new BufferValue(7168, BufferUnit.M, BufferUnit.K) jobInfo.usedResources.cores == 1 jobInfo.usedResources.nodes == 1 jobInfo.usedResources.walltime == Duration.parse("PT3M58S") - jobInfo.usedResources.queue == "compute" + jobInfo.usedResources.queue == "compute" jobInfo.runTime == Duration.parse("PT3M58S") /** Directories and files */ @@ -119,6 +119,17 @@ class SlurmJobManagerSpec extends Specification { jobInfo.executionHosts == ["compute015"] } + def "test processSacctOutputFromJson with sacct_not_started.json"() { + when: + GenericJobInfo jobInfo = jobManager.processSacctOutputFromJson(getResourceFile("sacct_not_started.json").text) + + then: + jobInfo.submitTime.toInstant().toEpochMilli() / 1000 == 1688463559 + !jobInfo.eligibleTime + jobInfo.endTime.toInstant().toEpochMilli() / 1000 == 1688498739 + jobInfo.startTime == jobInfo.endTime + } + def "test fillFromSupplement"() { given: GenericJobInfo primary = new GenericJobInfo("jobName", new File("command"), new BEJobID("1"), [:], []) diff --git a/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/slurm/sacct_not_started.json b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/slurm/sacct_not_started.json new file mode 100644 index 00000000..cc19a9d5 --- /dev/null +++ b/src/test/resources/de/dkfz/roddy/execution/jobs/cluster/slurm/sacct_not_started.json @@ -0,0 +1,146 @@ +{ + "meta": { + "plugin": { + "type": "openapi\/dbv0.0.37", + "name": "Slurm OpenAPI DB v0.0.37" + }, + "Slurm": { + "version": { + "major": 22, + "micro": 2, + "minor": 5 + }, + "release": "22.05.2" + } + }, + "errors": [ + ], + "jobs": [ + { + "account": "users", + "comment": { + "administrator": null, + "job": null, + "system": null + }, + "allocation_nodes": 0, + "array": { + "job_id": 0, + "limits": { + "max": { + "running": { + "tasks": 0 + } + } + }, + "task": null, + "task_id": null + }, + "association": { + "account": "users", + "cluster": "slurm-cluster", + "partition": null, + "user": "user" + }, + "cluster": "slurm-cluster", + "constraints": "", + "derived_exit_code": { + "status": "SUCCESS", + "return_code": 0 + }, + "time": { + "elapsed": 0, + "eligible": 0, + "end": 1688498739, + "start": 0, + "submission": 1688463559, + "suspended": 0, + "system": { + "seconds": 0, + "microseconds": 0 + }, + "limit": 7200, + "total": { + "seconds": 0, + "microseconds": 0 + }, + "user": { + "seconds": 0, + "microseconds": 0 + } + }, + "exit_code": { + "status": "SUCCESS", + "return_code": 0 + }, + "flags": [ + ], + "group": "users", + "het": { + "job_id": 0, + "job_offset": null + }, + "job_id": 275097, + "name": "jobName", + "mcs": { + "label": "" + }, + "nodes": "None assigned", + "partition": "devel", + "priority": 1116, + "qos": "normal", + "required": { + "CPUs": 6, + "memory": 102400 + }, + "kill_request_user": "user", + "reservation": { + "id": 0, + "name": 0 + }, + "state": { + "current": "CANCELLED", + "reason": "Dependency" + }, + "steps": [ + ], + "tres": { + "allocated": [ + ], + "requested": [ + { + "type": "cpu", + "name": null, + "id": 1, + "count": 6 + }, + { + "type": "mem", + "name": null, + "id": 2, + "count": 102400 + }, + { + "type": "node", + "name": null, + "id": 4, + "count": 1 + }, + { + "type": "billing", + "name": null, + "id": 5, + "count": 31 + } + ] + }, + "user": "user", + "wckey": { + "wckey": "", + "flags": [ + ] + }, + "working_directory": "\/path\/to\/file" + } + ] +} \ No newline at end of file