-
Notifications
You must be signed in to change notification settings - Fork 0
/
flow.cylc
121 lines (97 loc) · 2.92 KB
/
flow.cylc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
#!jinja2
{% set TASK_RUN_COMMAND = 'rose task-run --verbose' %}
[scheduler]
UTC mode = True
[scheduling]
#initial cycle point = next(T00) # start from midnight tonight
initial cycle point = previous(T00) # start from midnight last night
runahead limit = P10D
[[graph]]
{% if TEST %}
R1 = """
@wall_clock => archive_logs:finish =>
check_logs:fail? => sync_puma2_logs => untar_logs
check_logs? | untar_logs => process_logs =>
concat_logs => analyse_data
"""
{% else %}
PT12H = """
@wall_clock => archive_logs:finish =>
check_logs:fail? => sync_puma2_logs => untar_logs
check_logs? | untar_logs => housekeeping
"""
P1D = """
check_logs? | untar_logs => process_logs =>
concat_logs => analyse_data => housekeeping
"""
{% endif %}
[[special tasks]]
clock-expire = LOGS(P1D), PROCESS(P1D), ANALYSIS(P1D), housekeeping(P1D)
[runtime]
# Global settings
[[root]]
init-script = """
export CYLC_VERSION={{CYLC_VERSION}}
export ROSE_VERSION={{ROSE_VERSION}}
"""
script = {{TASK_RUN_COMMAND}}
env-script = "eval $(rose task-env)"
submission retry delays = {{RETRIES}}
execution time limit = PT30M
[[[events]]]
handlers = notify.py
handler events = submisison retry, submission failed, submission timeout, retry, failed, timeout
[[[environment]]]
ARCHIVE_DIR = {{ARCHIVE_DIR}}
DATA_DIR = {{DATA_DIR}}
PLOT_DIR = {{PLOT_DIR}}
# Log archiving
[[LOGS]]
[[[environment]]]
ROSE_TASK_APP = process_logs
# Rsync needs to run on login nodes
[[archive_logs]]
inherit = None, LOGS
platform = ln04_bg
pre-script = "module load gct"
execution retry delays = {{RETRIES}}
submission retry delays = {{RETRIES}}
execution time limit = PT6H
[[check_logs]]
inherit = None, LOGS
platform = sci_bg
[[sync_puma2_logs]]
inherit = None, LOGS
platform = localhost
execution time limit = PT3H
[[untar_logs]]
inherit = None, LOGS
platform = lotus
execution time limit = PT3H
[[[directives]]]
--partition=short-serial
# Log processing
[[PROCESS]]
pre-script = "module load jaspy"
[[[environment]]]
ROSE_TASK_APP = process_logs
[[concat_logs]]
inherit = None, PROCESS
platform = sci_bg
[[process_logs]]
inherit = None, PROCESS
platform = lotus
execution time limit = PT3H
[[[directives]]]
--partition=short-serial
# Analysis
[[ANALYSIS]]
platform = sci_bg
pre-script = "module load jaspy"
[[[environment]]]
ROSE_TASK_APP = analyse_data
[[analyse_data]]
inherit = None, ANALYSIS
# Housekeeping
[[housekeeping]]
platform = localhost