|
1 | | -// Obtain files from source control system. |
| 1 | +// JWQL Jenkinsfile |
| 2 | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - |
| 3 | +// |
| 4 | +// Authors: |
| 5 | +// -------- |
| 6 | +// - Matthew Bourque |
| 7 | +// - Lauren Chambers |
| 8 | +// - Joshua Alexander |
| 9 | +// - Sara Ogaz |
| 10 | +// - Matt Rendina |
| 11 | +// |
| 12 | +// Notes: |
| 13 | +// ------ |
| 14 | +// - More info here: https://github.com/spacetelescope/jenkinsfile_ci_examples |
| 15 | +// - Syntax defined here: https://github.com/spacetelescope/jenkins_shared_ci_utils |
| 16 | +// |
| 17 | +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - |
| 18 | +// |
| 19 | +// scm_checkout() does the following: |
| 20 | +// 1. Disables pipeline execution if [skip ci] or [ci skip] is present in the |
| 21 | +// commit message, letting users exclude individual commits from CI |
| 22 | +// 2. Clones the Git repository |
| 23 | +// 3. Creates a local cache of the repository to avoid commit drift between tasks |
| 24 | +// (i.e. Each stage is guaranteed to receive the same source code regardless of |
| 25 | +// commits taking place after the pipeline has started.) |
2 | 26 | if (utils.scm_checkout()) return |
3 | 27 |
|
4 | | -matrix_os = ["linux-stable"] |
| 28 | +// Establish OS and Python version variables for the matrix |
| 29 | +matrix_os = ["linux-stable"] // (Note that Jenkins can only be run with Linux, not MacOSX/Windows) |
5 | 30 | matrix_python = ["3.5", "3.6"] |
| 31 | + |
| 32 | +// Set up the matrix of builds |
6 | 33 | matrix = [] |
7 | 34 |
|
8 | | -withCredentials([string( |
9 | | - credentialsId: 'jwql-codecov', |
10 | | - variable: 'codecov_token')]) { |
11 | | - |
12 | | - for (os in matrix_os) { |
13 | | - for (python_ver in matrix_python) { |
14 | | - // Define each build configuration, copying and overriding values as necessary. |
15 | | - env_py = "_python_${python_ver}".replace(".", "_") |
16 | | - bc = new BuildConfig() |
17 | | - bc.nodetype = os |
18 | | - bc.name = "debug-${os}-${env_py}" |
19 | | - bc.conda_packages = ["python=${python_ver}"] |
20 | | - bc.build_cmds = [ |
21 | | - "conda env update --file=environment${env_py}.yml", |
22 | | - "pip install codecov pytest-cov", |
23 | | - "python setup.py install"] |
24 | | - bc.test_cmds = [ |
25 | | - "pytest -s --junitxml=results.xml --cov=./jwql/ --cov-report=xml:coverage.xml", |
26 | | - "sed -i 's/file=\"[^\"]*\"//g;s/line=\"[^\"]*\"//g;s/skips=\"[^\"]*\"//g' results.xml", |
27 | | - "codecov --token=${codecov_token}", |
28 | | - "mkdir -v reports", |
29 | | - "mv -v coverage.xml reports/coverage.xml"] |
30 | | - matrix += bc |
| 35 | +// Define IDs that live on the Jenkins server (here, for CodeCov and PyPI) |
| 36 | +withCredentials([ |
| 37 | + string(credentialsId: 'jwql-codecov', variable: 'codecov_token'), |
| 38 | + usernamePassword(credentialsId:'jwql-pypi', usernameVariable: 'pypi_username', passwordVariable: 'pypi_password')]) |
| 39 | + |
| 40 | +// Iterate over the above variables to define the build matrix. |
| 41 | +{ |
| 42 | + for (os in matrix_os) { |
| 43 | + for (python_ver in matrix_python) { |
| 44 | + // Define each build configuration, copying and overriding values as necessary. |
| 45 | + |
| 46 | + // Define a string variable to reflect the python version of this build |
| 47 | + env_py = "_python_${python_ver}".replace(".", "_") |
| 48 | + |
| 49 | + // Create a new build configuration |
| 50 | + bc = new BuildConfig() |
| 51 | + |
| 52 | + // Define the OS (only "linux-stable" used here) |
| 53 | + bc.nodetype = os |
| 54 | + |
| 55 | + // Give the build configuration a name. This string becomes the |
| 56 | + // stage header on Jenkins' UI. Keep it short! |
| 57 | + bc.name = "debug-${os}-${env_py}" |
| 58 | + |
| 59 | + // (Required) Define what packages to include in the base conda environment. |
| 60 | + // This specification also tells Jenkins to spin up a new conda environment for |
| 61 | + // your build, rather than using the default environment. |
| 62 | + bc.conda_packages = ["python=${python_ver}"] |
| 63 | + |
| 64 | + // Execute a series of commands to set up the build, including |
| 65 | + // any packages that have to be installed with pip |
| 66 | + bc.build_cmds = [ |
| 67 | + "conda env update --file=environment${env_py}.yml", // Update env from file |
| 68 | + "pip install codecov pytest-cov", // Install additional packages |
| 69 | + "python setup.py install", // Install JWQL package |
| 70 | + "python setup.py sdist bdist_wheel" // Build JWQL pacakge wheel for PyPI |
| 71 | + ] |
| 72 | + |
| 73 | + // Execute a series of test commands |
| 74 | + bc.test_cmds = [ |
| 75 | + // Run pytest |
| 76 | + "pytest ./jwql/tests/ -s --junitxml=results.xml --cov=./jwql/ --cov-report=xml:coverage.xml", |
| 77 | + // Add a truly magical command that makes Jenkins work for Python 3.5 |
| 78 | + "sed -i 's/file=\"[^\"]*\"//g;s/line=\"[^\"]*\"//g;s/skips=\"[^\"]*\"//g' results.xml", |
| 79 | + // Define CodeCov token |
| 80 | + "codecov --token=${codecov_token}", |
| 81 | + // Move the CodeCov report to a different dir to not confuse Jenkins about results.xml |
| 82 | + "mkdir -v reports", |
| 83 | + "mv -v coverage.xml reports/coverage.xml", |
| 84 | + // Update the package wheel to PYPI |
| 85 | + "twine upload -u '${pypi_username}' -p '${pypi_password}' --repository-url https://upload.pypi.org/legacy/ --skip-existing dist/*"] |
| 86 | + |
| 87 | + // Add the build to the matrix |
| 88 | + matrix += bc |
| 89 | + } |
31 | 90 | } |
32 | | - } |
33 | | - // bc1 = utils.copy(bc0) |
34 | | - // bc1.build_cmds[0] = "conda install -q -y python=3.5" |
35 | 91 |
|
36 | | - // Iterate over configurations that define the (distibuted) build matrix. |
37 | | - // Spawn a host of the given nodetype for each combination and run in parallel. |
38 | | - utils.run(matrix) |
| 92 | + // Submit the build configurations and execute them in parallel |
| 93 | + utils.run(matrix) |
39 | 94 | } |
0 commit comments