From 72278da9464cee10fa0fe4d651e9f89ca43d410d Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Mon, 12 Aug 2024 13:47:37 +0100 Subject: [PATCH 1/6] refactor: update packaging content + better site infra --- .github/workflows/build_site.yml | 41 +++--- .gitignore | 3 + Gemfile | 3 +- README.md | 7 + ch04packaging/03Packaging.ipynb.py | 222 ++++++++++++++++++----------- 5 files changed, 174 insertions(+), 102 deletions(-) diff --git a/.github/workflows/build_site.yml b/.github/workflows/build_site.yml index b3a89731..f79ec17e 100644 --- a/.github/workflows/build_site.yml +++ b/.github/workflows/build_site.yml @@ -10,18 +10,18 @@ jobs: Build-website: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Adds keys to work with other repositories used in this lesson (e.g., UCL/github-example) - - uses: webfactory/ssh-agent@v0.5.4 + - uses: webfactory/ssh-agent@v0.9.0 with: ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - uses: actions/cache@v2 # FIXME: add apt(latex) + - uses: actions/cache@v4 # FIXME: add apt(latex) with: path: vendor/bundle key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile') }} restore-keys: | ${{ runner.os }}-gems- - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt', '.github/python/requirements.txt') }} @@ -31,23 +31,28 @@ jobs: uses: DanySK/setup-texlive-action@0.1.1 - id: setup-python name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.12 - name: Install dependencies - uses: py-actions/py-dependency-install@v2 - with: - path: ".github/python/requirements.txt" + run: python -m pip install -r .github/python/requirements.txt - name: Building notes run: make ready - - name: Builds website - uses: helaili/jekyll-action@v2 + - uses: ruby/setup-ruby@v1 with: - target_branch: 'gh-pages' - build_only: true - - name: Build website and deploy - if: github.ref == 'refs/heads/main' - uses: helaili/jekyll-action@v2 + ruby-version: '3.3' + - name: Install apt requirements + run: gem install bundler + - name: Install apt requirements + run: bundle exec jekyll build + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 with: - target_branch: 'gh-pages' - token: ${{ secrets.GITHUB_TOKEN }} + path: _site + - name: Deploy to GitHub Pages + if: > + success() + && github.ref == 'refs/heads/main' + && github.repository == 'UCL/rsd-engineeringcourse' + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 7d18429b..a33f613f 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,6 @@ master.zip session04/greetings/Greetings.egg-info/ session04/greetings/doc/ session04/greetings/scripts/ +Gemfile.lock +.env/ + diff --git a/Gemfile b/Gemfile index 6ad3a913..0688490b 100644 --- a/Gemfile +++ b/Gemfile @@ -1,5 +1,6 @@ source 'https://rubygems.org' -gem 'jekyll', '~> 4.2.1' +gem "jekyll" gem 'kramdown', '~> 2.3.1' gem "jekyll-remote-theme" +gem "jekyll-sass-converter" diff --git a/README.md b/README.md index fd6194e4..831e376b 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,13 @@ $ docker run --rm --volume="$PWD:/srv/jekyll" --volume="$PWD/vendor/bundle:/usr/ and open http://localhost:4000/rsd-engineeringcourse (or the link provided). Note that this is mounting the `bundle` directory where `act` will create them. +or if you want to simply build and serve the site locally (with `ruby` and `bundle` installed): + +```bash +$ bundle install +$ bundle exec jekyll serve +``` + # Migration from jupyter notebooks to py:percent Using `jupytext` we've converted all the jupyter notebooks into plain text python files (py:percent) with: diff --git a/ch04packaging/03Packaging.ipynb.py b/ch04packaging/03Packaging.ipynb.py index a0b97411..da065962 100644 --- a/ch04packaging/03Packaging.ipynb.py +++ b/ch04packaging/03Packaging.ipynb.py @@ -21,6 +21,18 @@ # A good cross-platform build tool is the most important thing: you can always # have collaborators build from source. # +# Along with the content below, the Python ecosystem has a few fantastic guides on both simple +# and complex packaging. Some of the recommended ones are listed below - +# - [Python Packaging User Guide](https://packaging.python.org/en/latest/) +# - [pyOpenSci Python Package Guide](https://www.pyopensci.org/python-package-guide/) +# - [Scientific Python Packaging Guide]https://learn.scientific-python.org/development/tutorials/packaging/ +# +# Besides the guides, there are numerous "cookie" templates available for developers. These templates +# allow developers to generate an empty Python package following good practices and guidelines with a +# simple CLI command. +# - [Scientific Python cookie](https://github.com/scientific-python/cookie) +# - [Open Science Labs' scicookie](https://github.com/osl-incubator/scicookie) +# - Domain/ecosystem specific cookies like [pybamm-cookiecutter](https://github.com/pybamm-team/pybamm-cookiecutter) for battery modeling projects in Python exist too # %% [markdown] # ### Distribution tools @@ -28,16 +40,16 @@ # %% [markdown] # Distribution tools allow one to obtain a working copy of someone else's package. # -# - Language-specific tools: -# - python: PyPI, -# - ruby: Ruby Gems, -# - perl: CPAN, -# - R: CRAN +# - Language-specific tools: +# - python: PyPI, +# - ruby: Ruby Gems, +# - perl: CPAN, +# - R: CRAN # -# - Platform specific packagers e.g.: -# - [`brew`](https://brew.sh/) for MacOS, -# - `apt`/`dnf`/`pacman` for Linux or -# - [`choco`](https://chocolatey.org/) for Windows. +# - Platform specific packagers e.g.: +# - [`brew`](https://brew.sh/) for MacOS, +# - `apt`/`dnf`/`pacman` for Linux or +# - [`choco`](https://chocolatey.org/) for Windows. # %% [markdown] # ### Laying out a project @@ -45,19 +57,21 @@ # %% [markdown] # # When planning to package a project for distribution, defining a suitable -# project layout is essential. A typical layout might look like this: +# project layout is essential. A typical scientific python compliant layout +# might look like this: # # ``` # repository_name -# |-- module_name -# | |-- __init__.py -# | |-- python_file.py -# | |-- another_python_file.py -# | `-- test -# | |-- fixtures -# | | `-- fixture_file.yaml +# |-- src +# | `-- package_name # | |-- __init__.py -# | `-- test_python_file.py +# | |-- python_file.py +# | |-- another_python_file.py +# `-- tests +# |-- fixtures +# | `-- fixture_file.yaml +# |-- __init__.py +# `-- test_python_file.py # |-- LICENSE.md # |-- CITATION.md # |-- README.md @@ -72,7 +86,8 @@ # To achieve this for our `greetings.py` file from the previous session, we can use the commands shown below. We can start by making our directory structure. You can create many nested directories at once using the `-p` switch on `mkdir`. # %% language="bash" -# mkdir -p greetings_repo/greetings/test/fixtures +# mkdir -p greetings_repo/src/greetings +# mkdir -p greetings_repo/tests/fixtures # %% [markdown] # For this notebook, since we are going to be modifying the files bit by bit, we are going to use the [autoreload ipython magic](https://ipython.readthedocs.io/en/stable/config/extensions/autoreload.html) so that we don't need to restart the kernel. @@ -94,30 +109,45 @@ # This information is included in the `project` section of our `pyproject.toml` file. # # But we also need to tell users how to build the package from these specifications. -# This information is specified in the `build-system` section of our `toml` file. -# In this case, we'll be using `setuptools` to build our package, so we list it in the `requires` field. -# We also need `setuptools_scm[toml]` so that `setuptools` can understand the settings we give it in our `.toml` file, and `wheel` to make the package distribution. -# -# Finally, we can set specific options for `setuptools` using additional sections in `pyproject.toml`: in this case, we will tell `setuptools` that it needs to find **and include** all of the files in our `greetings` folder. +# This information is specified in the `build-system` section of our `toml` file. Python packages are shipped on PyPI as "wheel" files, +# which are installable via `pip` and are generated by build backends. Wheels can be different for different platforms +# and different Python versions, and `pip` only resorts to installing a library through its source distribution (often referred as `SDist`) +# if it fails to find a compatible wheel file on PyPI. Python ecosystem houses a number of build backends, each of them spacializing in a different task. +# Some common Python build backends - +# - [`setuptools`](https://setuptools.pypa.io): allows building pure Python and Python + C/C++ projects +# - [`hatch`](https://hatch.pypa.io): allows building (and recommended for) pure Python projects +# - [`flit`](https://flit.pypa.io) allows building pure Python projects with minimal extra configurations +# - [`poetry`](https://python-poetry.org) allows building pure Python projects (a full blown dependency and environment management system) +# - [`scikit-build-core`](https://scikit-build-core.readthedocs.io) allows building pure Python and Python + C/C++ projects (under active development) +# - [`meson`](https://meson-python.readthedocs.io) allows building pure Python and Python + C/C++ projects (has a custom DSL) +# - [`maturin`](https://www.maturin.rs): allows building Rust binary extensions +# +# In this case, we'll be using `hatch` to build our package, so we list it in the `requires` field. Technically speaking, `hatch` is the front-end (a CLI utility) +# for the actual build-backend `hatchling`. `hatchling` is installed with hatch and can be specified as the `build-backend` in `pyproject.toml`. +# +# Finally, we can set specific options for `hatch` using additional sections in `pyproject.toml`: in this case, we will tell `hatch` that it needs to find **and include** all of the files in our `greetings` folder. +# The best way to look at all the options of a build-backend is by going through its documentation. # %% # %%writefile greetings_repo/pyproject.toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + [project] name = "Greetings" version = "0.1.0" -[build-system] -requires = ["setuptools", "setuptools_scm[toml]>=6.2", "wheel"] - -[tool.setuptools.packages.find] -include = ["greetings*"] - -# Add setuptools_scm if you need to generate version numbers from the git hash -#[tool.setuptools_scm] +[tool.hatch.build.targets.sdist] +include = [ + "src*", +] # %% [markdown] -# We can now install this "package" with pip: +# Some of the build-backends allow users to automate the package's version using VCS. +# For instance, you might want to look into [`hatch-vcs`](https://github.com/ofek/hatch-vcs) to enable VCS versioning with `hatch`. +# We can now install this "package" with pip (make sure `hatch` is installed): # %% language="bash" # cd greetings_repo @@ -138,13 +168,13 @@ # %% language="bash" # -# touch greetings_repo/greetings/__init__.py +# touch greetings_repo/src/greetings/__init__.py # %% [markdown] # And we can copy the `greet` function from the [previous section](https://github-pages.ucl.ac.uk/rsd-engineeringcourse/ch04packaging/02Argparse.html) in the `greeter.py` file. # %% -# %%writefile greetings_repo/greetings/greeter.py +# %%writefile greetings_repo/src/greetings/greeter.py def greet(personal, family, title="", polite=False): greeting = "How do you do, " if polite else "Hey, " @@ -168,7 +198,8 @@ def greet(personal, family, title="", polite=False): # %% from greetings.greeter import greet -greet("Terry","Gilliam") + +greet("Terry", "Gilliam") # %% [markdown] @@ -182,7 +213,7 @@ def greet(personal, family, title="", polite=False): # We need to add docstrings to our functions, so people can know how to use them. # %% -# %%writefile greetings_repo/greetings/greeter.py +# %%writefile greetings_repo/src/greetings/greeter.py def greet(personal, family, title="", polite=False): """ Generate a greeting string for a person. @@ -242,7 +273,7 @@ def greet(personal, family, title="", polite=False): # Note how we are importing `greet` using [relative imports](https://www.python.org/dev/peps/pep-0328/), where `.greeter` means to look for a `greeter` module within the same directory. # %% -# %%writefile greetings_repo/greetings/command.py +# %%writefile greetings_repo/src/greetings/command.py from argparse import ArgumentParser @@ -259,8 +290,14 @@ def process(): arguments = parser.parse_args() - print(greet(arguments.personal, arguments.family, - arguments.title, arguments.polite)) + print( + greet( + arguments.personal, + arguments.family, + arguments.title, + arguments.polite + ) + ) if __name__ == "__main__": @@ -277,6 +314,10 @@ def process(): # %% # %%writefile greetings_repo/pyproject.toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + [project] name = "Greetings" version = "0.1.0" @@ -284,18 +325,14 @@ def process(): [project.scripts] greet = "greetings.command:process" -[build-system] -requires = ["setuptools", "setuptools_scm[toml]>=6.2", "wheel"] - -[tool.setuptools.packages.find] -include = ["greetings*"] - -# Add setuptools_scm if you need to generate version numbers from the git hash -#[tool.setuptools_scm] +[tool.hatch.build.targets.sdist] +include = [ + "greetings/", +] # %% language="bash" # cd greetings_repo -# pip install -e . +# pip install . # %% [markdown] # @@ -319,7 +356,7 @@ def process(): # Let's give some life to our output using ascii art # %% -# %%writefile greetings_repo/greetings/command.py +# %%writefile greetings_repo/src/greetings/command.py from argparse import ArgumentParser @@ -342,6 +379,7 @@ def process(): arguments.title, arguments.polite) print(art("cute face"), message) + if __name__ == "__main__": process() @@ -352,6 +390,10 @@ def process(): # %% # %%writefile greetings_repo/pyproject.toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + [project] name = "Greetings" version = "0.1.0" @@ -362,21 +404,18 @@ def process(): [project.scripts] greet = "greetings.command:process" -[build-system] -requires = ["setuptools", "setuptools_scm[toml]>=6.2", "wheel"] - -[tool.setuptools.packages.find] -include = ["greetings*"] +[tool.hatch.build.targets.sdist] +include = [ + "greetings/", +] -# Add setuptools_scm if you need to generate version numbers from the git hash -#[tool.setuptools_scm] # %% [markdown] # When installing the package now, pip will also install the dependencies automatically. # %% language="bash" # cd greetings_repo -# pip install -e . +# pip install . # %% language="bash" # greet Terry Gilliam @@ -386,7 +425,7 @@ def process(): # %% [markdown] # -# We could now submit "greeter" to PyPI for approval, so everyone could `pip install` it. +# We could now submit "greeter" to PyPI, so everyone could `pip install` it. # # However, when using git, we don't even need to do that: we can install directly from any git URL: # @@ -468,7 +507,7 @@ def process(): Portions of the material are taken from [Software Carpentry](http://software-carpentry.org/) # %% [markdown] -# You may well want to formalise this using the [codemeta.json](https://codemeta.github.io/) standard or the [citation file format](http://citation-file-format.github.io/) - these don't have wide adoption yet, but we recommend it. +# You may well want to formalise this using the [codemeta.json](https://codemeta.github.io/) standard or the [citation file format](http://citation-file-format.github.io/). # %% [markdown] # ### Define packages and executables @@ -476,8 +515,8 @@ def process(): # %% [markdown] # We need to create `__init__` files for the source and the tests. # ```bash -# touch greetings/greetings/test/__init__.py -# touch greetings/greetings/__init__.py +# touch greetings_repo/tests/__init__.py +# touch greetings_repo/src/greetings/__init__.py # ``` # %% [markdown] @@ -489,7 +528,7 @@ def process(): # Remember, that we need to create the empty `__init__.py` files so that `pytest` can follow the relative imports. # %% language="bash" -# touch greetings_repo/greetings/test/__init__.py +# touch greetings_repo/tests/__init__.py # %% [markdown] # @@ -502,18 +541,22 @@ def process(): # # %% -# %%writefile greetings_repo/greetings/test/test_greeter.py +# %%writefile greetings_repo/tests/test_greeter.py import os import yaml -from ..greeter import greet +from greetings.greeter import greet def test_greet(): - with open(os.path.join(os.path.dirname(__file__), - 'fixtures', - 'samples.yaml')) as fixtures_file: + with open( + os.path.join( + os.path.dirname(__file__), + 'fixtures', + 'samples.yaml' + ) + ) as fixtures_file: fixtures = yaml.safe_load(fixtures_file) for fixture in fixtures: answer = fixture.pop('answer') @@ -533,7 +576,7 @@ def test_greet(): # # %% -# %%writefile greetings_repo/greetings/test/fixtures/samples.yaml +# %%writefile greetings_repo/tests/fixtures/samples.yaml - personal: Eric family: Idle @@ -559,19 +602,23 @@ def test_greet(): # However, this hasn't told us that also the third test is wrong too! A better aproach is to parametrize the testfile `greetings_repo/greetings/test/test_greeter.py` as follows: # %% -# %%writefile greetings_repo/greetings/test/test_greeter.py +# %%writefile greetings_repo/tests/test_greeter.py import os import pytest import yaml -from ..greeter import greet +from greetings.greeter import greet def read_fixture(): - with open(os.path.join(os.path.dirname(__file__), - 'fixtures', - 'samples.yaml')) as fixtures_file: + with open( + os.path.join( + os.path.dirname(__file__), + 'fixtures', + 'samples.yaml' + ) + ) as fixtures_file: fixtures = yaml.safe_load(fixtures_file) return fixtures @@ -599,7 +646,8 @@ def test_greeter(fixture): # %% [markdown] # Finally, we typically don't want to include the tests when we distribute our software for our users. -# We can make sure they are not included using the `exclude` option on when telling `setuptools` to find packages. +# We can make sure they are not included using the `exclude` option on when telling `hatch` to find packages. +# We can also add pytest as an "optional" dependency for the developers of our package. # # Additionally, we can make sure that our README and LICENSE are included in our package metadata by declaring them in the `readme` and `license` fields under the `project` section. # If you're using a particularly common or standard license, you can even provide the name of the license, rather than the file, and your package builder will take care of the rest! @@ -607,11 +655,15 @@ def test_greeter(fixture): # %% # %%writefile greetings_repo/pyproject.toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + [project] name = "Greetings" version = "0.1.0" -readme = "README.md" -license = { file = "LICENSE.md" } +readme = { file = "README.md", content-type = "text/markdown" } +license-files = { paths = ["LICENSE.md"] } dependencies = [ "art", "pyyaml", @@ -620,15 +672,13 @@ def test_greeter(fixture): [project.scripts] greet = "greetings.command:process" -[build-system] -requires = ["setuptools", "setuptools_scm[toml]>=6.2", "wheel"] +[project.optional-dependencies] +dev = ["pytest >= 6"] -[tool.setuptools.packages.find] -include = ["greetings*"] +[tool.hatch.build.targets.sdist] +include = ["src*"] exclude = ["tests*"] -# Add setuptools_scm if you need to generate version numbers from the git hash -#[tool.setuptools_scm] # %% [markdown] # ### Developer Install @@ -651,6 +701,12 @@ def test_greeter(fixture): # ```bash # pip install -e . # ``` +# +# with installing the `dev` dependencies: +# +# ```bash +# pip install -e ".[dev]" +# ``` # %% [markdown] # ### Distributing compiled code From 57ac03f77ed716909b1c7ebe3646d283aa1f3d1d Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Mon, 12 Aug 2024 13:50:01 +0100 Subject: [PATCH 2/6] Don't install anything on your system --- .github/workflows/build_site.yml | 4 ++-- README.md | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_site.yml b/.github/workflows/build_site.yml index f79ec17e..581e8402 100644 --- a/.github/workflows/build_site.yml +++ b/.github/workflows/build_site.yml @@ -41,9 +41,9 @@ jobs: - uses: ruby/setup-ruby@v1 with: ruby-version: '3.3' - - name: Install apt requirements + - name: Install bundler run: gem install bundler - - name: Install apt requirements + - name: Build website run: bundle exec jekyll build - name: Upload artifact uses: actions/upload-pages-artifact@v3 diff --git a/README.md b/README.md index 831e376b..6ae34ec7 100644 --- a/README.md +++ b/README.md @@ -79,12 +79,6 @@ $ docker run --rm --volume="$PWD:/srv/jekyll" --volume="$PWD/vendor/bundle:/usr/ and open http://localhost:4000/rsd-engineeringcourse (or the link provided). Note that this is mounting the `bundle` directory where `act` will create them. -or if you want to simply build and serve the site locally (with `ruby` and `bundle` installed): - -```bash -$ bundle install -$ bundle exec jekyll serve -``` # Migration from jupyter notebooks to py:percent From bd8d660286ace337a7ad1ce004bcee5ba38e5bf8 Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Mon, 12 Aug 2024 14:05:08 +0100 Subject: [PATCH 3/6] fix: run bundle install --- .github/workflows/build_site.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_site.yml b/.github/workflows/build_site.yml index 581e8402..0dc5c60d 100644 --- a/.github/workflows/build_site.yml +++ b/.github/workflows/build_site.yml @@ -41,8 +41,10 @@ jobs: - uses: ruby/setup-ruby@v1 with: ruby-version: '3.3' - - name: Install bundler - run: gem install bundler + - name: Install bundler and dependencies + run: | + gem install bundler + bundle install - name: Build website run: bundle exec jekyll build - name: Upload artifact From 1a57dbbb920cca353806b637c76fb072b5f52554 Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Mon, 12 Aug 2024 16:30:01 +0100 Subject: [PATCH 4/6] Add PackagingCon --- ch04packaging/03Packaging.ipynb.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ch04packaging/03Packaging.ipynb.py b/ch04packaging/03Packaging.ipynb.py index da065962..334df2f7 100644 --- a/ch04packaging/03Packaging.ipynb.py +++ b/ch04packaging/03Packaging.ipynb.py @@ -33,6 +33,10 @@ # - [Scientific Python cookie](https://github.com/scientific-python/cookie) # - [Open Science Labs' scicookie](https://github.com/osl-incubator/scicookie) # - Domain/ecosystem specific cookies like [pybamm-cookiecutter](https://github.com/pybamm-team/pybamm-cookiecutter) for battery modeling projects in Python exist too +# +# Finally, the packaging community regularly organises [PackagingCon](https://packaging-con.org) to discuss the packaging ecosystems +# of multiple languages and operating systems at a single place. +# # %% [markdown] # ### Distribution tools From 0ad179719c66e70611744dd7d2ee37568674cca5 Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Mon, 12 Aug 2024 16:38:28 +0100 Subject: [PATCH 5/6] More src migration + format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: David Pérez-Suárez --- ch04packaging/03Packaging.ipynb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ch04packaging/03Packaging.ipynb.py b/ch04packaging/03Packaging.ipynb.py index 334df2f7..6b2cd60e 100644 --- a/ch04packaging/03Packaging.ipynb.py +++ b/ch04packaging/03Packaging.ipynb.py @@ -129,7 +129,7 @@ # In this case, we'll be using `hatch` to build our package, so we list it in the `requires` field. Technically speaking, `hatch` is the front-end (a CLI utility) # for the actual build-backend `hatchling`. `hatchling` is installed with hatch and can be specified as the `build-backend` in `pyproject.toml`. # -# Finally, we can set specific options for `hatch` using additional sections in `pyproject.toml`: in this case, we will tell `hatch` that it needs to find **and include** all of the files in our `greetings` folder. +# Finally, we can set specific options for `hatch` using additional sections in `pyproject.toml`: in this case, we will tell `hatch` that it needs to find **and include** all of the files in our `src` folder. # The best way to look at all the options of a build-backend is by going through its documentation. # %% @@ -151,6 +151,7 @@ # %% [markdown] # Some of the build-backends allow users to automate the package's version using VCS. # For instance, you might want to look into [`hatch-vcs`](https://github.com/ofek/hatch-vcs) to enable VCS versioning with `hatch`. +# # We can now install this "package" with pip (make sure `hatch` is installed): # %% language="bash" @@ -410,7 +411,7 @@ def process(): [tool.hatch.build.targets.sdist] include = [ - "greetings/", + "src/", ] From ee3c6ff69ab31d2ba5ae2869133ca1aed9b37144 Mon Sep 17 00:00:00 2001 From: Saransh Chopra Date: Tue, 13 Aug 2024 12:17:22 +0100 Subject: [PATCH 6/6] better package managers and repo + no __init__ and exclude --- ch04packaging/03Packaging.ipynb.py | 63 ++++++++++++++---------------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/ch04packaging/03Packaging.ipynb.py b/ch04packaging/03Packaging.ipynb.py index 6b2cd60e..60dbd0a2 100644 --- a/ch04packaging/03Packaging.ipynb.py +++ b/ch04packaging/03Packaging.ipynb.py @@ -43,17 +43,36 @@ # %% [markdown] # Distribution tools allow one to obtain a working copy of someone else's package. -# -# - Language-specific tools: -# - python: PyPI, -# - ruby: Ruby Gems, -# - perl: CPAN, -# - R: CRAN +# The package managers are usually CLI utilities that allow you to query inside +# a repository of existing packages. +# +# - Language specific package/library managers: +# - python: [pip](https://pip.pypa.io), [pipx](https://pipx.pypa.io), [uv](https://github.com/astral-sh/uv), [conda](https://conda.org) +# - Julia: [Pkg](https://docs.julialang.org/en/v1/stdlib/Pkg/) +# - Rust: [Cargo](https://doc.rust-lang.org/cargo/) +# - R: using install.packages("package_name"), [devtools](https://cran.r-project.org/web/packages/devtools/index.html) +# - ruby: [gem](https://guides.rubygems.org/rubygems-basics/) +# - perl: [cpan](https://metacpan.org/dist/CPAN/view/scripts/cpan), [cpanm](https://metacpan.org/dist/App-cpanminus/view/bin/cpanm), [cpanp](https://metacpan.org/dist/CPANPLUS/view/bin/cpanp) # -# - Platform specific packagers e.g.: +# - Platform specific package/library managers e.g.: # - [`brew`](https://brew.sh/) for MacOS, # - `apt`/`dnf`/`pacman` for Linux or # - [`choco`](https://chocolatey.org/) for Windows. +# +# Every language has a repository or a central database of packages submitted +# by the developers. +# +# - Language-specific repositories: +# - python: [PyPI](https://pypi.org), [conda-forge](https://conda-forge.org) +# - Julia: [JuliaHub](https://juliahub.com/ui/Packages) +# - Rust: [Crates](https://crates.io) +# - R: [CRAN](https://www.cpan.org) +# - ruby: [RubyGems](https://rubygems.org) +# - perl: [CPAN](https://www.cpan.org) +# +# The difference between the package management tools and the package repositories is +# similar to the difference between Git and GitHub. +# # %% [markdown] # ### Laying out a project @@ -68,13 +87,12 @@ # repository_name # |-- src # | `-- package_name -# | |-- __init__.py +# | |-- __init__.py # optional; required for exporting things under package's namespace # | |-- python_file.py # | |-- another_python_file.py # `-- tests # |-- fixtures # | `-- fixture_file.yaml -# |-- __init__.py # `-- test_python_file.py # |-- LICENSE.md # |-- CITATION.md @@ -165,17 +183,12 @@ # %% [markdown] # -# To create a regular package, we needed to have `__init__.py` files on each subdirectory that we want to be able to import. This is, since version 3.3 and the introduction of [Implicit Namespaces Packages](https://www.python.org/dev/peps/pep-0420/), not needed anymore. However, if you want to use relative imports and `pytest`, then you [still need to have these files](https://github.com/pytest-dev/pytest/issues/1927). +# To create a regular package, we needed to have `__init__.py` files on each subdirectory that we want to be able to import. This is, since version 3.3 and the introduction of [Implicit Namespaces Packages](https://www.python.org/dev/peps/pep-0420/), not needed anymore. # # The `__init__.py` files can contain any initialisation code you want to run when the (sub)module is imported. # -# For this example, and because we are using relative imports in the tests, we are creating the needed `__init__.py` files. - -# %% language="bash" +# For this example, we don't need to create the `__init__.py` files. # -# touch greetings_repo/src/greetings/__init__.py - -# %% [markdown] # And we can copy the `greet` function from the [previous section](https://github-pages.ucl.ac.uk/rsd-engineeringcourse/ch04packaging/02Argparse.html) in the `greeter.py` file. # %% @@ -514,15 +527,6 @@ def process(): # %% [markdown] # You may well want to formalise this using the [codemeta.json](https://codemeta.github.io/) standard or the [citation file format](http://citation-file-format.github.io/). -# %% [markdown] -# ### Define packages and executables - -# %% [markdown] -# We need to create `__init__` files for the source and the tests. -# ```bash -# touch greetings_repo/tests/__init__.py -# touch greetings_repo/src/greetings/__init__.py -# ``` # %% [markdown] # ### Write some unit tests @@ -530,13 +534,6 @@ def process(): # %% [markdown] # We can now write some tests to our library. # -# Remember, that we need to create the empty `__init__.py` files so that `pytest` can follow the relative imports. - -# %% language="bash" -# touch greetings_repo/tests/__init__.py - -# %% [markdown] -# # Separating the script from the logical module made this possible. # # @@ -651,7 +648,6 @@ def test_greeter(fixture): # %% [markdown] # Finally, we typically don't want to include the tests when we distribute our software for our users. -# We can make sure they are not included using the `exclude` option on when telling `hatch` to find packages. # We can also add pytest as an "optional" dependency for the developers of our package. # # Additionally, we can make sure that our README and LICENSE are included in our package metadata by declaring them in the `readme` and `license` fields under the `project` section. @@ -682,7 +678,6 @@ def test_greeter(fixture): [tool.hatch.build.targets.sdist] include = ["src*"] -exclude = ["tests*"] # %% [markdown]