Merge pull request #27243 from baszalmstra/jolt-physics3

Adds rattler-build recipe compatibility (and adds jolt-physics)

.ci_support/build_all.py

@@ -1,3 +1,5 @@
+from shutil import rmtree
+import tempfile
 import conda.base.context
 import conda.core.index
 import conda.resolve
@@ -6,6 +8,7 @@
 import networkx as nx
 from compute_build_graph import construct_graph
 import argparse
+import atexit
 import re
 import os
 from collections import OrderedDict
@@ -19,6 +22,9 @@
     from yaml import BaseLoader, load
 
 
+EXAMPLE_RECIPE_FOLDERS = ["example", "example-new-recipe"]
+
+
 def get_host_platform():
     from sys import platform
     if platform == "linux" or platform == "linux2":
@@ -44,17 +50,32 @@ def build_all(recipes_dir, arch):
     script_dir = os.path.dirname(os.path.realpath(__file__))
     variant_config_file = os.path.join(script_dir, "{}.yaml".format(get_config_name(arch)))
 
+    has_meta_yaml = False
+    has_recipe_yaml = False
+
     found_cuda = False
     found_centos7 = False
     for folder in folders:
         meta_yaml = os.path.join(recipes_dir, folder, "meta.yaml")
         if os.path.exists(meta_yaml):
+            has_meta_yaml = True
             with(open(meta_yaml, "r", encoding="utf-8")) as f:
                 text = ''.join(f.readlines())
                 if 'cuda' in text:
                     found_cuda = True
                 if 'sysroot_linux-64' in text:
                     found_centos7 = True
+
+        recipe_yaml = os.path.join(recipes_dir, folder, "recipe.yaml")
+        if os.path.exists(recipe_yaml):
+            has_recipe_yaml = True
+            with open(recipe_yaml, "r", encoding="utf-8") as f:
+                text = "".join(f.readlines())
+                if "cuda" in text:
+                    found_cuda = True
+                if "sysroot_linux-64" in text:
+                    found_centos7 = True
+
         cbc = os.path.join(recipes_dir, folder, "conda_build_config.yaml")
         if os.path.exists(cbc):
             with open(cbc, "r") as f:
@@ -72,6 +93,11 @@ def build_all(recipes_dir, arch):
                         print(f"Found c_stdlib_version for linux: {version=}")
                         found_centos7 |= version == (2, 17)
 
+    if has_meta_yaml and has_recipe_yaml:
+        raise ValueError("Mixing meta.yaml and recipe.yaml recipes is not supported")
+    if not has_meta_yaml and not has_recipe_yaml:
+        raise ValueError("Neither a meta.yaml or a recipe.yaml recipes was found")
+
     if found_cuda:
         print('##vso[task.setvariable variable=NEED_CUDA;isOutput=true]1')
     if found_centos7:
@@ -147,8 +173,17 @@ def build_all(recipes_dir, arch):
 
     if 'conda-forge' not in channel_urls:
         raise ValueError('conda-forge needs to be part of channel_sources')
-    print("Building {} with {}".format(','.join(folders), ','.join(channel_urls)))
-    build_folders(recipes_dir, folders, arch, channel_urls)
+
+    if has_meta_yaml:
+        print("Building {} with {}".format(','.join(folders), ','.join(channel_urls)))
+        build_folders(recipes_dir, folders, arch, channel_urls)
+    elif has_recipe_yaml:
+        print(
+            "Building {} with {} using rattler-build".format(
+                ",".join(folders), ",".join(channel_urls)
+            )
+        )
+        build_folders_rattler_build(recipes_dir, platform, arch, channel_urls)
 
 
 def get_config(arch, channel_urls):
@@ -207,6 +242,50 @@ def build_folders(recipes_dir, folders, arch, channel_urls):
         conda_build.api.build([recipe], config=get_config(arch, channel_urls))
 
 
+def build_folders_rattler_build(
+    recipes_dir: str, platform, arch, channel_urls: list[str]
+):
+    config = get_config(arch, channel_urls)
+
+    # Remove the example recipes to ensure that they are not also build.
+    for example_recipe in EXAMPLE_RECIPE_FOLDERS:
+        rmtree(os.path.join(recipes_dir, example_recipe), ignore_errors=True)
+
+    # Determine the locations for the variant config files.
+    specs = OrderedDict()
+    for f in config.exclusive_config_files:
+        specs[f] = conda_build.variants.parse_config_file(
+            os.path.abspath(os.path.expanduser(os.path.expandvars(f))), config
+        )
+
+    # Combine all the variant config files together
+    combined_spec = conda_build.variants.combine_specs(specs, log_output=config.verbose)
+    variant_config = yaml.dump(combined_spec)
+
+    # Define the arguments for rattler-build
+    args = [
+        "rattler-build",
+        "build",
+        "--recipe-dir",
+        recipes_dir,
+        "--target-platform",
+        f"{platform}-{arch}",
+    ]
+    for channel_url in channel_urls:
+        # Local is automatically added by rattler-build so we just remove it.
+        if channel_url != "local":
+            args.extend(["-c", channel_url])
+
+    # Construct a temporary file where we write the combined variant config. We can then pass that
+    # to rattler-build.
+    with tempfile.NamedTemporaryFile(delete=False) as fp:
+        fp.write(variant_config.encode("utf-8"))   
+        atexit.register(os.unlink, fp.name)
+
+    # Execute rattler-build.
+    subprocess.run(args + ["--variant-config", fp.name], check=True)
+
+
 def check_recipes_in_correct_dir(root_dir, correct_dir):
     from pathlib import Path
     for path in Path(root_dir).rglob('meta.yaml'):
@@ -228,7 +307,7 @@ def read_mambabuild(recipes_dir):
     folders = os.listdir(recipes_dir)
     conda_build_tools = []
     for folder in folders:
-        if folder == "example":
+        if folder in EXAMPLE_RECIPE_FOLDERS:
             continue
         cf = os.path.join(recipes_dir, folder, "conda-forge.yml")
         if os.path.exists(cf):

.ci_support/requirements.txt

@@ -6,3 +6,4 @@ conda-forge-ci-setup=4.*
 conda-forge-pinning
 frozendict
 networkx=2.4
+rattler-build-conda-compat>=1.2.0,<2.0.0a0

.github/workflows/correct_directory.yml

@@ -31,6 +31,7 @@ jobs:
                     '\nUnfortunately, the recipe was added directly in the `recipes` folder without its own subfolder.\n' +
                     'Please move the recipe file into a folder with the name of the package you want to submit.\n\n' +
                     'For example: if your recipe is currently under `recipes/<your_package>.yaml`, ' +
-                    'it should be moved to `recipes/<your_package>/meta.yaml`.\n' +
+                    'it should be moved to `recipes/<your_package>/meta.yaml` ' +
+                    'or, if you are using the recipe v1 format, `recipes/<your_package>/recipe.yaml`\n' +
                     'Thanks!'
             })

.github/workflows/scripts/create_feedstocks

@@ -39,14 +39,14 @@ source ~/Miniforge3/bin/activate
 
 conda install --yes --quiet \
   conda-forge-ci-setup=4.* \
-  "conda-smithy>=3.37.0,<4.0.0a0" \
+  "conda-smithy>=3.38.0,<4.0.0a0" \
   conda-forge-pinning \
   "conda-build>=24.3" \
   "gitpython>=3.0.8,<3.1.20" \
   requests \
   ruamel.yaml \
   "pygithub>=2.1.1" \
-  "rattler-build-conda-compat>=0.0.6,<0.1"
+  "rattler-build-conda-compat>=1.2.0,<2.0.0a0"
 
 conda info
 mamba info

.github/workflows/scripts/create_feedstocks.py

@@ -61,7 +61,7 @@ def list_recipes() -> Iterator[tuple[str, str]]:
         # to be helpful.
         # .DS_Store is created by macOS to store custom attributes of its
         # containing folder.
-        if recipe_dir.name in ["example", ".DS_Store"]:
+        if recipe_dir.name in ["example", "example-new-recipe", ".DS_Store"]:
             continue
 
         # Try to look for a conda-build recipe.

README.md

@@ -16,7 +16,7 @@ If the issue persists, support can be found [on Gitter](https://gitter.im/conda-
 ## Getting started
 
 1. Fork this repository.
-2. Make a new folder in `recipes` for your package. Look at the example recipe, our [documentation](http://conda-forge.org/docs/maintainer/adding_pkgs.html#) and the [FAQ](https://github.com/conda-forge/staged-recipes#faq)  for help.
+2. Make a new folder in `recipes` for your package. Look at the example recipes, our [documentation](http://conda-forge.org/docs/maintainer/adding_pkgs.html#) and the [FAQ](https://github.com/conda-forge/staged-recipes#faq) for help.
 3. Open a pull request. Building of your package will be tested on Windows, Mac and Linux.
 4. When your pull request is merged a new repository, called a feedstock, will be created in the github conda-forge organization, and build/upload of your package will automatically be triggered. Once complete, the package is available on conda-forge.
 

recipes/example-new-recipe/recipe.yaml

@@ -0,0 +1,101 @@
+# This example shows how to define a recipe using the new YAML based recipe format introduced by 
+# CEP 13.
+
+# For more information about this format see: https://prefix-dev.github.io/rattler-build/latest/reference/recipe_file/
+
+# The main differences with the old format is that no preprocessing is required for the file to be valid YAML.
+# This means:
+# - No "selectors", use YAML if-then-else expressions instead (https://prefix-dev.github.io/rattler-build/latest/selectors/)
+# - Jinja expressions are formatted with `${{}}`
+
+# Note: there are many handy hints in comments in this example -- remove them when you've finalized your recipe
+
+# Define variables in this section that you can use in other parts.
+context:
+  name: simplejson
+  version: "3.8.2"
+
+package:
+  name: ${{ name|lower }}
+  version: ${{ version }}
+
+source:
+  url: https://pypi.io/packages/source/${{ name[0] }}/${{ name }}/${{ name }}-${{ version }}.tar.gz
+  # If getting the source from GitHub, remove the line above,
+  # uncomment the line below, and modify as needed. Use releases if available:
+  # url: https://github.com/simplejson/simplejson/releases/download/${{ version }}/simplejson-${{ version }}.tar.gz
+  # and otherwise fall back to archive:
+  # url: https://github.com/simplejson/simplejson/archive/v${{ version }}.tar.gz
+  sha256: d58439c548433adcda98e695be53e526ba940a4b9c44fb9a05d92cd495cdd47f
+  # sha256 is the preferred checksum -- you can get it for a file with:
+  #  `openssl sha256 <file name>`,
+  # or, in one go without keeping the file with:
+  # `curl -L <url> | sha256
+  # You may need the packages or openssl and/or curl, available on conda-forge:
+  #  `conda install openssl curl -c conda-forge``
+
+build:
+  # Uncomment the following line if the package is pure Python and the recipe is exactly the same for all platforms.
+  # It is okay if the dependencies are not built for all platforms/versions, although selectors are still not allowed.
+  # See https://conda-forge.org/docs/maintainer/knowledge_base.html#noarch-python for more details.
+  # noarch: python
+  # If the installation is complex, or different between Unix and Windows, use separate build.bat and build.sh files instead of this key.
+  # By default, the package will be built for the Python versions supported by conda-forge and for all major OSs.
+  # Uncomment the following lines to limit to Python 3.5 and newer (for example)
+  # skip:
+  #   - match(python, "<3.5")
+  # or the following to limit to Windows.
+  # skip:
+  #   - win
+  script: python -m pip install . -vv
+  number: 0
+
+requirements:
+  build:
+    # If your project compiles code (such as a C extension) then add the required compilers as separate entries here.
+    # Compiler names include 'c', 'cxx' and 'fortran', among others.
+    - ${{ compiler('c') }}
+    - ${{ stdlib('c') }}  # If you need any compiler, add the C standard library ("stdlib") too
+  host:
+    - python
+    - pip
+  run:
+    - python
+
+tests:
+  # Some packages might need a `test/commands` key to check CLI.
+  - python:
+      # A list of modules that the test will try to import
+      imports:
+      - simplejson
+      - simplejson.tests
+      # Also run `pip check` to verify the integrity 
+      pip_check: true
+
+about:
+  homepage: https://github.com/simplejson/simplejson
+  summary: 'Simple, fast, extensible JSON encoder/decoder for Python'
+  description: |
+    simplejson is a simple, fast, complete, correct and extensible
+    JSON <https://json.org> encoder and decoder for Python 2.5+ and
+    Python 3.3+. It is pure Python code with no dependencies, but includes
+    an optional C extension for a serious speed boost.
+  # Remember to specify the license variants for BSD, Apache, GPL, and LGPL.
+  # Use the SPDX identifier, e.g: GPL-2.0-only instead of GNU General Public License version 2.0
+  # See https://spdx.org/licenses/
+  license: MIT
+  # It is required to include a license file in the package,
+  # (even if the license doesn't require it) using the license_file entry.
+  # Please also note that some projects have multiple license files which all need to be added using a valid yaml list.
+  # See https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#license-file
+  license_file: LICENSE.txt
+  # The documentation and repository URLs are optional.
+  documentation: https://simplejson.readthedocs.io/
+  repository: https://github.com/simplejson/simplejson
+
+extra:
+  recipe-maintainers:
+    # GitHub IDs for maintainers of the recipe.
+    # Always check with the people listed below if they are OK becoming maintainers of the recipe. (There will be spam!)
+    - LisaSimpson
+    - LandoCalrissian

recipes/jolt-physics/recipe.yaml

@@ -0,0 +1,63 @@
+context:
+  version: 5.1.0
+
+package:
+  name: jolt-physics
+  version: ${{ version }}
+
+source:
+  url: https://github.com/jrouwe/JoltPhysics/archive/refs/tags/v${{ version }}.zip
+  sha256: 10fcc863ae2b9d48c2f22d8b0204034820e57a55f858b7c388ac9579d8cf4095
+
+requirements: 
+  build: 
+    - ${{ compiler("cxx") }}
+    - ${{ stdlib("c") }}
+    - cmake
+    - ninja
+
+build:
+  number: 0
+  script:
+    - if: win
+      then: |
+        cmake -GNinja ^
+            %CMAKE_ARGS% ^
+            -DCMAKE_INSTALL_PREFIX=%LIBRARY_PREFIX% ^
+            -DBUILD_SHARED_LIBS=ON ^
+            -DCMAKE_BUILD_TYPE=Distribution ^
+            -DCROSS_PLATFORM_DETERMINISTIC=ON ^
+            -DTARGET_VIEWER=OFF ^
+            -DTARGET_SAMPLES=OFF ^
+            -DTARGET_HELLO_WORLD=OFF ^
+            -DTARGET_UNIT_TESTS=OFF ^
+            -DTARGET_PERFORMANCE_TEST=OFF ^
+            -S %SRC_DIR%\Build
+        cmake --build . --target install
+      else: |
+        cmake -GNinja \
+            $CMAKE_ARGS \
+            -DCMAKE_INSTALL_PREFIX=$PREFIX \
+            -DBUILD_SHARED_LIBS=ON \
+            -DCMAKE_BUILD_TYPE=Distribution \
+            -DCROSS_PLATFORM_DETERMINISTIC=ON \
+            -DTARGET_VIEWER=OFF \
+            -DTARGET_SAMPLES=OFF \
+            -DTARGET_HELLO_WORLD=OFF \
+            -DTARGET_UNIT_TESTS=OFF \
+            -DTARGET_PERFORMANCE_TEST=OFF \
+            -S $SRC_DIR/Build
+        cmake --build . --target install
+
+about:
+  homepage: https://github.com/jrouwe/JoltPhysics
+  license: MIT
+  license_file: LICENSE
+  summary: A multi core friendly rigid body physics and collision detection library.
+  description: A multi core friendly rigid body physics and collision detection library. Written in C++. Suitable for games and VR applications. Used by Horizon Forbidden West. 
+  documentation: https://jrouwe.github.io/JoltPhysics/
+  repository: https://github.com/jrouwe/JoltPhysics
+
+extra:
+  recipe-maintainers:
+    - baszalmstra