feat: rMPP support (#58)

* feat: rMPP support

* fix/builder.py: fix arch detection

* feat/test: test for rMPP things too

* fix/builder: use startwith per Eeems' suggestion

* fix/docs: per suggestions

* feat/builder: auto switch aarch64 envvars

* feat: rMPPM support too

* fix: setting build env

* feat: arch test case

* feat: arch test case actually

* fix: run formatter to please linter

* fix: run formatter to please linter, part 2

* fix: run formatter to please linter, part 3

* fix: now black strikes again!

* fix: remove unneeded manual arch switch guide

Author: NoaHimesaka1873

docs/recipe-format.md

@@ -40,9 +40,11 @@ The following values are accepted:
 
 Name    | Meaning
 --------|-------------------------------------------------------------------------
-`rmall` | Packages which work on all reMarkable devices without modification.
+`rmall` | Packages which work on all reMarkable devices without modification. These packages must be cpu architecture independent, which generally means that they do not contain binaries, only scripts and configuration files.
 `rm1`   | Packages requiring reMarkable 1-specific resources or compilation flags.
 `rm2`   | Packages requiring reMarkable 2-specific resources or compilation flags.
+`rmpp`  | Packages requiring reMarkable Paper Pro-specific resources or compilation flags.
+`rmppm` | Packages requiring reMarkable Paper Pro Move-specific resources or compilation flags.
 
 For example, use `archs=(rm1)` for a package that only works on reMarkable 1, or `archs=(rm1 rm2)` for a package that works both on reMarkable 1 and reMarkable 2 but needs different dependencies or compilation flags for each of those.
 

pyproject.toml

@@ -1,9 +1,10 @@
 [project]
 name = "toltecmk"
-version = "0.3.7"
+version = "0.4.0"
 authors = [
   { name="Mattéo Delabre", email="[email protected]" },
   { name="Eeems", email="[email protected]" },
+  { name="Noa Himesaka", email="[email protected]" },
 ]
 description = "Build system used for the Toltec community repository"
 requires-python = ">=3.11"

tests/fixtures/hello/package

@@ -11,8 +11,9 @@ license=MIT
 pkgver=0.0.1-1
 section="utils"
 flags=()
+archs=(rm1 rmpp)
 
-image=base:v2.1
+image=base:v4.0
 source=(hello.c)
 sha256sums=(SKIP)
 

tests/recipe_parsers/test_installdepends.py

@@ -35,7 +35,7 @@ def test_installdepends(self) -> None:
         with open(path.join(rec_path, "package"), "w") as rec_def_file:
             rec_def_file.write(
                 """
-archs=(rmall rmallos2 rmallos3 rm1 rm1os2 rm1os3 rm2 rm2os2 rm2os3)
+archs=(rmall rmallos2 rmallos3 rm1 rm1os2 rm1os3 rm2 rm2os2 rm2os3 rmpp rmppos3 rmppm rmppmos3)
 pkgnames=(toltec-base)
 pkgdesc="Metapackage defining the base set of packages in a Toltec install"
 url=https://toltec-dev.org/
@@ -49,6 +49,10 @@ def test_installdepends(self) -> None:
 installdepends_rm1os3=(open-remarkable-shutdown)
 installdepends_rm2os2=(rm2-suspend-fix)
 installdepends_rm2os3=(rm2-suspend-fix)
+installdepends_rmpp=(rmpp-make-root-rw)
+installdepends_rmppos3=(rmpp-make-root-rw)
+installdepends_rmppm=(rmpp-make-root-rw)
+installdepends_rmppmos3=(rmpp-make-root-rw)
 
 image=base:v2.1
 source=("https://example.org/toltec/${pkgnames[0]}/release-${pkgver%-*}.zip")
@@ -80,6 +84,8 @@ def test_installdepends(self) -> None:
             Dependency(DependencyKind.HOST, "open-remarkable-shutdown")
         ]
         rm2_depends = [Dependency(DependencyKind.HOST, "rm2-suspend-fix")]
+        rmpp_depends = [Dependency(DependencyKind.HOST, "rmpp-make-root-rw")]
+        rmppm_depends = [Dependency(DependencyKind.HOST, "rmpp-make-root-rw")]
 
         recipes = parse_recipe(rec_path)
 
@@ -95,6 +101,10 @@ def test_installdepends(self) -> None:
                 "rm2",
                 "rm2os2",
                 "rm2os3",
+                "rmpp",
+                "rmppos3",
+                "rmppm",
+                "rmppmos3",
             ],
         )
         recipe = recipes["rmall"]
@@ -177,3 +187,39 @@ def test_installdepends(self) -> None:
             package.installdepends,
             set(basic_depends + rm2_depends),
         )
+
+        recipe = recipes["rmpp"]
+        self.assertIs(type(recipe), Recipe)
+        package = recipe.packages["toltec-base"]
+        self.assertEqual(list(recipe.packages.keys()), ["toltec-base"])
+        self.assertEqual(
+            package.installdepends,
+            set(basic_depends + rmpp_depends),
+        )
+
+        recipe = recipes["rmppos3"]
+        self.assertIs(type(recipe), Recipe)
+        package = recipe.packages["toltec-base"]
+        self.assertEqual(list(recipe.packages.keys()), ["toltec-base"])
+        self.assertEqual(
+            package.installdepends,
+            set(basic_depends + rmpp_depends),
+        )
+
+        recipe = recipes["rmppm"]
+        self.assertIs(type(recipe), Recipe)
+        package = recipe.packages["toltec-base"]
+        self.assertEqual(list(recipe.packages.keys()), ["toltec-base"])
+        self.assertEqual(
+            package.installdepends,
+            set(basic_depends + rmppm_depends),
+        )
+
+        recipe = recipes["rmppmos3"]
+        self.assertIs(type(recipe), Recipe)
+        package = recipe.packages["toltec-base"]
+        self.assertEqual(list(recipe.packages.keys()), ["toltec-base"])
+        self.assertEqual(
+            package.installdepends,
+            set(basic_depends + rmppm_depends),
+        )

tests/test_arch.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2023 The Toltec Contributors
+# SPDX-License-Identifier: MIT
+
+import unittest
+import subprocess
+
+from os import path
+from tempfile import TemporaryDirectory
+from elftools.elf.elffile import ELFFile
+
+
+class TestBuild(unittest.TestCase):
+    def setUp(self) -> None:
+        self.dir = path.dirname(path.realpath(__file__))
+        self.fixtures_dir = path.join(self.dir, "fixtures")
+
+    def test_arch(self) -> None:
+        with TemporaryDirectory() as tmp_dir:
+            rec_dir = path.join(self.fixtures_dir, "hello")
+            work_dir = path.join(tmp_dir, "build")
+            dist_dir = path.join(tmp_dir, "dist")
+
+            result = subprocess.run(
+                [
+                    "python3",
+                    "-m",
+                    "toltec",
+                    "--work-dir",
+                    work_dir,
+                    "--dist-dir",
+                    dist_dir,
+                    "--",
+                    rec_dir,
+                ],
+                capture_output=True,
+                check=False,
+            )
+            self.assertEqual(
+                result.returncode, 0, result.stderr.decode("utf-8")
+            )
+            self.assertEqual(result.stdout.decode("utf-8"), "")
+            with open(
+                path.join(tmp_dir, "build", "rm1", "src", "hello"), "rb"
+            ) as f:
+                self.assertEqual(ELFFile(f).get_machine_arch(), "ARM")
+            with open(
+                path.join(tmp_dir, "build", "rmpp", "src", "hello"), "rb"
+            ) as f:
+                self.assertEqual(ELFFile(f).get_machine_arch(), "AArch64")

toltec/builder.py

@@ -274,6 +274,7 @@ def _prepare(recipe: Recipe, src_dir: str) -> None:
         )
         bash.pipe_logs(logger, logs, "prepare()")
 
+    # pylint: disable=too-many-locals
     def _build(self, recipe: Recipe, src_dir: str) -> None:
         """Build artifacts for a recipe."""
         if not recipe.build:
@@ -316,13 +317,31 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
             )
 
         if host_deps:
-            opkg_conf_path = "$SYSROOT/etc/opkg/opkg.conf"
+            opkg_conf_path = (
+                "$SYSROOT_AARCH64/etc/opkg/opkg.conf"
+                if recipe.arch.startswith("rmpp")
+                else "$SYSROOT/etc/opkg/opkg.conf"
+            )
+            opkg_exec = (
+                "opkg-aarch64" if recipe.arch.startswith("rmpp") else "opkg"
+            )
+            opkg_arch = (
+                "aarch64-3.10"
+                if recipe.arch.startswith("rmpp")
+                else "armv7-3.2"
+            )
+            opkg_src = (
+                "aarch64-k3.10"
+                if recipe.arch.startswith("rmpp")
+                else "armv7sf-k3.2"
+            )
+
             pre_script.extend(
                 (
                     'echo -n "dest root /',
                     "arch all 100",
-                    "arch armv7-3.2 160",
-                    "src/gz entware https://bin.entware.net/armv7sf-k3.2",
+                    f"arch {opkg_arch} 160",
+                    f"src/gz entware https://bin.entware.net/{opkg_src}",
                     "arch rmall 200",
                     "src/gz toltec-rmall file:///repo/rmall",
                     f'" > "{opkg_conf_path}"',
@@ -340,12 +359,15 @@ def _build(self, recipe: Recipe, src_dir: str) -> None:
 
             pre_script.extend(
                 (
-                    "opkg update --verbosity=0",
-                    "opkg install --verbosity=0 --no-install-recommends"
+                    f"{opkg_exec} update --verbosity=0",
+                    f"{opkg_exec} install --verbosity=0 --no-install-recommends"
                     " -- " + " ".join(host_deps),
                 )
             )
 
+        if recipe.arch.startswith("rmpp"):
+            pre_script.append(("source /opt/x-tools/switch-aarch64.sh"))
+
         logs = bash.run_script_in_container(
             self.docker,
             image=self.IMAGE_PREFIX + recipe.image,

toltec/hooks/strip.py

@@ -20,7 +20,7 @@
 logger = logging.getLogger(__name__)
 
 MOUNT_SRC = "/src"
-TOOLCHAIN = "toolchain:v3.1"
+TOOLCHAIN = "toolchain:v4.0"
 
 
 def walk_elfs(src_dir: str, for_each: Callable) -> None:
@@ -73,6 +73,7 @@ def post_build(  # pylint: disable=too-many-locals,too-many-branches
 
         # Search for binary objects that can be stripped
         strip_arm: List[str] = []
+        strip_aarch64: List[str] = []
         strip_x86: List[str] = []
 
         def filter_elfs(info: ELFFile, file_path: str) -> None:
@@ -81,12 +82,14 @@ def filter_elfs(info: ELFFile, file_path: str) -> None:
                 return
             if info.get_machine_arch() == "ARM":
                 strip_arm.append(file_path)
+            elif info.get_machine_arch() == "AArch64":
+                strip_aarch64.append(file_path)
             elif info.get_machine_arch() in ("x86", "x64"):
                 strip_x86.append(file_path)
 
         walk_elfs(src_dir, filter_elfs)
 
-        if not strip_arm and not strip_x86:
+        if not strip_arm and not strip_aarch64 and not strip_x86:
             logger.debug("Skipping, no binaries found")
             return
 
@@ -139,6 +142,26 @@ def docker_file_path(file_path: str) -> str:
                     os.path.relpath(file_path, src_dir),
                 )
 
+        if strip_aarch64:
+            script.extend(
+                (
+                    "source /opt/x-tools/switch-aarch64.sh",
+                    '"${CROSS_COMPILE}strip" --strip-all -- '
+                    + " ".join(
+                        docker_file_path(file_path)
+                        for file_path in strip_aarch64
+                    ),
+                )
+            )
+
+            logger.debug("AArch64 binaries to be stripped:")
+
+            for file_path in strip_aarch64:
+                logger.debug(
+                    " - %s",
+                    os.path.relpath(file_path, src_dir),
+                )
+
         run_in_container(builder, src_dir, logger, script)
 
         # Restore original mtimes