Remove implicit package installs (#1413)

* Remove implicit package installs

Signed-off-by: Shah, Karan <[email protected]>

* Update CI

Signed-off-by: Shah, Karan <[email protected]>

* Install pkg cmd for tests

Signed-off-by: Shah, Karan <[email protected]>

* Patch gandlf test

Signed-off-by: Shah, Karan <[email protected]>

---------

Signed-off-by: Shah, Karan <[email protected]>

Author: MasterSkepticista

.github/workflows/tr_docker_gramine_direct.yml

@@ -33,6 +33,7 @@ jobs:
       run: |
         fx workspace create --prefix example_workspace --template keras/mnist
         cd example_workspace
+        pip install -r requirements.txt
         fx plan initialize -a localhost
         
         # Disable tensorboard logging as multiprocessing is not supported in Gramine

.github/workflows/tr_docker_native.yml

@@ -33,6 +33,7 @@ jobs:
       run: |
         fx workspace create --prefix example_workspace --template keras/mnist
         cd example_workspace
+        pip install -r requirements.txt
         fx plan initialize -a localhost
         fx workspace dockerize --base-image openfl --save
 

docs/tutorials/taskrunner.ipynb

@@ -31,15 +31,12 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {
-        "tags": [
-          "hide-output"
-        ]
-      },
+      "metadata": {},
       "outputs": [],
       "source": [
         "!fx workspace create --prefix ./mnist_example --template keras/mnist\n",
-        "%cd ./mnist_example"
+        "%cd ./mnist_example\n",
+        "!pip install -r requirements.txt > /dev/null"
       ]
     },
     {
@@ -62,9 +59,6 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "tags": [
-          "hide-output"
-        ],
         "vscode": {
           "languageId": "shellscript"
         }

openfl/interface/plan.py

@@ -4,14 +4,12 @@
 
 """Plan module."""
 
-import os
 import sys
 from logging import getLogger
 from os import makedirs
 from os.path import isfile
 from pathlib import Path
 from shutil import copyfile, rmtree
-from subprocess import check_call  # nosec
 
 from click import Path as ClickPath
 from click import echo, group, option, pass_context
@@ -92,14 +90,6 @@ def plan(context):
     required=False,
     help="GaNDLF Configuration File Path",
 )
-@option(
-    "-r",
-    "--install_reqs",
-    required=False,
-    help="If set, installs packages listed under 'requirements.txt'.",
-    default=True,
-    show_default=True,
-)
 @option(
     "-i",
     "--init_model_path",
@@ -115,7 +105,6 @@ def initialize(
     aggregator_address,
     input_shape,
     gandlf_config,
-    install_reqs,
     init_model_path,
 ):
     """
@@ -134,10 +123,6 @@ def initialize(
     if gandlf_config is not None:
         gandlf_config = Path(gandlf_config).absolute()
 
-    if install_reqs:
-        requirements_path = Path("requirements.txt").absolute()
-        _handle_requirements_install(requirements_path)
-
     plan = Plan.parse(
         plan_config_path=plan_config,
         cols_config_path=cols_config,
@@ -205,37 +190,6 @@ def initialize(
     logger.info(f"{context.obj['plans']}")
 
 
-def _handle_requirements_install(requirements_path):
-    """Handle the installation of requirements and process restart if needed.
-
-    This method checks if a requirements.txt file exists at the provided path.
-    If found, it installs the packages listed in the file using pip. After
-    successful installation, it restarts the current process with the same
-    arguments, but with the --install_reqs flag set to False to avoid
-    re-installing requirements.
-
-    If no requirements.txt file is found, it prints a message indicating that
-    no additional requirements are defined for the workspace and skips the
-    installation.
-
-    Args:
-        requirements_path (str or Path): The path to the requirements.txt file.
-    """
-    if isfile(str(requirements_path)):
-        check_call(
-            [sys.executable, "-m", "pip", "install", "-r", str(requirements_path)],
-            shell=False,
-        )
-        echo(f"Successfully installed packages from {requirements_path}.")
-
-        # Required to restart the process for newly installed packages to be recognized
-        args_restart = [arg for arg in sys.argv if not arg.startswith("--install_reqs")]
-        args_restart.append("--install_reqs=False")
-        os.execv(args_restart[0], args_restart)
-    else:
-        echo("No additional requirements for workspace defined. Skipping...")
-
-
 def _initialize_tensor_dict(plan, input_shape, init_model_path):
     """Initialize and return the tensor dictionary.
 

openfl/interface/workspace.py

@@ -12,8 +12,6 @@
 import tempfile
 from hashlib import sha256
 from pathlib import Path
-from subprocess import check_call  # nosec
-from sys import executable
 from typing import Union
 
 from click import Choice, echo, group, option, pass_context
@@ -23,7 +21,7 @@
 from openfl.cryptography.ca import generate_root_cert, generate_signing_csr, sign_certificate
 from openfl.federated.plan import Plan
 from openfl.interface import plan
-from openfl.interface.cli_helper import CERT_DIR, OPENFL_USERDIR, SITEPACKS, WORKSPACE, print_tree
+from openfl.interface.cli_helper import CERT_DIR, SITEPACKS, WORKSPACE, print_tree
 
 
 @group()
@@ -129,40 +127,11 @@ def create(prefix, template):
         prefix: The prefix for the directories to be created.
         template: The template to use for creating the workspace.
     """
-
-    if not OPENFL_USERDIR.exists():
-        OPENFL_USERDIR.mkdir()
-
     prefix = Path(prefix).absolute()
 
     create_dirs(prefix)
     create_temp(prefix, template)
 
-    requirements_filename = "requirements.txt"
-
-    if os.path.isfile(f"{str(prefix)}/{requirements_filename}"):
-        check_call(
-            [
-                executable,
-                "-m",
-                "pip",
-                "install",
-                "-r",
-                f"{prefix}/requirements.txt",
-            ],
-            shell=False,
-        )
-        echo(f"Successfully installed packages from {prefix}/requirements.txt.")
-    else:
-        echo("No additional requirements for workspace defined. Skipping...")
-    prefix_hash = _get_dir_hash(str(prefix.absolute()))
-    with open(
-        OPENFL_USERDIR / f"requirements.{prefix_hash}.txt",
-        "w",
-        encoding="utf-8",
-    ) as f:
-        check_call([executable, "-m", "pip", "freeze"], shell=False, stdout=f)
-
     apply_template_plan(prefix, template)
 
     print_tree(prefix, level=3)

tests/end_to_end/models/model_owner.py

@@ -68,6 +68,13 @@ def create_workspace(self):
                 raise_exception=True
             )
 
+            return_code, output, error = fh.run_command(
+                "pip install -r requirements.txt",
+                workspace_path=ws_path,
+                error_msg="Failed to install the requirements",
+                container_id=self.container_id,
+            )
+
         except Exception as e:
             log.error(f"{error_msg}: {e}")
             raise e

tests/github/test_gandlf.py

@@ -49,6 +49,7 @@ def main():
     shutil.rmtree(fed_workspace, ignore_errors=True)
     check_call(['fx', 'workspace', 'create', '--prefix', fed_workspace, '--template', template])
     os.chdir(fed_workspace)
+    check_call(['pip', 'install', '-r', 'requirements.txt'])
     Path(Path.cwd().resolve() / 'data' / col1).mkdir(exist_ok=True)
     with os.scandir(origin_dir) as iterator:
         for entry in iterator:

tests/github/utils.py

@@ -50,6 +50,7 @@ def create_certified_workspace(path, template, fqdn, rounds_to_train):
     shutil.rmtree(path, ignore_errors=True)
     check_call(['fx', 'workspace', 'create', '--prefix', path, '--template', template])
     os.chdir(path)
+    check_call(['pip', 'install', '-r', 'requirements.txt'])
 
     # Initialize FL plan
     check_call(['fx', 'plan', 'initialize', '-a', fqdn])