Merged #40 into main

README.md

@@ -37,6 +37,7 @@ environment, running tasks, and much more.
     - [Repositories](#repositories)
       - [Authentication](#authentication)
     - [Requirements](#requirements)
+    - [Find links](#find-links)
     - [Tasks section](#tasks-section)
       - [Run](#tasks-section)
       - [Test](#tasks-section)
@@ -391,6 +392,8 @@ environment:
     your platform
     given [here](https://github.com/pyenv/pyenv/wiki#suggested-build-environment).
   - The downloaded and installed interpreter is cached in the `~/.paddle/interpreters` folder.
+- `noIndex` (*optional*): if True, this ignores the PyPi index, and make resolving only with url
+  from `findLinks` section. The flag is set to `False` by default.
 
 #### Repositories
 
@@ -543,6 +546,26 @@ convert it to its own format.
 
 <img src="assets/copypaste-paddle.png" alt="Copy-paste example">
 
+#### Find links
+
+`findLink` is a list of URLs or paths to the external non-indexed packages (e.g. local-built
+package). This is similar to pip's `--find-link` option.
+
+For local path or URLs starting from `file://` to a directory, then PyPI will look for
+archives in the directory.
+
+For paths and URLs to an HTML file, PyPI will look for link to archives as
+sdist (`.tar.gz`) or wheel (`.whl`).
+
+```yaml
+findLinks:
+    - /home/example/sample-wheel/dist
+    - https://example.com/python_packages.html
+    - file:///usr/share/packages/sample-wheel.whl 
+```
+
+**NB**: VCS links (e.g. `git://`) are not supported.
+
 #### Tasks section
 
 The `tasks` section consists of several subsections that provide run configurations for
@@ -675,6 +698,10 @@ Here is a reference for all the built-in Paddle tasks available at the moment.
 - `pylint`: runs [Pylint](https://pylint.pycqa.org/en/latest/) linter on the `sources` of the Paddle
   project.
 
+- `requirements`: generates `requirements.txt` in the root directory of every project.
+    - Note, that generated `requirements.txt` does not represent actual structure of Paddle source.
+      It would only generate dependencies for a project. 
+
 ## Example: multi-project build
 
 Let's consider the following example of a Paddle multi-project build: the parental project in the monorepo does not

idea/src/main/resources/schema/paddle-schema.json

@@ -108,6 +108,10 @@
         },
         "python": {
           "description": "Version of the Python interpreter to be used"
+        },
+        "noIndex": {
+          "description": "Ignore package index (only looking at --find-links URLs instead)",
+          "type": "boolean"
         }
       }
     },
@@ -153,6 +157,13 @@
         }
       }
     },
+    "findLinks": {
+      "description": "List of links, that will be appended as --find-link argument on the resolve stage",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
     "repositories": {
       "description": "List of the available PyPI repositories. Any custom repository will have precedence over pypi.org",
       "type": "array",

plugins/python/src/main/kotlin/io/paddle/plugin/python/PythonPlugin.kt

@@ -4,15 +4,14 @@ import io.paddle.plugin.Plugin
 import io.paddle.plugin.python.dependencies.authentication.AuthenticationProvider
 import io.paddle.plugin.python.dependencies.index.PyPackageRepositoryIndexer
 import io.paddle.plugin.python.extensions.*
+import io.paddle.plugin.python.tasks.generate.GenerateRequirements
 import io.paddle.plugin.python.tasks.install.CiTask
 import io.paddle.plugin.python.tasks.install.InstallTask
 import io.paddle.plugin.python.tasks.install.LockTask
 import io.paddle.plugin.python.tasks.lint.MyPyTask
 import io.paddle.plugin.python.tasks.lint.PyLintTask
 import io.paddle.plugin.python.tasks.publish.TwinePublishTask
-import io.paddle.plugin.python.tasks.resolve.ResolveInterpreterTask
-import io.paddle.plugin.python.tasks.resolve.ResolveRepositoriesTask
-import io.paddle.plugin.python.tasks.resolve.ResolveRequirementsTask
+import io.paddle.plugin.python.tasks.resolve.*
 import io.paddle.plugin.python.tasks.run.RunTask
 import io.paddle.plugin.python.tasks.test.PyTestTask
 import io.paddle.plugin.python.tasks.venv.VenvTask
@@ -45,7 +44,8 @@ object PythonPlugin : Plugin {
             MyPyTask(project),
             PyLintTask(project),
             WheelTask(project),
-            TwinePublishTask(project)
+            TwinePublishTask(project),
+            GenerateRequirements(project),
         ) + RunTask.from(project) + PyTestTask.from(project)
     }
 

plugins/python/src/main/kotlin/io/paddle/plugin/python/dependencies/packages/PyPackage.kt

@@ -2,8 +2,7 @@ package io.paddle.plugin.python.dependencies.packages
 
 import io.paddle.plugin.python.dependencies.repositories.PyPackageRepoMetadataSerializer
 import io.paddle.plugin.python.dependencies.repositories.PyPackageRepository
-import io.paddle.plugin.python.utils.PyPackageName
-import io.paddle.plugin.python.utils.PyPackageUrl
+import io.paddle.plugin.python.utils.*
 import kotlinx.serialization.Serializable
 
 /**
@@ -15,7 +14,8 @@ class PyPackage(
     override val version: PyPackageVersion,
     @Serializable(with = PyPackageRepoMetadataSerializer::class) override val repo: PyPackageRepository,
     override val distributionUrl: PyPackageUrl,
-    var comesFrom: PyPackage? = null
+    var comesFrom: PyPackage? = null,
+    val findLinkSource: PyUrl? = null
 ) : IResolvedPyPackage {
     override fun hashCode(): Int {
         if (distributionUrl.contains('#')) {

plugins/python/src/main/kotlin/io/paddle/plugin/python/dependencies/repositories/PyPackageRepositories.kt

@@ -4,23 +4,20 @@ import io.paddle.plugin.python.dependencies.authentication.authProvider
 import io.paddle.plugin.python.dependencies.index.distributions.PyDistributionInfo
 import io.paddle.plugin.python.extensions.Repositories
 import io.paddle.plugin.python.extensions.pyLocations
-import io.paddle.plugin.python.utils.PyPackageName
-import io.paddle.plugin.python.utils.isValidUrl
-import io.paddle.plugin.python.utils.parallelForEach
+import io.paddle.plugin.python.utils.*
 import io.paddle.project.PaddleProject
-import kotlinx.coroutines.async
-import kotlinx.coroutines.awaitAll
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.*
 
 class PyPackageRepositories(
     private val repositories: Set<PyPackageRepository>,
     val primarySource: PyPackageRepository,
+    val linkSources: List<PyUrl>,
     val project: PaddleProject,
     useCachedIndex: Boolean = true,
     downloadIndex: Boolean = false
 ) {
     companion object {
-        fun resolve(repoDescriptors: List<Repositories.Descriptor>, project: PaddleProject): PyPackageRepositories {
+        fun resolve(repoDescriptors: List<Repositories.Descriptor>, project: PaddleProject, findLinks: List<PyUrl>): PyPackageRepositories {
             val repositories = hashSetOf(PyPackageRepository.PYPI_REPOSITORY)
             var primarySource = PyPackageRepository.PYPI_REPOSITORY
 
@@ -41,8 +38,16 @@ class PyPackageRepositories(
 
                 repositories.add(repo)
             }
-
-            return PyPackageRepositories(repositories, primarySource, project)
+            findLinks.forEach {
+                require(it.isValidUrl() || it.isValidPath()) { "The provided find link is invalid: $it" }
+            }
+            val processedFindLinks = findLinks.map {
+                when {
+                    it.isValidUrl() -> it
+                    else -> "file://$it"
+                }
+            }
+            return PyPackageRepositories(repositories, primarySource, processedFindLinks, project)
         }
 
         private fun updateIndex(repositories: Set<PyPackageRepository>, project: PaddleProject) = runBlocking {
@@ -53,7 +58,7 @@ class PyPackageRepositories(
                 } catch (e: Throwable) {
                     project.terminal.warn(
                         "Failed to update index for PyPI repository: ${it.urlSimple}. " +
-                                "Autocompletion for package names will not be available at the moment."
+                            "Autocompletion for package names will not be available at the moment."
                     )
                 }
             }
@@ -131,6 +136,10 @@ class PyPackageRepositories(
                     add(credentials.authenticate(repo.urlSimple))
                 }
             }
+            for (link in this@PyPackageRepositories.linkSources) {
+                add("--find-link")
+                add(link)
+            }
         }
 
     fun findByName(name: String): PyPackageRepository? {

plugins/python/src/main/kotlin/io/paddle/plugin/python/dependencies/resolvers/PipResolver.kt

@@ -4,6 +4,7 @@ import io.paddle.plugin.python.dependencies.index.distributions.ArchivePyDistrib
 import io.paddle.plugin.python.dependencies.index.distributions.WheelPyDistributionInfo
 import io.paddle.plugin.python.dependencies.index.webIndexer
 import io.paddle.plugin.python.dependencies.packages.PyPackage
+import io.paddle.plugin.python.dependencies.repositories.PyPackageRepositories
 import io.paddle.plugin.python.dependencies.repositories.PyPackageRepository
 import io.paddle.plugin.python.extensions.*
 import io.paddle.plugin.python.utils.*
@@ -39,6 +40,7 @@ object PipResolver {
             noCacheDir = project.pythonRegistry.noCacheDir
             packages = requirementsAsPipArgs
             additionalArgs = project.repositories.resolved.asPipArgs
+            noIndex = project.environment.noIndex
         }.args
         val executable = project.environment.localInterpreterPath.absolutePathString()
         val input = (pipResolveArgs + executable).map { it.hashable() }.hashable().hash()
@@ -104,50 +106,60 @@ object PipResolver {
             // val fileExtension = lines[i + 2].substringAfter(": ")
             val filename = lines[i + 3].substringAfter(": ")
             val repoUrl = lines[i + 4].substringAfter(": ").substringBeforeLast("/simple/")
-            var distributionUrl = lines[i + 5].substringAfter(": ")
+            val distributionUrl = lines[i + 5].substringAfter(": ")
             val comesFromDistributionUrl = lines[i + 6].substringAfter(": ")
 
             val pyDistributionInfo = WheelPyDistributionInfo.fromString(filename)
                 ?: ArchivePyDistributionInfo.fromString(filename)
                 ?: throw Task.ActException("FIXME: Unknown distribution type: $filename")
 
             val version = pyDistributionInfo.version
+            val findLinkSourceUrl = distributionUrl.findLinkSourceIn(project.repositories.resolved)
 
-            val repo = if (repoUrl == "None") { // it was resolved as a local file distribution file://...
-                runBlocking {
-                    project.webIndexer.getDistributionUrl(
-                        pyDistributionInfo,
-                        PyPackageRepository.PYPI_REPOSITORY
-                    )
-                } // if null, it was not found in the PyPi repo
-                    ?: if (project.pythonRegistry.autoRemove) {
-                        val localDistribution = File(URI(distributionUrl))
-                        if (!localDistribution.exists()) {
-                            throw Task.ActException("Failed to delete local distribution $distributionUrl: file not found.")
-                        }
-                        if (!localDistribution.delete()) {
-                            throw Task.ActException("Failed to delete local distribution $distributionUrl. Please, do it manually and re-run the task.")
-                        }
-                        retry = true
-                    } else {
-                        if (!project.pythonRegistry.noCacheDir) {
-                            throw Task.ActException("Failed to find distribution $filename  in the repository ${PyPackageRepository.PYPI_REPOSITORY.url.getSecure()}")
-                        }
-                        project.terminal.warn(
-                            "Distribution $filename was not found in the repository ${PyPackageRepository.PYPI_REPOSITORY.url.getSecure()}.\n" +
-                                "It is possible that it was resolved from your local cache, " +
-                                "which is deprecated since it is not available online anymore.\n" +
-                                "Please, consider removing $distributionUrl from cache and re-running the task.\n" +
-                                "Or run again with disabled pip cache using `usePipCache: false`"
+
+            val repo = when {
+                findLinkSourceUrl != null -> {
+                    project.repositories.resolved.primarySource // FIXME: make this null requires a lot of code work
+                }
+
+                repoUrl == "None" -> { // it was resolved as a local file distribution file://...
+                    runBlocking {
+                        project.webIndexer.getDistributionUrl(
+                            pyDistributionInfo,
+                            PyPackageRepository.PYPI_REPOSITORY
                         )
-                    }
-                PyPackageRepository.PYPI_REPOSITORY
-            } else {
-                project.repositories.resolved.all.find { it.url.trimmedEquals(repoUrl) }
-                    ?: throw IllegalStateException("Unknown repository: $repoUrl")
+                    } // if null, it was not found in the PyPi repo
+                        ?: if (project.pythonRegistry.autoRemove) {
+                            val localDistribution = File(URI(distributionUrl))
+                            if (!localDistribution.exists()) {
+                                throw Task.ActException("Failed to delete local distribution $distributionUrl: file not found.")
+                            }
+                            if (!localDistribution.delete()) {
+                                throw Task.ActException("Failed to delete local distribution $distributionUrl. Please, do it manually and re-run the task.")
+                            }
+                            retry = true
+                        } else {
+                            if (!project.pythonRegistry.noCacheDir) {
+                                throw Task.ActException("Failed to find distribution $filename  in the repository ${PyPackageRepository.PYPI_REPOSITORY.url.getSecure()}")
+                            }
+                            project.terminal.warn(
+                                "Distribution $filename was not found in the repository ${PyPackageRepository.PYPI_REPOSITORY.url.getSecure()}.\n" +
+                                    "It is possible that it was resolved from your local cache, " +
+                                    "which is deprecated since it is not available online anymore.\n" +
+                                    "Please, consider removing $distributionUrl from cache and re-running the task.\n" +
+                                    "Or run again with disabled pip cache using `usePipCache: false`"
+                            )
+                        }
+                    PyPackageRepository.PYPI_REPOSITORY
+                }
+
+                else -> {
+                    project.repositories.resolved.all.find { it.url.trimmedEquals(repoUrl) }
+                        ?: throw IllegalStateException("Unknown repository: $repoUrl")
+                }
             }
 
-            val pkg = PyPackage(name, version, repo, distributionUrl)
+            val pkg = PyPackage(name, version, repo, distributionUrl, findLinkSource = findLinkSourceUrl)
             comesFromUrlByPackage[pkg] = comesFromDistributionUrl
         }
 
@@ -164,5 +176,8 @@ object PipResolver {
         return comesFromUrlByPackage.keys + satisfiedRequirements
     }
 
-    class RetrySignal() : Exception()
+    private fun PyPackageUrl.findLinkSourceIn(repositories: PyPackageRepositories): PyUrl? =
+        repositories.linkSources.find { findLinksSource -> this.startsWith(findLinksSource) }
+
+    class RetrySignal : Exception()
 }

plugins/python/src/main/kotlin/io/paddle/plugin/python/extensions/Environment.kt

@@ -26,7 +26,7 @@ import kotlin.io.path.absolutePathString
 val PaddleProject.environment: Environment
     get() = checkNotNull(extensions.get(Environment.Extension.key)) { "Could not load extension Environment for project $routeAsString" }
 
-class Environment(val project: PaddleProject, val venv: VenvDir) : Hashable {
+class Environment(val project: PaddleProject, val venv: VenvDir, val noIndex: Boolean) : Hashable {
 
     val localInterpreterPath: Path
         get() = venv.getInterpreterPath(project)
@@ -47,9 +47,10 @@ class Environment(val project: PaddleProject, val venv: VenvDir) : Hashable {
         override fun create(project: PaddleProject): Environment {
             val config = object : ConfigurationView("environment", project.config) {
                 val venv by string("path", default = ".venv")
+                val noIndex by lazy { get<String>("noIndex")?.toBoolean() ?: false }
             }
 
-            return Environment(project, VenvDir(File(project.workDir, config.venv)))
+            return Environment(project, VenvDir(File(project.workDir, config.venv)), config.noIndex)
         }
     }
 

plugins/python/src/main/kotlin/io/paddle/plugin/python/extensions/Repositories.kt

@@ -15,10 +15,10 @@ import io.paddle.utils.hash.hashable
 val PaddleProject.repositories: Repositories
     get() = checkNotNull(extensions.get(Repositories.Extension.key)) { "Could not load extension Repositories for project $routeAsString" }
 
-class Repositories(val project: PaddleProject, val descriptors: List<Descriptor>) : Hashable {
+class Repositories(val project: PaddleProject, val descriptors: List<Descriptor>, val findLinks: List<String>) : Hashable {
 
     val resolved: PyPackageRepositories by lazy {
-        PyPackageRepositories.resolve(descriptors, project)
+        PyPackageRepositories.resolve(descriptors, project, findLinks)
     }
 
     object Extension : PaddleProject.Extension<Repositories> {
@@ -27,6 +27,7 @@ class Repositories(val project: PaddleProject, val descriptors: List<Descriptor>
         @Suppress("UNCHECKED_CAST")
         override fun create(project: PaddleProject): Repositories {
             val reposConfig = project.config.get<List<Map<String, Any>>>("repositories") ?: emptyList()
+            val findLinks = project.config.get<List<String>>("findLinks") ?: emptyList()
 
             val descriptors = reposConfig.map { repo ->
                 val repoName = checkNotNull(repo["name"]) {
@@ -63,7 +64,7 @@ class Repositories(val project: PaddleProject, val descriptors: List<Descriptor>
                 )
             }
 
-            return Repositories(project, descriptors)
+            return Repositories(project, descriptors, findLinks)
         }
     }
 
@@ -95,7 +96,7 @@ class Repositories(val project: PaddleProject, val descriptors: List<Descriptor>
     }
 
     override fun hash(): String {
-        return descriptors.hashable().hash()
+        return (descriptors + findLinks.map { it.hashable() }).hashable().hash()
     }
 }
 

plugins/python/src/main/kotlin/io/paddle/plugin/python/tasks/PythonPluginTaskGroups.kt

@@ -5,4 +5,5 @@ object PythonPluginTaskGroups {
     const val INSTALL = "install"
     const val VENV = "venv"
     const val PUBLISH = "publish"
+    const val GENERATE = "generate"
 }