Parapred environment

.changeset/early-boxes-relate.md

@@ -0,0 +1,6 @@
+---
+'@platforma-open/milaboratories.runenv-python-3.12.10-parapred': minor
+'runenv-python-builder': minor
+---
+
+Support git dependencies in requirements, add new environment runenv-python-3.12.10-parapred

.github/workflows/build.yaml

@@ -82,7 +82,13 @@ jobs:
           {"os":"ubuntu-large-amd64", "arch":"amd64", "selector":"./python-3.12.10-sccoda"},
           {"os":"macos-14", "arch":"arm64", "selector":"./python-3.12.10-sccoda"},
           {"os":"macos-14-large", "arch":"amd64", "selector":"./python-3.12.10-sccoda"},
-          {"os":"windows-latest", "arch":"amd64", "selector":"./python-3.12.10-sccoda"}
+          {"os":"windows-latest", "arch":"amd64", "selector":"./python-3.12.10-sccoda"},
+
+          {"os":"ubuntu-large-arm64", "arch":"arm64", "selector":"./python-3.12.10-parapred"},
+          {"os":"ubuntu-large-amd64", "arch":"amd64", "selector":"./python-3.12.10-parapred"},
+          {"os":"macos-14", "arch":"arm64", "selector":"./python-3.12.10-parapred"},
+          {"os":"macos-14-large", "arch":"amd64", "selector":"./python-3.12.10-parapred"},
+          {"os":"windows-latest", "arch":"amd64", "selector":"./python-3.12.10-parapred"}
         ]
 
       sign-binaries: |

builder/src/build.ts

@@ -9,6 +9,9 @@ import * as linux from './linux';
 import * as macos from './macos';
 import * as windows from './windows';
 
+// Matches git URL deps: "git+https://github.com/org/repo.git" or "git+https://...repo.git@commit"
+const GIT_URL_RE = /^git\+https?:\/\/.*\/([^\/]+?)(?:\.git)?(?:@[^@]+)?$/;
+
 /*
  * Argument Parsing and Validation
  */
@@ -145,7 +148,16 @@ async function buildFromSources(version: string, osType: util.OS, archType: util
   );
 }
 
+function isGitUrl(packageSpec: string): boolean {
+  return GIT_URL_RE.test(packageSpec);
+}
+
 function getPackageName(packageSpec: string): string {
+  // Handle bare git URLs — extract repo name as the package name
+  const gitMatch = packageSpec.match(GIT_URL_RE);
+  if (gitMatch) {
+    return gitMatch[1];
+  }
   // Extract package name from spec (e.g., "parasail==1.3.4" -> "parasail")
   return packageSpec.split(/[<>=!]/)[0].trim();
 }
@@ -203,7 +215,32 @@ function getResolutionPolicy(osType: util.OS, archType: util.Arch): ResolutionPo
   return mergeResolution(base, plat);
 }
 
-function buildPipArgs(packageSpec: string, destinationDir: string): string[] {
+// Validate that git URL deps are covered by source-allowing resolution policy.
+// Without allowSourceList/allowSourceAll/forceSource, git deps silently get skipped
+// because the binary wheel attempt always fails and strictMissing defaults to false.
+function validateGitDeps(deps: string[], resolution: ResolutionPolicy, osType: util.OS, archType: util.Arch): void {
+  for (const depSpec of deps) {
+    const spec = depSpec.trim();
+    if (!spec || !isGitUrl(spec)) continue;
+
+    const name = getPackageName(spec);
+    const nameNorm = normalizePackageName(name);
+    const coveredByForceSource = shouldForceSource(name, osType, archType)
+      || resolution.forceNoBinaryList?.includes(nameNorm);
+    const coveredByAllowSource = resolution.allowSourceAll
+      || resolution.allowSourceList?.includes(nameNorm);
+
+    if (!coveredByForceSource && !coveredByAllowSource) {
+      throw new Error(
+        `Git dependency "${spec}" is not in allowSourceList or forceSource. ` +
+        `Without this, the package will be silently skipped after binary wheel lookup fails. ` +
+        `Add "${name}" to packages.resolution.allowSourceList in config.json.`
+      );
+    }
+  }
+}
+
+function buildPipArgs(packageSpec: string, destinationDir: string, noDeps: boolean = false): string[] {
   const args = [
     '-m',
     'pip',
@@ -213,12 +250,22 @@ function buildPipArgs(packageSpec: string, destinationDir: string): string[] {
     destinationDir
   ];
 
+  // Git URL deps may produce a file during the binary wheel attempt that still exists
+  // when the source fallback runs. Use --exists-action=w to overwrite silently in that case.
+  if (isGitUrl(packageSpec)) {
+    args.push('--exists-action', 'w');
+  }
+
   // Add additional registries (pip will use PyPI.org as default)
   const additionalRegistries = config.registries.additional || [];
   for (const url of additionalRegistries) {
     args.push('--extra-index-url=' + url);
   }
 
+  if (noDeps) {
+    args.push('--no-deps');
+  }
+
   return args;
 }
 
@@ -251,6 +298,10 @@ async function downloadPackages(pyBin: string, destinationDir: string, osType: u
   const resolution = getResolutionPolicy(osType, archType);
   console.log(`[DEBUG] Resolution policy: ${JSON.stringify(resolution)}`);
 
+  const noDepsList = (config.packages.noDeps || []).map(normalizePackageName);
+
+  validateGitDeps(allDeps, resolution, osType, archType);
+
   for (const depSpec of allDeps) {
     const depSpecClean = depSpec.trim();
     if (!depSpecClean) {
@@ -260,6 +311,7 @@ async function downloadPackages(pyBin: string, destinationDir: string, osType: u
 
     const packageName = getPackageName(depSpecClean);
     const packageNameNorm = normalizePackageName(packageName);
+    const shouldNoDeps = noDepsList.includes(packageNameNorm);
     console.log(`\nProcessing package: ${depSpecClean}`);
 
     // Check if package should be skipped for this platform
@@ -278,7 +330,7 @@ async function downloadPackages(pyBin: string, destinationDir: string, osType: u
       // Skip binary wheel attempt and go straight to source
       console.log(`  Building from source (forced)...`);
       try {
-        const pipArgs = buildPipArgs(depSpecClean, destinationDir);
+        const pipArgs = buildPipArgs(depSpecClean, destinationDir, shouldNoDeps);
         // Only force source for this package to preserve wheels for its dependencies
         pipArgs.push('--no-binary', packageName);
         await util.runCommand(pyBin, pipArgs);
@@ -292,7 +344,7 @@ async function downloadPackages(pyBin: string, destinationDir: string, osType: u
       // Try binary wheel first, then fall back to source
       try {
         console.log(`  Attempting to download binary wheel...`);
-        const pipArgs = buildPipArgs(depSpecClean, destinationDir);
+        const pipArgs = buildPipArgs(depSpecClean, destinationDir, shouldNoDeps);
         pipArgs.push('--only-binary', ':all:');
         await util.runCommand(pyBin, pipArgs);
         console.log(`  ✓ Successfully downloaded binary wheel for ${depSpecClean} (current platform)`);
@@ -326,7 +378,7 @@ async function downloadPackages(pyBin: string, destinationDir: string, osType: u
 
         console.log(`  ✗ Binary wheel not available for ${depSpecClean}, building from source (policy-allowed)...`);
         try {
-          const pipArgs = buildPipArgs(depSpecClean, destinationDir);
+          const pipArgs = buildPipArgs(depSpecClean, destinationDir, shouldNoDeps);
           // Only force source for this package, not its dependencies
           pipArgs.push('--no-binary', packageName);
           await util.runCommand(pyBin, pipArgs);

builder/src/config-merger.ts

@@ -59,6 +59,12 @@ export function mergeConfig(repoRoot: string, packageRoot: string): any {
                 ...(versionConfig.packages?.forceSource || {}),
             },
 
+            // 'noDeps': list of package names to install without transitive dependencies
+            noDeps: Array.from(new Set([
+                ...(sharedConfig.packages?.noDeps || []),
+                ...(versionConfig.packages?.noDeps || []),
+            ])),
+
             // 'copyFiles': This is for non-platform-specific files. Concatenate arrays.
             copyFiles: [
                 ...(sharedConfig.packages?.copyFiles || []),

catalogue/package.json

@@ -26,6 +26,9 @@
       },
       "3.12.10-h5ad": {
         "reference": "@platforma-open/milaboratories.runenv-python-3.12.10-h5ad/dist/tengo/software/main.sw.json"
+      },
+      "3.12.10-parapred": {
+        "reference": "@platforma-open/milaboratories.runenv-python-3.12.10-parapred/dist/tengo/software/main.sw.json"
       }
     }
   },
@@ -38,7 +41,8 @@
     "@platforma-open/milaboratories.runenv-python-3.12.10-atls": "workspace:*",
     "@platforma-open/milaboratories.runenv-python-3.12.10-sccoda": "workspace:*",
     "@platforma-open/milaboratories.runenv-python-3.12.10-rapids": "workspace:*",
-    "@platforma-open/milaboratories.runenv-python-3.12.10-h5ad": "workspace:*"
+    "@platforma-open/milaboratories.runenv-python-3.12.10-h5ad": "workspace:*",
+    "@platforma-open/milaboratories.runenv-python-3.12.10-parapred": "workspace:*"
   },
   "devDependencies": {
     "@platforma-sdk/package-builder": "catalog:"

checker/whitelists/python-3.12.10-parapred/linux-aarch64.json

@@ -0,0 +1,5 @@
+{
+  "torch-2.7.0+cpu-cp312-cp312-manylinux_2_28_aarch64.whl": {
+    "functorch._C": "initialization failed"
+  }
+}

checker/whitelists/python-3.12.10-parapred/linux-x64.json

@@ -0,0 +1,5 @@
+{
+  "torch-2.7.0+cpu-cp312-cp312-manylinux_2_28_x86_64.whl": {
+    "functorch._C": "initialization failed"
+  }
+}

checker/whitelists/python-3.12.10-parapred/macosx-aarch64.json

@@ -0,0 +1,5 @@
+{
+  "torch-2.7.0-cp312-none-macosx_11_0_arm64.whl": {
+    "functorch._C": "initialization failed"
+  }
+}

checker/whitelists/python-3.12.10-parapred/windows-x64.json

@@ -0,0 +1,5 @@
+{
+  "torch-2.7.0+cpu-cp312-cp312-win_amd64.whl": {
+    "functorch._C": "initialization failed"
+  }
+}

pnpm-workspace.yaml

@@ -6,6 +6,7 @@ packages:
   - python-3.12.10-sccoda
   - python-3.12.10-rapids
   - python-3.12.10-h5ad
+  - python-3.12.10-parapred
   - catalogue
 
 catalog:

python-3.12.10-parapred/config.json

@@ -0,0 +1,25 @@
+{
+  "registries": {
+    "additional": [
+      "https://download.pytorch.org/whl/cpu"
+    ]
+  },
+  "packages": {
+    "dependencies": [
+      "pandas==2.2.3",
+      "numpy==2.2.6",
+      "polars-lts-cpu==1.33.1",
+      "polars-ds-lts-cpu==0.10.2",
+      "torch==2.7.0",
+      "click==8.1.8",
+      "git+https://github.com/alchemab/parapred-pytorch.git@e0548be"
+    ],
+    "noDeps": ["parapred-pytorch"],
+    "skip": {},
+    "overrides": {},
+    "platformSpecific": {},
+    "resolution": {
+      "allowSourceList": ["parapred-pytorch"]
+    }
+  }
+}

python-3.12.10-parapred/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@platforma-open/milaboratories.runenv-python-3.12.10-parapred",
+  "version": "1.0.0",
+  "description": "Python 3.12.10 run environment for Platforma Backend with parapred-pytorch",
+  "scripts": {
+    "cleanup": "rm -rf ./pkg-*.tgz ./pydist ./dist/ ./build/",
+    "reset": "pnpm run cleanup && rm -rf ./node_modules ./.turbo",
+    "build": "pl-py-builder",
+    "after-prebuild": "pl-pkg publish packages",
+    "before-publish": "pl-pkg prepublish"
+  },
+  "files": [
+    "dist/"
+  ],
+  "block-software": {
+    "entrypoints": {
+      "main": {
+        "environment": {
+          "artifact": {
+            "type": "environment",
+            "runtime": "python",
+            "registry": "platforma-open",
+            "python-version": "3.12.10",
+            "roots": {
+              "linux-x64": "./pydist/linux-x64",
+              "linux-aarch64": "./pydist/linux-aarch64",
+              "macosx-x64": "./pydist/macosx-x64",
+              "macosx-aarch64": "./pydist/macosx-aarch64",
+              "windows-x64": "./pydist/windows-x64"
+            },
+            "binDir": "bin"
+          }
+        }
+      }
+    }
+  },
+  "license": "UNLICENSED",
+  "devDependencies": {
+    "@platforma-sdk/package-builder": "catalog:",
+    "runenv-python-builder": "workspace:*",
+    "tsx": "catalog:"
+  }
+}