diff --git a/.changeset/early-boxes-relate.md b/.changeset/early-boxes-relate.md new file mode 100644 index 0000000..e7071d6 --- /dev/null +++ b/.changeset/early-boxes-relate.md @@ -0,0 +1,6 @@ +--- +'@platforma-open/milaboratories.runenv-python-3.12.10-parapred': minor +'runenv-python-builder': minor +--- + +Support git dependencies in requirements, add new environment runenv-python-3.12.10-parapred diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ada29d7..f35a36c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -82,7 +82,13 @@ jobs: {"os":"ubuntu-large-amd64", "arch":"amd64", "selector":"./python-3.12.10-sccoda"}, {"os":"macos-14", "arch":"arm64", "selector":"./python-3.12.10-sccoda"}, {"os":"macos-14-large", "arch":"amd64", "selector":"./python-3.12.10-sccoda"}, - {"os":"windows-latest", "arch":"amd64", "selector":"./python-3.12.10-sccoda"} + {"os":"windows-latest", "arch":"amd64", "selector":"./python-3.12.10-sccoda"}, + + {"os":"ubuntu-large-arm64", "arch":"arm64", "selector":"./python-3.12.10-parapred"}, + {"os":"ubuntu-large-amd64", "arch":"amd64", "selector":"./python-3.12.10-parapred"}, + {"os":"macos-14", "arch":"arm64", "selector":"./python-3.12.10-parapred"}, + {"os":"macos-14-large", "arch":"amd64", "selector":"./python-3.12.10-parapred"}, + {"os":"windows-latest", "arch":"amd64", "selector":"./python-3.12.10-parapred"} ] sign-binaries: | diff --git a/builder/src/build.ts b/builder/src/build.ts index 32fd3c5..3f04674 100755 --- a/builder/src/build.ts +++ b/builder/src/build.ts @@ -9,6 +9,9 @@ import * as linux from './linux'; import * as macos from './macos'; import * as windows from './windows'; +// Matches git URL deps: "git+https://github.com/org/repo.git" or "git+https://...repo.git@commit" +const GIT_URL_RE = /^git\+https?:\/\/.*\/([^\/]+?)(?:\.git)?(?:@[^@]+)?$/; + /* * Argument Parsing and Validation */ @@ -145,7 +148,16 @@ async function buildFromSources(version: string, osType: util.OS, archType: util ); } +function isGitUrl(packageSpec: string): boolean { + return GIT_URL_RE.test(packageSpec); +} + function getPackageName(packageSpec: string): string { + // Handle bare git URLs — extract repo name as the package name + const gitMatch = packageSpec.match(GIT_URL_RE); + if (gitMatch) { + return gitMatch[1]; + } // Extract package name from spec (e.g., "parasail==1.3.4" -> "parasail") return packageSpec.split(/[<>=!]/)[0].trim(); } @@ -203,7 +215,32 @@ function getResolutionPolicy(osType: util.OS, archType: util.Arch): ResolutionPo return mergeResolution(base, plat); } -function buildPipArgs(packageSpec: string, destinationDir: string): string[] { +// Validate that git URL deps are covered by source-allowing resolution policy. +// Without allowSourceList/allowSourceAll/forceSource, git deps silently get skipped +// because the binary wheel attempt always fails and strictMissing defaults to false. +function validateGitDeps(deps: string[], resolution: ResolutionPolicy, osType: util.OS, archType: util.Arch): void { + for (const depSpec of deps) { + const spec = depSpec.trim(); + if (!spec || !isGitUrl(spec)) continue; + + const name = getPackageName(spec); + const nameNorm = normalizePackageName(name); + const coveredByForceSource = shouldForceSource(name, osType, archType) + || resolution.forceNoBinaryList?.includes(nameNorm); + const coveredByAllowSource = resolution.allowSourceAll + || resolution.allowSourceList?.includes(nameNorm); + + if (!coveredByForceSource && !coveredByAllowSource) { + throw new Error( + `Git dependency "${spec}" is not in allowSourceList or forceSource. ` + + `Without this, the package will be silently skipped after binary wheel lookup fails. ` + + `Add "${name}" to packages.resolution.allowSourceList in config.json.` + ); + } + } +} + +function buildPipArgs(packageSpec: string, destinationDir: string, noDeps: boolean = false): string[] { const args = [ '-m', 'pip', @@ -213,12 +250,22 @@ function buildPipArgs(packageSpec: string, destinationDir: string): string[] { destinationDir ]; + // Git URL deps may produce a file during the binary wheel attempt that still exists + // when the source fallback runs. Use --exists-action=w to overwrite silently in that case. + if (isGitUrl(packageSpec)) { + args.push('--exists-action', 'w'); + } + // Add additional registries (pip will use PyPI.org as default) const additionalRegistries = config.registries.additional || []; for (const url of additionalRegistries) { args.push('--extra-index-url=' + url); } + if (noDeps) { + args.push('--no-deps'); + } + return args; } @@ -251,6 +298,10 @@ async function downloadPackages(pyBin: string, destinationDir: string, osType: u const resolution = getResolutionPolicy(osType, archType); console.log(`[DEBUG] Resolution policy: ${JSON.stringify(resolution)}`); + const noDepsList = (config.packages.noDeps || []).map(normalizePackageName); + + validateGitDeps(allDeps, resolution, osType, archType); + for (const depSpec of allDeps) { const depSpecClean = depSpec.trim(); if (!depSpecClean) { @@ -260,6 +311,7 @@ async function downloadPackages(pyBin: string, destinationDir: string, osType: u const packageName = getPackageName(depSpecClean); const packageNameNorm = normalizePackageName(packageName); + const shouldNoDeps = noDepsList.includes(packageNameNorm); console.log(`\nProcessing package: ${depSpecClean}`); // Check if package should be skipped for this platform @@ -278,7 +330,7 @@ async function downloadPackages(pyBin: string, destinationDir: string, osType: u // Skip binary wheel attempt and go straight to source console.log(` Building from source (forced)...`); try { - const pipArgs = buildPipArgs(depSpecClean, destinationDir); + const pipArgs = buildPipArgs(depSpecClean, destinationDir, shouldNoDeps); // Only force source for this package to preserve wheels for its dependencies pipArgs.push('--no-binary', packageName); await util.runCommand(pyBin, pipArgs); @@ -292,7 +344,7 @@ async function downloadPackages(pyBin: string, destinationDir: string, osType: u // Try binary wheel first, then fall back to source try { console.log(` Attempting to download binary wheel...`); - const pipArgs = buildPipArgs(depSpecClean, destinationDir); + const pipArgs = buildPipArgs(depSpecClean, destinationDir, shouldNoDeps); pipArgs.push('--only-binary', ':all:'); await util.runCommand(pyBin, pipArgs); console.log(` ✓ Successfully downloaded binary wheel for ${depSpecClean} (current platform)`); @@ -326,7 +378,7 @@ async function downloadPackages(pyBin: string, destinationDir: string, osType: u console.log(` ✗ Binary wheel not available for ${depSpecClean}, building from source (policy-allowed)...`); try { - const pipArgs = buildPipArgs(depSpecClean, destinationDir); + const pipArgs = buildPipArgs(depSpecClean, destinationDir, shouldNoDeps); // Only force source for this package, not its dependencies pipArgs.push('--no-binary', packageName); await util.runCommand(pyBin, pipArgs); diff --git a/builder/src/config-merger.ts b/builder/src/config-merger.ts index 68d51de..f8ccf5e 100644 --- a/builder/src/config-merger.ts +++ b/builder/src/config-merger.ts @@ -59,6 +59,12 @@ export function mergeConfig(repoRoot: string, packageRoot: string): any { ...(versionConfig.packages?.forceSource || {}), }, + // 'noDeps': list of package names to install without transitive dependencies + noDeps: Array.from(new Set([ + ...(sharedConfig.packages?.noDeps || []), + ...(versionConfig.packages?.noDeps || []), + ])), + // 'copyFiles': This is for non-platform-specific files. Concatenate arrays. copyFiles: [ ...(sharedConfig.packages?.copyFiles || []), diff --git a/catalogue/package.json b/catalogue/package.json index 6a06d92..52aab67 100644 --- a/catalogue/package.json +++ b/catalogue/package.json @@ -26,6 +26,9 @@ }, "3.12.10-h5ad": { "reference": "@platforma-open/milaboratories.runenv-python-3.12.10-h5ad/dist/tengo/software/main.sw.json" + }, + "3.12.10-parapred": { + "reference": "@platforma-open/milaboratories.runenv-python-3.12.10-parapred/dist/tengo/software/main.sw.json" } } }, @@ -38,7 +41,8 @@ "@platforma-open/milaboratories.runenv-python-3.12.10-atls": "workspace:*", "@platforma-open/milaboratories.runenv-python-3.12.10-sccoda": "workspace:*", "@platforma-open/milaboratories.runenv-python-3.12.10-rapids": "workspace:*", - "@platforma-open/milaboratories.runenv-python-3.12.10-h5ad": "workspace:*" + "@platforma-open/milaboratories.runenv-python-3.12.10-h5ad": "workspace:*", + "@platforma-open/milaboratories.runenv-python-3.12.10-parapred": "workspace:*" }, "devDependencies": { "@platforma-sdk/package-builder": "catalog:" diff --git a/checker/whitelists/python-3.12.10-parapred/linux-aarch64.json b/checker/whitelists/python-3.12.10-parapred/linux-aarch64.json new file mode 100644 index 0000000..a905061 --- /dev/null +++ b/checker/whitelists/python-3.12.10-parapred/linux-aarch64.json @@ -0,0 +1,5 @@ +{ + "torch-2.7.0+cpu-cp312-cp312-manylinux_2_28_aarch64.whl": { + "functorch._C": "initialization failed" + } +} diff --git a/checker/whitelists/python-3.12.10-parapred/linux-x64.json b/checker/whitelists/python-3.12.10-parapred/linux-x64.json new file mode 100644 index 0000000..16b1033 --- /dev/null +++ b/checker/whitelists/python-3.12.10-parapred/linux-x64.json @@ -0,0 +1,5 @@ +{ + "torch-2.7.0+cpu-cp312-cp312-manylinux_2_28_x86_64.whl": { + "functorch._C": "initialization failed" + } +} diff --git a/checker/whitelists/python-3.12.10-parapred/macosx-aarch64.json b/checker/whitelists/python-3.12.10-parapred/macosx-aarch64.json new file mode 100644 index 0000000..ade8943 --- /dev/null +++ b/checker/whitelists/python-3.12.10-parapred/macosx-aarch64.json @@ -0,0 +1,5 @@ +{ + "torch-2.7.0-cp312-none-macosx_11_0_arm64.whl": { + "functorch._C": "initialization failed" + } +} diff --git a/checker/whitelists/python-3.12.10-parapred/windows-x64.json b/checker/whitelists/python-3.12.10-parapred/windows-x64.json new file mode 100644 index 0000000..8acd935 --- /dev/null +++ b/checker/whitelists/python-3.12.10-parapred/windows-x64.json @@ -0,0 +1,5 @@ +{ + "torch-2.7.0+cpu-cp312-cp312-win_amd64.whl": { + "functorch._C": "initialization failed" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0d374ec..2884034 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,6 +68,9 @@ importers: '@platforma-open/milaboratories.runenv-python-3.12.10-h5ad': specifier: workspace:* version: link:../python-3.12.10-h5ad + '@platforma-open/milaboratories.runenv-python-3.12.10-parapred': + specifier: workspace:* + version: link:../python-3.12.10-parapred '@platforma-open/milaboratories.runenv-python-3.12.10-rapids': specifier: workspace:* version: link:../python-3.12.10-rapids @@ -127,6 +130,18 @@ importers: specifier: 'catalog:' version: 4.20.6 + python-3.12.10-parapred: + devDependencies: + '@platforma-sdk/package-builder': + specifier: 'catalog:' + version: 3.10.7 + runenv-python-builder: + specifier: workspace:* + version: link:../builder + tsx: + specifier: 'catalog:' + version: 4.20.6 + python-3.12.10-rapids: devDependencies: '@platforma-sdk/package-builder': @@ -1123,6 +1138,7 @@ packages: glob@10.4.5: resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true globby@11.1.0: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 951f208..45c4068 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,6 +6,7 @@ packages: - python-3.12.10-sccoda - python-3.12.10-rapids - python-3.12.10-h5ad + - python-3.12.10-parapred - catalogue catalog: diff --git a/python-3.12.10-parapred/config.json b/python-3.12.10-parapred/config.json new file mode 100644 index 0000000..956a1ff --- /dev/null +++ b/python-3.12.10-parapred/config.json @@ -0,0 +1,25 @@ +{ + "registries": { + "additional": [ + "https://download.pytorch.org/whl/cpu" + ] + }, + "packages": { + "dependencies": [ + "pandas==2.2.3", + "numpy==2.2.6", + "polars-lts-cpu==1.33.1", + "polars-ds-lts-cpu==0.10.2", + "torch==2.7.0", + "click==8.1.8", + "git+https://github.com/alchemab/parapred-pytorch.git@e0548be" + ], + "noDeps": ["parapred-pytorch"], + "skip": {}, + "overrides": {}, + "platformSpecific": {}, + "resolution": { + "allowSourceList": ["parapred-pytorch"] + } + } +} diff --git a/python-3.12.10-parapred/package.json b/python-3.12.10-parapred/package.json new file mode 100644 index 0000000..0729b20 --- /dev/null +++ b/python-3.12.10-parapred/package.json @@ -0,0 +1,43 @@ +{ + "name": "@platforma-open/milaboratories.runenv-python-3.12.10-parapred", + "version": "1.0.0", + "description": "Python 3.12.10 run environment for Platforma Backend with parapred-pytorch", + "scripts": { + "cleanup": "rm -rf ./pkg-*.tgz ./pydist ./dist/ ./build/", + "reset": "pnpm run cleanup && rm -rf ./node_modules ./.turbo", + "build": "pl-py-builder", + "after-prebuild": "pl-pkg publish packages", + "before-publish": "pl-pkg prepublish" + }, + "files": [ + "dist/" + ], + "block-software": { + "entrypoints": { + "main": { + "environment": { + "artifact": { + "type": "environment", + "runtime": "python", + "registry": "platforma-open", + "python-version": "3.12.10", + "roots": { + "linux-x64": "./pydist/linux-x64", + "linux-aarch64": "./pydist/linux-aarch64", + "macosx-x64": "./pydist/macosx-x64", + "macosx-aarch64": "./pydist/macosx-aarch64", + "windows-x64": "./pydist/windows-x64" + }, + "binDir": "bin" + } + } + } + } + }, + "license": "UNLICENSED", + "devDependencies": { + "@platforma-sdk/package-builder": "catalog:", + "runenv-python-builder": "workspace:*", + "tsx": "catalog:" + } +}