diff --git a/.chronus/changes/playground-upload-python-2026-3-1-16-58-54.md b/.chronus/changes/playground-upload-python-2026-3-1-16-58-54.md new file mode 100644 index 00000000000..2807f7ce27e --- /dev/null +++ b/.chronus/changes/playground-upload-python-2026-3-1-16-58-54.md @@ -0,0 +1,7 @@ +--- +changeKind: internal +packages: + - "@typespec/http-client-python" +--- + +Extend publish pipeline to upload emitter bundle to Playground storage account \ No newline at end of file diff --git a/.github/actions/build-playground-emitters/action.yml b/.github/actions/build-playground-emitters/action.yml new file mode 100644 index 00000000000..ad9276fbdaf --- /dev/null +++ b/.github/actions/build-playground-emitters/action.yml @@ -0,0 +1,16 @@ +name: Build playground emitters +description: Build emitter packages required by the playground (excluded from pnpm workspace) + +runs: + using: composite + + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Build http-client-python emitter + shell: bash + working-directory: packages/http-client-python + run: npm ci && npm run build diff --git a/.github/workflows/core-ci.yml b/.github/workflows/core-ci.yml index 38d69874097..ccf95da4465 100644 --- a/.github/workflows/core-ci.yml +++ b/.github/workflows/core-ci.yml @@ -72,6 +72,8 @@ jobs: - name: Install dependencies run: pnpm install + - uses: ./.github/actions/build-playground-emitters + - name: Install Playwright browsers run: npx playwright install --with-deps @@ -98,6 +100,8 @@ jobs: - name: Install dependencies run: pnpm install + - uses: ./.github/actions/build-playground-emitters + - name: Install Playwright browsers run: | sudo dpkg --configure -a diff --git a/eng/emitters/pipelines/templates/stages/emitter-stages.yml b/eng/emitters/pipelines/templates/stages/emitter-stages.yml index 9fc53174f91..1b16d7fb624 100644 --- a/eng/emitters/pipelines/templates/stages/emitter-stages.yml +++ b/eng/emitters/pipelines/templates/stages/emitter-stages.yml @@ -82,6 +82,11 @@ parameters: type: string default: "3.12" + # Whether to bundle and upload the emitter package to the playground package storage. + - name: UploadPlaygroundBundle + type: boolean + default: false + stages: # Build stage # Responsible for building the autorest generator and typespec emitter packages @@ -333,6 +338,29 @@ stages: ArtifactPath: $(buildArtifactsPath) LanguageShortName: ${{ parameters.LanguageShortName }} + - ${{ if parameters.UploadPlaygroundBundle }}: + - script: npm ci + displayName: Install emitter dependencies for playground bundle + workingDirectory: $(Build.SourcesDirectory)/${{ parameters.PackagePath }} + - script: npm run build + displayName: Build emitter for playground bundle + workingDirectory: $(Build.SourcesDirectory)/${{ parameters.PackagePath }} + - script: npm install -g pnpm + displayName: Install pnpm for playground bundle upload + - script: pnpm install --filter "@typespec/bundle-uploader..." + displayName: Install bundle-uploader dependencies + workingDirectory: $(Build.SourcesDirectory) + - script: pnpm --filter "@typespec/bundle-uploader..." build + displayName: Build bundle-uploader + workingDirectory: $(Build.SourcesDirectory) + - task: AzureCLI@1 + displayName: Upload playground bundle + inputs: + azureSubscription: "Azure SDK Engineering System" + scriptLocation: inlineScript + inlineScript: node ./eng/emitters/scripts/upload-bundled-emitter.js ${{ parameters.PackagePath }} + workingDirectory: $(Build.SourcesDirectory) + templateContext: outputs: - output: pipelineArtifact diff --git a/eng/emitters/scripts/upload-bundled-emitter.js b/eng/emitters/scripts/upload-bundled-emitter.js new file mode 100644 index 00000000000..81b373b09ac --- /dev/null +++ b/eng/emitters/scripts/upload-bundled-emitter.js @@ -0,0 +1,18 @@ +// @ts-check +import { resolve } from "path"; +import { bundleAndUploadStandalonePackage } from "../../../packages/bundle-uploader/dist/src/index.js"; +import { repoRoot } from "../../common/scripts/helpers.js"; + +const packageRelativePath = process.argv[2]; +if (!packageRelativePath) { + // eslint-disable-next-line no-console + console.error("Usage: node upload-bundled-emitter.js "); + // eslint-disable-next-line no-console + console.error(" e.g. node upload-bundled-emitter.js packages/http-client-csharp"); + process.exit(1); +} + +// remove leading slash if exists, then resolve to absolute path +const packagePath = resolve(repoRoot, packageRelativePath.replace(/^\//, "")); + +await bundleAndUploadStandalonePackage({ packagePath }); diff --git a/eng/tsp-core/pipelines/jobs/e2e.yml b/eng/tsp-core/pipelines/jobs/e2e.yml index db375014ab8..3663e7fe83a 100644 --- a/eng/tsp-core/pipelines/jobs/e2e.yml +++ b/eng/tsp-core/pipelines/jobs/e2e.yml @@ -18,6 +18,7 @@ jobs: steps: - template: /eng/tsp-core/pipelines/templates/install.yml + - template: /eng/tsp-core/pipelines/templates/build-playground-emitters.yml - template: /eng/tsp-core/pipelines/templates/install-browsers.yml - template: /eng/tsp-core/pipelines/templates/setup-linux-ui.yml - template: /eng/tsp-core/pipelines/templates/build.yml diff --git a/eng/tsp-core/pipelines/jobs/website.yml b/eng/tsp-core/pipelines/jobs/website.yml index 1756ac1f8c7..9b6dc30a11f 100644 --- a/eng/tsp-core/pipelines/jobs/website.yml +++ b/eng/tsp-core/pipelines/jobs/website.yml @@ -1,5 +1,6 @@ steps: - template: /eng/tsp-core/pipelines/templates/install.yml + - template: /eng/tsp-core/pipelines/templates/build-playground-emitters.yml - script: pnpm exec playwright install displayName: Install browsers diff --git a/eng/tsp-core/pipelines/templates/build-playground-emitters.yml b/eng/tsp-core/pipelines/templates/build-playground-emitters.yml new file mode 100644 index 00000000000..1e0dfead2ec --- /dev/null +++ b/eng/tsp-core/pipelines/templates/build-playground-emitters.yml @@ -0,0 +1,9 @@ +# Build emitter packages required by the playground (excluded from pnpm workspace) +steps: + - task: UsePythonVersion@0 + displayName: Set up Python + inputs: + versionSpec: "3.x" + + - script: cd packages/http-client-python && npm ci && npm run build + displayName: Build http-client-python emitter diff --git a/packages/bundle-uploader/package.json b/packages/bundle-uploader/package.json index 8be768faefa..45c9a799f28 100644 --- a/packages/bundle-uploader/package.json +++ b/packages/bundle-uploader/package.json @@ -41,6 +41,7 @@ "@azure/storage-blob": "catalog:", "@pnpm/workspace.find-packages": "catalog:", "@typespec/bundler": "workspace:^", + "globby": "catalog:", "json5": "catalog:", "picocolors": "catalog:", "semver": "catalog:" diff --git a/packages/bundle-uploader/src/index.ts b/packages/bundle-uploader/src/index.ts index 757403df988..abf805d40a9 100644 --- a/packages/bundle-uploader/src/index.ts +++ b/packages/bundle-uploader/src/index.ts @@ -1,7 +1,9 @@ import { AzureCliCredential } from "@azure/identity"; import { findWorkspacePackagesNoCheck } from "@pnpm/workspace.find-packages"; import { createTypeSpecBundle } from "@typespec/bundler"; -import { resolve } from "path"; +import { readFile } from "fs/promises"; +import { globby } from "globby"; +import { relative, resolve } from "path"; import { join as joinUnix } from "path/posix"; import pc from "picocolors"; import { parse } from "semver"; @@ -16,6 +18,137 @@ function logSuccess(message: string) { logInfo(pc.green(`✔ ${message}`)); } +export interface PlaygroundAssetConfig { + /** Glob pattern relative to the package root (e.g. "generator/dist/pygen-*.whl"). */ + path: string; + /** MIME content type for the blob upload. */ + contentType: string; +} + +export interface PlaygroundConfig { + /** Static files to upload as binary blobs. Paths support simple glob patterns. */ + assets?: PlaygroundAssetConfig[]; + /** Peer dependencies that should be bundled and uploaded. */ + bundlePeerDependencies?: string[]; +} + +export interface BundleAndUploadStandalonePackageOptions { + /** + * Absolute path to the package directory. + */ + packagePath: string; +} + +/** + * Bundle and upload a standalone package that is not part of the pnpm workspace. + * Uploads the bundle files and writes a `latest.json` under the package's blob directory + * (e.g. `@typespec/http-client-csharp/latest.json`). + * + * If the package's `package.json` contains a `playgroundConfig` section, this function + * will also upload static assets (resolved via glob patterns) and bundle peer dependencies. + */ +export async function bundleAndUploadStandalonePackage({ + packagePath, +}: BundleAndUploadStandalonePackageOptions) { + const pkgJsonPath = resolve(packagePath, "package.json"); + const pkgJson = JSON.parse(await readFile(pkgJsonPath, "utf-8")); + const playgroundConfig: PlaygroundConfig | undefined = pkgJson.playgroundConfig; + + const bundle = await createTypeSpecBundle(packagePath); + const manifest = bundle.manifest; + logInfo(`Bundling standalone package: ${manifest.name}@${manifest.version}`); + + const uploader = new TypeSpecBundledPackageUploader(new AzureCliCredential()); + await uploader.createIfNotExists(); + + const result = await uploader.upload(bundle); + if (result.status === "uploaded") { + logSuccess(`Bundle for package ${manifest.name}@${manifest.version} uploaded.`); + } else { + logInfo(`Bundle for package ${manifest.name} already exists for version ${manifest.version}.`); + } + + const importMap: Record = {}; + for (const [key, value] of Object.entries(result.imports)) { + importMap[joinUnix(manifest.name, key)] = value; + } + + await uploadPlaygroundAssets(uploader, packagePath, manifest, importMap, playgroundConfig); + + await uploader.updatePackageLatest(manifest.name, { + version: manifest.version, + imports: importMap, + }); + logSuccess(`Updated ${manifest.name}/latest.json for version ${manifest.version}.`); +} + +/** + * Upload playground assets and bundle peer dependencies based on the provided config. + */ +async function uploadPlaygroundAssets( + uploader: TypeSpecBundledPackageUploader, + packagePath: string, + manifest: { name: string; version: string }, + importMap: Record, + config: PlaygroundConfig | undefined, +) { + if (!config) { + return; + } + + // Upload static assets (e.g. .whl files) + if (config.assets) { + for (const asset of config.assets) { + const matchedFiles = await globby(asset.path, { cwd: packagePath, absolute: true }); + if (matchedFiles.length === 0) { + logInfo(pc.yellow(`⚠ No files matched asset pattern: ${asset.path}`)); + continue; + } + for (const filePath of matchedFiles) { + const relativePath = relative(packagePath, filePath).replace(/\\/g, "/"); + const blobPath = joinUnix(manifest.name, manifest.version, relativePath); + const content = await readFile(filePath); + const assetResult = await uploader.uploadBinaryAsset(blobPath, content, asset.contentType); + const importKey = joinUnix(manifest.name, relativePath); + importMap[importKey] = assetResult.url; + if (assetResult.status === "uploaded") { + logSuccess(`Uploaded asset: ${relativePath}`); + } else { + logInfo(`Asset already exists: ${relativePath}`); + } + } + } + } + + // Bundle and upload peer dependencies + if (config.bundlePeerDependencies) { + for (const depName of config.bundlePeerDependencies) { + const depPath = resolve(packagePath, "node_modules", depName); + try { + const depBundle = await createTypeSpecBundle(depPath); + const depResult = await uploader.upload(depBundle); + if (depResult.status === "uploaded") { + logSuccess( + `Bundle for peer dep ${depBundle.manifest.name}@${depBundle.manifest.version} uploaded.`, + ); + } else { + logInfo( + `Bundle for peer dep ${depBundle.manifest.name} already exists for version ${depBundle.manifest.version}.`, + ); + } + for (const [key, value] of Object.entries(depResult.imports)) { + importMap[joinUnix(depBundle.manifest.name, key)] = value; + } + } catch (e: unknown) { + throw new Error( + `Failed to bundle peer dependency ${depName}: ${e instanceof Error ? e.message : e}`, + { cause: e }, + ); + } + } + } +} + export interface BundleAndUploadPackagesOptions { repoRoot: string; /** @@ -85,9 +218,12 @@ export async function bundleAndUploadPackages({ } } logInfo(`Import map for ${indexVersion}:`, importMap); - await uploader.updateIndex(indexName, { + const index = { version: indexVersion, imports: importMap, - }); + }; + await uploader.updateIndex(indexName, index); logSuccess(`Updated index for version ${indexVersion}.`); + await uploader.updateLatestIndex(indexName, index); + logSuccess(`Updated latest index for version ${indexVersion}.`); } diff --git a/packages/bundle-uploader/src/upload-browser-package.ts b/packages/bundle-uploader/src/upload-browser-package.ts index eec752029d0..c2815847b9f 100644 --- a/packages/bundle-uploader/src/upload-browser-package.ts +++ b/packages/bundle-uploader/src/upload-browser-package.ts @@ -75,6 +75,80 @@ export class TypeSpecBundledPackageUploader { }); } + async uploadBinaryAsset( + blobPath: string, + content: Buffer, + contentType: string, + ): Promise<{ status: "uploaded" | "already-exists"; url: string }> { + const normalizedPath = normalizePath(blobPath); + const blob = this.#container.getBlockBlobClient(normalizedPath); + const url = `${this.#container.url}/${normalizedPath}`; + try { + await blob.uploadData(content, { + blobHTTPHeaders: { + blobContentType: contentType, + }, + conditions: { + ifNoneMatch: "*", + }, + }); + return { status: "uploaded", url }; + } catch (e: any) { + if (e.code === "BlobAlreadyExists") { + return { status: "already-exists", url }; + } + throw e; + } + } + + async getLatestIndex(name: string): Promise { + const blob = this.#container.getBlockBlobClient(`indexes/${name}/latest.json`); + if (await blob.exists()) { + const response = await blob.download(); + const body = await response.blobBody; + const existingContent = await body?.text(); + if (existingContent) { + return JSON.parse(existingContent); + } + } + return undefined; + } + + async updateLatestIndex(name: string, index: PackageIndex) { + const blob = this.#container.getBlockBlobClient(`indexes/${name}/latest.json`); + const content = JSON.stringify(index); + await blob.upload(content, content.length, { + blobHTTPHeaders: { + blobContentType: "application/json; charset=utf-8", + }, + }); + } + + /** Read the latest.json for a package from `{pkgName}/latest.json`. */ + async getPackageLatest(pkgName: string): Promise { + const blob = this.#container.getBlockBlobClient(normalizePath(join(pkgName, "latest.json"))); + if (await blob.exists()) { + const response = await blob.download(); + const body = await response.blobBody; + const existingContent = await body?.text(); + if (existingContent) { + return JSON.parse(existingContent); + } + } + return undefined; + } + + /** Write the latest.json for a package at `{pkgName}/latest.json`. */ + async updatePackageLatest(pkgName: string, index: PackageIndex) { + const blob = this.#container.getBlockBlobClient(normalizePath(join(pkgName, "latest.json"))); + const content = JSON.stringify(index); + await blob.upload(content, content.length, { + blobHTTPHeaders: { + blobContentType: "application/json; charset=utf-8", + }, + }); + } + async #uploadManifest(manifest: BundleManifest) { try { const blob = this.#container.getBlockBlobClient( diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 9a3b66b883a..720bee96daa 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -1,10 +1,11 @@ import { createSdkContext } from "@azure-tools/typespec-client-generator-core"; -import { EmitContext, NoTarget } from "@typespec/compiler"; +import { EmitContext, emitFile, joinPaths, NoTarget } from "@typespec/compiler"; import { execSync } from "child_process"; import fs from "fs"; +import jsyaml from "js-yaml"; import os from "os"; import path, { dirname } from "path"; -import { loadPyodide } from "pyodide"; +import { loadPyodide, PyodideInterface } from "pyodide"; import { fileURLToPath } from "url"; import { emitCodeModel } from "./code-model.js"; import { saveCodeModelAsYaml } from "./external-process.js"; @@ -12,6 +13,16 @@ import { PythonEmitterOptions, PythonSdkContext, reportDiagnostic } from "./lib. import { runPython3 } from "./run-python3.js"; import { disableGenerationMap, simpleTypesMap, typesMap } from "./types.js"; import { getRootNamespace, md2Rst } from "./utils.js"; +import pkgJson from "../../package.json" with { type: "json" }; + +const PYODIDE_VERSION = "0.26.2"; +const PYGEN_WHEEL_FILENAME = "pygen-0.1.0-py3-none-any.whl"; +const BLOB_STORAGE_BASE_URL = "https://typespec.blob.core.windows.net/pkgs"; +const PACKAGE_NAME = "@typespec/http-client-python"; + +function getBrowserPygenWheelUrl(): string { + return `${BLOB_STORAGE_BASE_URL}/${PACKAGE_NAME}/${pkgJson.version}/generator/dist/${PYGEN_WHEEL_FILENAME}`; +} function addDefaultOptions(sdkContext: PythonSdkContext) { const defaultOptions = { @@ -106,6 +117,60 @@ function cleanAllCache() { disableGenerationMap.clear(); } +const pyodideGenerationCode = ` +async def main(): + import warnings + with warnings.catch_warnings(): + from pygen import preprocess, codegen, black + preprocess.PreProcessPlugin(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs).process() + codegen.CodeGenerator(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs).process() + black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process() + +await main()`; + +async function runPyodideGeneration( + pyodide: PyodideInterface, + outputFolder: string, + yamlFile: string, + commandArgs: Record, +) { + const globals = pyodide.toPy({ + outputFolder, + yamlFile, + commandArgs, + }); + + await pyodide.runPythonAsync(pyodideGenerationCode, { globals }); +} + +async function copyPyodideOutputToHost( + context: EmitContext, + pyodide: PyodideInterface, + memfsDir: string, + relativeDir: string = "", +) { + const entries = pyodide.FS.readdir(memfsDir).filter( + (entry: string) => entry !== "." && entry !== "..", + ); + + for (const entry of entries) { + const memfsPath = `${memfsDir}/${entry}`; + const relativePath = relativeDir ? `${relativeDir}/${entry}` : entry; + const stats = pyodide.FS.stat(memfsPath); + + if (pyodide.FS.isDir(stats.mode)) { + await copyPyodideOutputToHost(context, pyodide, memfsPath, relativePath); + continue; + } + + const content = pyodide.FS.readFile(memfsPath, { encoding: "utf8" }); + await emitFile(context.program, { + path: joinPaths(context.emitterOutputDir, relativePath), + content, + }); + } +} + export async function $onEmit(context: EmitContext) { try { await onEmitMain(context); @@ -129,13 +194,21 @@ async function onEmitMain(context: EmitContext) { const program = context.program; const sdkContext = await createPythonSdkContext(context); - const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", ".."); + const outputDir = context.emitterOutputDir; addDefaultOptions(sdkContext); const yamlMap = emitCodeModel(sdkContext); const parsedYamlMap = walkThroughNodes(yamlMap); - const yamlPath = await saveCodeModelAsYaml("python-yaml-path", parsedYamlMap); + // Python emitter requires an SDK client in the TypeSpec + if (sdkContext.sdkPackage.clients.length === 0) { + reportDiagnostic(program, { + code: "no-sdk-clients", + target: NoTarget, + }); + return; + } + const resolvedOptions = sdkContext.emitContext.options; const commandArgs: Record = {}; if (resolvedOptions["packaging-files-config"]) { @@ -162,15 +235,31 @@ async function onEmitMain(context: EmitContext) { commandArgs["from-typespec"] = "true"; commandArgs["models-mode"] = (resolvedOptions as any)["models-mode"] ?? "dpg"; - if (!program.compilerOptions.noEmit && !program.hasError()) { - // if not using pyodide and there's no venv, we try to create venv - if (!resolvedOptions["use-pyodide"] && !fs.existsSync(path.join(root, "venv"))) { - try { - await runPython3(path.join(root, "/eng/scripts/setup/install.py")); - await runPython3(path.join(root, "/eng/scripts/setup/prepare.py")); - } catch (error) { - // if the python env is not ready, we use pyodide instead - resolvedOptions["use-pyodide"] = true; + if (typeof window !== "undefined") { + // Running in browser with Pyodide - fileURLToPath and other filesystem operations are browser-incompatible + const pyodide = await setupPyodideCallBrowser(); + + const yamlFilePath = "/yaml/python-yaml-path.yaml"; + pyodide.FS.mkdirTree("/yaml"); + pyodide.FS.mkdirTree("/output"); + pyodide.FS.writeFile(yamlFilePath, jsyaml.dump(parsedYamlMap)); + + await runPyodideGeneration(pyodide, "/output", yamlFilePath, commandArgs); + await copyPyodideOutputToHost(context, pyodide, "/output"); + } else { + const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", ".."); + const yamlPath = await saveCodeModelAsYaml("python-yaml-path", parsedYamlMap); + + if (!program.compilerOptions.noEmit && !program.hasError()) { + // if not using pyodide and there's no venv, we try to create venv + if (!resolvedOptions["use-pyodide"] && !fs.existsSync(path.join(root, "venv"))) { + try { + await runPython3(path.join(root, "/eng/scripts/setup/install.py")); + await runPython3(path.join(root, "/eng/scripts/setup/prepare.py")); + } catch (error) { + // if the python env is not ready, we use pyodide instead + resolvedOptions["use-pyodide"] = true; + } } } @@ -187,22 +276,12 @@ async function onEmitMain(context: EmitContext) { // mount yaml file to pyodide pyodide.FS.mkdirTree("/yaml"); pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: path.dirname(yamlPath) }, "/yaml"); - const globals = pyodide.toPy({ - outputFolder: "/output", - yamlFile: `/yaml/${path.basename(yamlPath)}`, + await runPyodideGeneration( + pyodide, + "/output", + `/yaml/${path.basename(yamlPath)}`, commandArgs, - }); - const pythonCode = ` - async def main(): - import warnings - with warnings.catch_warnings(): - from pygen import preprocess, codegen, black - preprocess.PreProcessPlugin(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs).process() - codegen.CodeGenerator(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs).process() - black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process() - - await main()`; - await pyodide.runPythonAsync(pythonCode, { globals }); + ); } else { // here we run with native python let venvPath = path.join(root, "venv"); @@ -255,6 +334,20 @@ async function onEmitMain(context: EmitContext) { } } +async function setupPyodideCallBrowser() { + const pyodide = await loadPyodide({ + indexURL: `https://cdn.jsdelivr.net/pyodide/v${PYODIDE_VERSION}/full/`, + }); + + // use default MEMFS for browser, since NODEFS is not supported + pyodide.FS.mkdirTree("/generator"); + await pyodide.loadPackage("micropip"); + const micropip = pyodide.pyimport("micropip"); + await micropip.install(getBrowserPygenWheelUrl()); + + return pyodide; +} + async function setupPyodideCall(root: string) { const pyodide = await loadPyodide({ indexURL: path.dirname(fileURLToPath(import.meta.resolve("pyodide"))), @@ -284,7 +377,7 @@ async function setupPyodideCall(root: string) { ); await pyodide.loadPackage("micropip"); const micropip = pyodide.pyimport("micropip"); - await micropip.install("emfs:/generator/dist/pygen-0.1.0-py3-none-any.whl"); + await micropip.install(`emfs:/generator/dist/${PYGEN_WHEEL_FILENAME}`); fs.closeSync(fd); fs.unlinkSync(micropipLockPath); break; diff --git a/packages/http-client-python/emitter/src/lib.ts b/packages/http-client-python/emitter/src/lib.ts index 120d008cb62..44c27473304 100644 --- a/packages/http-client-python/emitter/src/lib.ts +++ b/packages/http-client-python/emitter/src/lib.ts @@ -127,6 +127,19 @@ const libDef = { "Python is not installed. Please follow https://www.python.org/ to install Python or set 'use-pyodide' to true.", }, }, + "no-sdk-clients": { + severity: "error", + messages: { + default: + "The Python emitter did not find any SDK clients in this TypeSpec program. The current Python generator expects at least one client/service to generate code.", + }, + }, + "browser-runtime-load-failed": { + severity: "error", + messages: { + default: paramMessage`Failed to initialize the browser Python runtime.${"details"}`, + }, + }, "invalid-paging-items": { severity: "warning", messages: { diff --git a/packages/http-client-python/emitter/tsconfig.build.json b/packages/http-client-python/emitter/tsconfig.build.json index 663c3cc0d58..1a8a8ed8f52 100644 --- a/packages/http-client-python/emitter/tsconfig.build.json +++ b/packages/http-client-python/emitter/tsconfig.build.json @@ -2,10 +2,11 @@ "extends": "./tsconfig.json", "compilerOptions": { "noEmit": false, + "resolveJsonModule": true, "rootDir": "./src", "outDir": "../dist/emitter", "tsBuildInfoFile": "temp/tsconfig.tsbuildinfo" }, "references": [], - "include": ["src/**/*"] + "include": ["src/**/*", "../package.json"] } diff --git a/packages/http-client-python/emitter/tsconfig.json b/packages/http-client-python/emitter/tsconfig.json index 32cb2aff3f2..99589e3efe4 100644 --- a/packages/http-client-python/emitter/tsconfig.json +++ b/packages/http-client-python/emitter/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "../../../tsconfig.base.json", "compilerOptions": { - "noEmit": true + "noEmit": true, + "resolveJsonModule": true }, "include": ["src/**/*"] } diff --git a/packages/http-client-python/eng/pipeline/publish.yml b/packages/http-client-python/eng/pipeline/publish.yml index 69fe74aa10e..d134468d578 100644 --- a/packages/http-client-python/eng/pipeline/publish.yml +++ b/packages/http-client-python/eng/pipeline/publish.yml @@ -31,3 +31,4 @@ extends: CadlRanchName: "@typespec/http-client-python" EnableCadlRanchReport: false PythonVersion: "3.11" + UploadPlaygroundBundle: true diff --git a/packages/http-client-python/package.json b/packages/http-client-python/package.json index 75f709d16dc..a14e5986a8a 100644 --- a/packages/http-client-python/package.json +++ b/packages/http-client-python/package.json @@ -1,6 +1,6 @@ { "name": "@typespec/http-client-python", - "version": "0.28.2", + "version": "0.28.2-alpha.1", "author": "Microsoft Corporation", "description": "TypeSpec emitter for Python SDKs", "homepage": "https://typespec.io", @@ -69,6 +69,21 @@ "@typespec/streams": ">=0.80.0 <1.0.0", "@typespec/xml": ">=0.80.0 <1.0.0" }, + "playgroundConfig": { + "assets": [ + { + "path": "generator/dist/pygen-*.whl", + "contentType": "application/zip" + } + ], + "bundlePeerDependencies": [ + "@azure-tools/typespec-client-generator-core", + "@azure-tools/typespec-azure-core", + "@azure-tools/typespec-autorest", + "@azure-tools/typespec-azure-resource-manager", + "@azure-tools/typespec-azure-rulesets" + ] + }, "dependencies": { "js-yaml": "~4.1.0", "marked": "^15.0.6", diff --git a/packages/playground-website/package.json b/packages/playground-website/package.json index 9e02692cf18..0b1c0cca1d6 100644 --- a/packages/playground-website/package.json +++ b/packages/playground-website/package.json @@ -35,7 +35,7 @@ "scripts": { "clean": "rimraf ./dist ./dist-dev ./temp ./samples/dist", "build-samples": "node ./samples/build.js", - "build": "pnpm build-samples && pnpm build:lib && vite build 2>&1", + "build": "node -e \"if(process.env.TYPESPEC_SKIP_WEBSITE_BUILD==='true'){console.log('Skipping playground-website build');process.exit(0)}else{process.exit(1)}\" || (pnpm build-samples && pnpm build:lib && vite build 2>&1)", "build:lib": "vite build --config vite.lib.config.ts", "preview": "pnpm build && vite preview", "start": "vite", @@ -59,6 +59,7 @@ "@typespec/events": "workspace:^", "@typespec/html-program-viewer": "workspace:^", "@typespec/http": "workspace:^", + "@typespec/http-client-python": "link:../http-client-python", "@typespec/json-schema": "workspace:^", "@typespec/openapi": "workspace:^", "@typespec/openapi3": "workspace:^", @@ -70,6 +71,11 @@ "@typespec/streams": "workspace:^", "@typespec/versioning": "workspace:^", "@typespec/xml": "workspace:^", + "@azure-tools/typespec-client-generator-core": ">=0.66.2 <1.0.0", + "@azure-tools/typespec-autorest": ">=0.66.0 <1.0.0", + "@azure-tools/typespec-azure-core": ">=0.66.0 <1.0.0", + "@azure-tools/typespec-azure-resource-manager": ">=0.66.0 <1.0.0", + "@azure-tools/typespec-azure-rulesets": ">=0.66.0 <1.0.0", "es-module-shims": "catalog:", "react": "catalog:", "react-dom": "catalog:" diff --git a/packages/playground-website/samples/build.js b/packages/playground-website/samples/build.js index 82c345d40b3..506c219cf75 100644 --- a/packages/playground-website/samples/build.js +++ b/packages/playground-website/samples/build.js @@ -29,6 +29,19 @@ await buildSamples_experimental(packageRoot, resolve(__dirname, "dist/samples.ts compilerOptions: { linterRuleSet: { extends: ["@typespec/http/all"] } }, description: "Use the REST framework for resource-oriented API design patterns.", }, + "Python HTTP client": { + filename: "samples/python-client.tsp", + preferredEmitter: "@typespec/http-client-python", + compilerOptions: { + linterRuleSet: { extends: ["@typespec/http/all"] }, + options: { + "@typespec/http-client-python": { + "use-pyodide": true, + }, + }, + }, + description: "Generate a Python client for a small HTTP service.", + }, "Protobuf Kiosk": { filename: "samples/kiosk.tsp", preferredEmitter: "@typespec/protobuf", diff --git a/packages/playground-website/samples/python-client.tsp b/packages/playground-website/samples/python-client.tsp new file mode 100644 index 00000000000..9148a7f63d4 --- /dev/null +++ b/packages/playground-website/samples/python-client.tsp @@ -0,0 +1,29 @@ +import "@typespec/http"; +import "@typespec/rest"; + +using Http; +using Rest; + +@service(#{ title: "Widget Service" }) +namespace DemoService; + +model Widget { + @key id: string; + name: string; + color: "red" | "blue"; +} + +@error +model ServiceError { + code: string; + message: string; +} + +@route("/widgets") +interface Widgets { + @get list(): Widget[]; + + @get + @route("/{id}") + get(@path id: string): Widget | ServiceError; +} diff --git a/packages/playground-website/src/config.ts b/packages/playground-website/src/config.ts index ee32fdfa694..337121cbeb1 100644 --- a/packages/playground-website/src/config.ts +++ b/packages/playground-website/src/config.ts @@ -15,6 +15,12 @@ export const TypeSpecPlaygroundConfig = { "@typespec/events", "@typespec/sse", "@typespec/xml", + "@azure-tools/typespec-client-generator-core", + "@azure-tools/typespec-autorest", + "@azure-tools/typespec-azure-resource-manager", + "@azure-tools/typespec-azure-rulesets", + "@azure-tools/typespec-azure-core", + "@typespec/http-client-python", ], samples, } as const; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f91e817d807..c4f8864d4b2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -758,6 +758,9 @@ importers: '@typespec/bundler': specifier: workspace:^ version: link:../bundler + globby: + specifier: 'catalog:' + version: 16.1.1 json5: specifier: 'catalog:' version: 2.2.3 @@ -2080,6 +2083,21 @@ importers: packages/playground-website: dependencies: + '@azure-tools/typespec-autorest': + specifier: '>=0.66.0 <1.0.0' + version: 0.66.1(5073cca808566b0813cd760b0ece4c80) + '@azure-tools/typespec-azure-core': + specifier: '>=0.66.0 <1.0.0' + version: 0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest) + '@azure-tools/typespec-azure-resource-manager': + specifier: '>=0.66.0 <1.0.0' + version: 0.66.1(@azure-tools/typespec-azure-core@0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest))(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/openapi@packages+openapi)(@typespec/rest@packages+rest)(@typespec/versioning@packages+versioning) + '@azure-tools/typespec-azure-rulesets': + specifier: '>=0.66.0 <1.0.0' + version: 0.66.0(43a25ef11631422e4edc1bf6c9f3e1d2) + '@azure-tools/typespec-client-generator-core': + specifier: '>=0.66.2 <1.0.0' + version: 0.66.4(@azure-tools/typespec-azure-core@0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest))(@typespec/compiler@packages+compiler)(@typespec/events@packages+events)(@typespec/http@packages+http)(@typespec/openapi@packages+openapi)(@typespec/rest@packages+rest)(@typespec/sse@packages+sse)(@typespec/streams@packages+streams)(@typespec/versioning@packages+versioning)(@typespec/xml@packages+xml) '@fluentui/react-components': specifier: 'catalog:' version: 9.73.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0) @@ -2098,6 +2116,9 @@ importers: '@typespec/http': specifier: workspace:^ version: link:../http + '@typespec/http-client-python': + specifier: link:../http-client-python + version: link:../http-client-python '@typespec/json-schema': specifier: workspace:^ version: link:../json-schema @@ -3400,6 +3421,66 @@ packages: '@azu/style-format@1.0.1': resolution: {integrity: sha512-AHcTojlNBdD/3/KxIKlg8sxIWHfOtQszLvOpagLTO+bjC3u7SAszu1lf//u7JJC50aUSH+BVWDD/KvaA6Gfn5g==} + '@azure-tools/typespec-autorest@0.66.1': + resolution: {integrity: sha512-9R2S9hr1nie5lvJQnubvywPajOhdUApTED5MIef5KlF1zZL+DKMFDDOKwnSvFvW7ROmL+Ph8FQagw/6+PStlOg==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@azure-tools/typespec-azure-core': ^0.66.1 + '@azure-tools/typespec-azure-resource-manager': ^0.66.0 + '@azure-tools/typespec-client-generator-core': ^0.66.4 + '@typespec/compiler': ^1.10.0 + '@typespec/http': ^1.10.0 + '@typespec/openapi': ^1.10.0 + '@typespec/rest': ^0.80.0 + '@typespec/versioning': ^0.80.0 + '@typespec/xml': ^0.80.0 + peerDependenciesMeta: + '@typespec/xml': + optional: true + + '@azure-tools/typespec-azure-core@0.66.1': + resolution: {integrity: sha512-i8lMegL4s0I6xQT61zIIhmN1aA6iYFoH+7owSl/msOD0yVWx3Khf3ETULX53yHFd7OoUDAjmFx7+8j9atWXzHQ==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@typespec/compiler': ^1.10.0 + '@typespec/http': ^1.10.0 + '@typespec/rest': ^0.80.0 + + '@azure-tools/typespec-azure-resource-manager@0.66.1': + resolution: {integrity: sha512-LExgmD3oK7iaX0bIdQsoxBYLCMKwPPBHYmLMhNW/GseMF1Dqi1Ne/VP7rmMP0dhrZNFh22LaZI2lTyCKm08DFQ==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@azure-tools/typespec-azure-core': ^0.66.1 + '@typespec/compiler': ^1.10.0 + '@typespec/http': ^1.10.0 + '@typespec/openapi': ^1.10.0 + '@typespec/rest': ^0.80.0 + '@typespec/versioning': ^0.80.0 + + '@azure-tools/typespec-azure-rulesets@0.66.0': + resolution: {integrity: sha512-Wf0SpphmKDDzHgaqpxl68DpP65VUWjpD3mrnZ3Lw4Pdtt8BcZf7+LKgFF06gPRnh15hR0VbjAERCzxI/qGY4ag==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@azure-tools/typespec-azure-core': ^0.66.0 + '@azure-tools/typespec-azure-resource-manager': ^0.66.0 + '@azure-tools/typespec-client-generator-core': ^0.66.1 + '@typespec/compiler': ^1.10.0 + + '@azure-tools/typespec-client-generator-core@0.66.4': + resolution: {integrity: sha512-KRMWLvojku2qFnPpUiZNTa/nm49IjSsGVhPmFhQ5a01KwI2T7zT+Ga39/xLTLHHT4aIgBaMWxD4ioZa2ZhbEKw==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@azure-tools/typespec-azure-core': ^0.66.1 + '@typespec/compiler': ^1.10.0 + '@typespec/events': ^0.80.0 + '@typespec/http': ^1.10.0 + '@typespec/openapi': ^1.10.0 + '@typespec/rest': ^0.80.0 + '@typespec/sse': ^0.80.0 + '@typespec/streams': ^0.80.0 + '@typespec/versioning': ^0.80.0 + '@typespec/xml': ^0.80.0 + '@azure/abort-controller@2.1.2': resolution: {integrity: sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==} engines: {node: '>=18.0.0'} @@ -14003,6 +14084,59 @@ snapshots: dependencies: '@azu/format-text': 1.0.2 + '@azure-tools/typespec-autorest@0.66.1(5073cca808566b0813cd760b0ece4c80)': + dependencies: + '@azure-tools/typespec-azure-core': 0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest) + '@azure-tools/typespec-azure-resource-manager': 0.66.1(@azure-tools/typespec-azure-core@0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest))(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/openapi@packages+openapi)(@typespec/rest@packages+rest)(@typespec/versioning@packages+versioning) + '@azure-tools/typespec-client-generator-core': 0.66.4(@azure-tools/typespec-azure-core@0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest))(@typespec/compiler@packages+compiler)(@typespec/events@packages+events)(@typespec/http@packages+http)(@typespec/openapi@packages+openapi)(@typespec/rest@packages+rest)(@typespec/sse@packages+sse)(@typespec/streams@packages+streams)(@typespec/versioning@packages+versioning)(@typespec/xml@packages+xml) + '@typespec/compiler': link:packages/compiler + '@typespec/http': link:packages/http + '@typespec/openapi': link:packages/openapi + '@typespec/rest': link:packages/rest + '@typespec/versioning': link:packages/versioning + optionalDependencies: + '@typespec/xml': link:packages/xml + + '@azure-tools/typespec-azure-core@0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest)': + dependencies: + '@typespec/compiler': link:packages/compiler + '@typespec/http': link:packages/http + '@typespec/rest': link:packages/rest + + '@azure-tools/typespec-azure-resource-manager@0.66.1(@azure-tools/typespec-azure-core@0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest))(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/openapi@packages+openapi)(@typespec/rest@packages+rest)(@typespec/versioning@packages+versioning)': + dependencies: + '@azure-tools/typespec-azure-core': 0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest) + '@typespec/compiler': link:packages/compiler + '@typespec/http': link:packages/http + '@typespec/openapi': link:packages/openapi + '@typespec/rest': link:packages/rest + '@typespec/versioning': link:packages/versioning + change-case: 5.4.4 + pluralize: 8.0.0 + + '@azure-tools/typespec-azure-rulesets@0.66.0(43a25ef11631422e4edc1bf6c9f3e1d2)': + dependencies: + '@azure-tools/typespec-azure-core': 0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest) + '@azure-tools/typespec-azure-resource-manager': 0.66.1(@azure-tools/typespec-azure-core@0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest))(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/openapi@packages+openapi)(@typespec/rest@packages+rest)(@typespec/versioning@packages+versioning) + '@azure-tools/typespec-client-generator-core': 0.66.4(@azure-tools/typespec-azure-core@0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest))(@typespec/compiler@packages+compiler)(@typespec/events@packages+events)(@typespec/http@packages+http)(@typespec/openapi@packages+openapi)(@typespec/rest@packages+rest)(@typespec/sse@packages+sse)(@typespec/streams@packages+streams)(@typespec/versioning@packages+versioning)(@typespec/xml@packages+xml) + '@typespec/compiler': link:packages/compiler + + '@azure-tools/typespec-client-generator-core@0.66.4(@azure-tools/typespec-azure-core@0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest))(@typespec/compiler@packages+compiler)(@typespec/events@packages+events)(@typespec/http@packages+http)(@typespec/openapi@packages+openapi)(@typespec/rest@packages+rest)(@typespec/sse@packages+sse)(@typespec/streams@packages+streams)(@typespec/versioning@packages+versioning)(@typespec/xml@packages+xml)': + dependencies: + '@azure-tools/typespec-azure-core': 0.66.1(@typespec/compiler@packages+compiler)(@typespec/http@packages+http)(@typespec/rest@packages+rest) + '@typespec/compiler': link:packages/compiler + '@typespec/events': link:packages/events + '@typespec/http': link:packages/http + '@typespec/openapi': link:packages/openapi + '@typespec/rest': link:packages/rest + '@typespec/sse': link:packages/sse + '@typespec/streams': link:packages/streams + '@typespec/versioning': link:packages/versioning + '@typespec/xml': link:packages/xml + change-case: 5.4.4 + pluralize: 8.0.0 + yaml: 2.8.3 + '@azure/abort-controller@2.1.2': dependencies: tslib: 2.8.1 diff --git a/website/src/components/playground-component/import-map.ts b/website/src/components/playground-component/import-map.ts index 88c30465fba..c4dfb196e4b 100644 --- a/website/src/components/playground-component/import-map.ts +++ b/website/src/components/playground-component/import-map.ts @@ -9,10 +9,41 @@ export interface ImportMap { imports: Record; } +const pkgsBaseUrl = "https://typespec.blob.core.windows.net/pkgs"; + +async function fetchAdditionalPackageImports( + packageNames: readonly string[], +): Promise> { + const imports: Record = {}; + + const results = await Promise.allSettled( + packageNames.map(async (name) => { + const url = `${pkgsBaseUrl}/${name}/latest.json`; + const response = await fetch(url); + if (!response.ok) { + // eslint-disable-next-line no-console + console.warn(`Failed to load latest index for ${name}: ${response.status}`); + return undefined; + } + return JSON.parse(await response.text()) as ImportMap; + }), + ); + + for (const result of results) { + if (result.status === "fulfilled" && result.value) { + Object.assign(imports, result.value.imports); + } + } + + return imports; +} + export async function loadImportMap({ latestVersion, + additionalPackages = [], }: { latestVersion: string; + additionalPackages?: readonly string[]; }): Promise { const optionsScript = document.querySelector("script[type=playground-options]"); if (optionsScript === undefined) { @@ -24,14 +55,15 @@ export async function loadImportMap({ const parsed = new URLSearchParams(window.location.search); const requestedVersion = parsed.get("version"); - const importMapUrl = `https://typespec.blob.core.windows.net/pkgs/indexes/typespec/${ - requestedVersion ?? latestVersion - }.json`; + const importMapUrl = `${pkgsBaseUrl}/indexes/typespec/${requestedVersion ?? latestVersion}.json`; - const response = await fetch(importMapUrl); - const content = await response.text(); + const [mainResponse, additionalImports] = await Promise.all([ + fetch(importMapUrl), + fetchAdditionalPackageImports(additionalPackages), + ]); - const importMap = JSON.parse(content); + const importMap: ImportMap = JSON.parse(await mainResponse.text()); + Object.assign(importMap.imports, additionalImports); (window as any).importShim.addImportMap(importMap); diff --git a/website/src/components/react-pages/playground.tsx b/website/src/components/react-pages/playground.tsx index 10e9d9bed06..5ed019fabbb 100644 --- a/website/src/components/react-pages/playground.tsx +++ b/website/src/components/react-pages/playground.tsx @@ -3,6 +3,8 @@ import { useEffect, useState, type ReactNode } from "react"; import { FluentLayout } from "../fluent/fluent-layout"; import { loadImportMap, type VersionData } from "../playground-component/import-map"; +const additionalPlaygroundPackages = ["@typespec/http-client-python"]; + export const AsyncPlayground = ({ latestVersion, fallback, @@ -15,7 +17,13 @@ export const AsyncPlayground = ({ WebsitePlayground: typeof import("../playground-component/playground").WebsitePlayground; }>(undefined as any); useEffect(() => { - Promise.all([loadImportMap({ latestVersion }), import("../playground-component/playground")]) + Promise.all([ + loadImportMap({ + latestVersion, + additionalPackages: additionalPlaygroundPackages, + }), + import("../playground-component/playground"), + ]) .then((x) => setMod({ versionData: x[0] as any, WebsitePlayground: x[1].WebsitePlayground })) .catch((e) => { throw e;