diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..a8e5685 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,187 @@ +name: Publish to npm + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + dry_run: + description: 'Dry run (skip actual publish)' + type: boolean + default: false + +# Required for npm provenance +permissions: + contents: read + id-token: write + +jobs: + validate: + name: Validate Release + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v4 + + - name: Extract version from tag + id: version + run: | + if [[ "${{ github.ref }}" == refs/tags/v* ]]; then + VERSION="${GITHUB_REF#refs/tags/v}" + else + VERSION=$(node -p "require('./package.json').version") + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Publishing version: $VERSION" + + - name: Validate version format + run: | + VERSION="${{ steps.version.outputs.version }}" + if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.-]+)?$ ]]; then + echo "Invalid version format: $VERSION" + exit 1 + fi + + - name: Check version matches package.json + if: startsWith(github.ref, 'refs/tags/v') + run: | + TAG_VERSION="${{ steps.version.outputs.version }}" + PKG_VERSION=$(node -p "require('./package.json').version") + if [[ "$TAG_VERSION" != "$PKG_VERSION" ]]; then + echo "Tag version ($TAG_VERSION) does not match package.json ($PKG_VERSION)" + exit 1 + fi + + test: + name: Test + needs: validate + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + node-version: [20, 22] + python-version: ['3.10', '3.11', '3.12'] + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: npm + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: npm ci --prefer-offline --no-audit + + - name: Install Python dependencies + run: | + cd tywrap_ir + pip install -e . + + - name: Lint + run: npm run lint + + - name: Typecheck + run: npm run typecheck + + - name: Type tests + run: npm run test:types + + - name: Build + run: npm run build + + - name: Run tests + env: + NODE_OPTIONS: --expose-gc + TYWRAP_PERF_BUDGETS: '1' + run: npm test + + test-os: + name: Test (${{ matrix.os }}) + needs: validate + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: npm ci --prefer-offline --no-audit + + - name: Install Python dependencies + run: | + cd tywrap_ir + pip install -e . + + - name: Run tests + env: + NODE_OPTIONS: --expose-gc + TYWRAP_PERF_BUDGETS: '1' + run: npm test + + publish: + name: Publish to npm + needs: [validate, test, test-os] + runs-on: ubuntu-latest + environment: npm + # Explicit permissions for OIDC trusted publishing + GitHub Release + permissions: + contents: write + id-token: write + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + registry-url: 'https://registry.npmjs.org' + + - name: Install dependencies + run: npm ci --prefer-offline --no-audit + + - name: Build + run: npm run build + + - name: Verify package contents + run: npm pack --dry-run + + - name: Publish (dry run) + if: inputs.dry_run == true + run: npm publish --dry-run + + # With OIDC trusted publishing, no NPM_TOKEN needed + # Provenance is automatic with trusted publishing + - name: Publish to npm + if: inputs.dry_run != true + run: npm publish --provenance --access public + + - name: Create GitHub Release + if: inputs.dry_run != true && startsWith(github.ref, 'refs/tags/v') + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + draft: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 69f7ee5..ebdefdb 100644 --- a/README.md +++ b/README.md @@ -1,327 +1,125 @@ # tywrap -TypeScript wrapper for Python libraries with full type safety. - -> **⚠️ EXPERIMENTAL SOFTWARE** -> **Version 0.1.0** - This project is in early experimental development. APIs may change significantly between versions. Not recommended for production use until version 1.0.0. +[![npm version](https://img.shields.io/npm/v/tywrap.svg)](https://www.npmjs.com/package/tywrap) +[![CI](https://github.com/bbopen/tywrap/actions/workflows/ci.yml/badge.svg)](https://github.com/bbopen/tywrap/actions/workflows/ci.yml) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -TypeScript is great. But there are many robust libraries in Python, especially for data and science. Sometimes, there's a library only in Python. Wouldn't it be great if you could those libraries them in your TypeScript project? +TypeScript wrapper for Python libraries with full type safety. -tywrap is a build-time code generation system that makes Python libraries feel native in TypeScript with zero runtime overhead and complete type safety. +> **⚠️ Experimental Software (v0.1.0)** - APIs may change between versions. Not recommended for production use until v1.0.0. ## Features -- **Full Type Safety** - Complete TypeScript type definitions generated from Python source -- **Zero Runtime Overhead** - Build-time code generation with optimized execution -- **Multi-Runtime Support** - Works in Node.js, Deno, Bun, and browsers -- **IR-First Generation** - Python IR extractor drives type-safe generation -- **Smart Caching** - Intelligent caching and batching for maximum performance -- **Developer Experience** - Hot reload, source maps, and IDE integration +- **Full Type Safety** - TypeScript definitions generated from Python source analysis +- **Multi-Runtime** - Node.js (subprocess) and browsers (Pyodide) +- **Rich Data Types** - numpy, pandas, scipy, torch, sklearn, and stdlib types +- **Efficient Serialization** - Apache Arrow binary format with JSON fallback ## Requirements -- Node.js >=20 (or Bun >=1.1 / Deno >=1.46) +- Node.js 20+ (or Bun 1.1+ / Deno 1.46+) - Python 3.10+ ## Quick Start ```bash -# npm npm install tywrap - -# pnpm -pnpm add tywrap - -# yarn -yarn add tywrap - -# bun -bun add tywrap - -# deno -deno add npm:tywrap +npx tywrap init # Create config +npx tywrap generate # Generate wrappers ``` -### Initialize a Config - -Create a starter config (defaults to `tywrap.config.ts`): - -```bash -npx tywrap init -``` - -Generate wrappers: - -```bash -npx tywrap generate -``` - -### Basic Usage - ```typescript import { NodeBridge } from 'tywrap/node'; import { setRuntimeBridge } from 'tywrap/runtime'; import * as math from './generated/math.generated.js'; -const bridge = new NodeBridge({ - pythonPath: '/usr/bin/python3', - virtualEnv: './venv' -}); - +const bridge = new NodeBridge({ pythonPath: 'python3' }); setRuntimeBridge(bridge); -const result = await math.sqrt(16); -console.log(result); +const result = await math.sqrt(16); // 4 ``` -## Runtime Support +## Runtime Bridges ### Node.js + ```typescript import { NodeBridge } from 'tywrap/node'; -import { setRuntimeBridge } from 'tywrap/runtime'; - const bridge = new NodeBridge({ - pythonPath: '/usr/bin/python3', - virtualEnv: './venv' + pythonPath: 'python3', + virtualEnv: './venv', + timeout: 30000 }); - -setRuntimeBridge(bridge); -``` - -### Deno -```typescript -import { tywrap } from 'npm:tywrap'; -``` - -### Bun -```typescript -import { tywrap } from 'tywrap'; -// Works out of the box with Bun's fast runtime ``` ### Browser (Pyodide) + ```typescript import { PyodideBridge } from 'tywrap/pyodide'; -import { setRuntimeBridge } from 'tywrap/runtime'; - const bridge = new PyodideBridge({ - indexURL: 'https://cdn.jsdelivr.net/pyodide/' + indexURL: 'https://cdn.jsdelivr.net/pyodide/v0.24.1/full/' }); - -setRuntimeBridge(bridge); +await bridge.init(); ``` -### Generated Classes +### Deno / Bun -Class wrappers are async-constructible: - -```ts -import { Counter } from './generated/collections.generated'; - -const counter = await Counter.create([1, 2, 2]); -const top = await counter.mostCommon(1); -await counter.disposeHandle(); +```typescript +import { NodeBridge } from 'npm:tywrap'; // Deno +import { NodeBridge } from 'tywrap'; // Bun ``` ## Configuration -Create a `tywrap.config.ts` file: - ```typescript +// tywrap.config.ts import { defineConfig } from 'tywrap'; export default defineConfig({ pythonModules: { - 'pandas': { - version: '2.1.0', - runtime: 'pyodide', - classes: ['DataFrame'], - functions: ['read_csv', 'concat'], - typeHints: 'strict' - }, - 'numpy': { - version: '1.24.0', - runtime: 'auto', // Auto-select best runtime - alias: 'np', - typeHints: 'strict' - }, - './custom_module.py': { - runtime: 'node', - typeHints: 'strict', - watch: true // Enable hot reload - } - }, - - output: { - dir: './src/generated', - format: 'esm', - declaration: true, - sourceMap: true + 'pandas': { classes: ['DataFrame'], functions: ['read_csv'] }, + 'numpy': { alias: 'np' } }, - - performance: { - caching: true, - batching: true, - compression: 'auto' - }, - - types: { - presets: ['stdlib', 'pandas'] - }, - - development: { - hotReload: true, - validation: 'runtime' - }, - - debug: true + output: { dir: './src/generated' } }); ``` -Enable `debug` to print cache and parallel processor diagnostics. - -Config file resolution order (CLI): `tywrap.config.ts`, `.mts`, `.js`, `.mjs`, `.cjs`, `.json`. - -### Configuration Fields - -- `pythonModules` – modules to wrap and their options -- `output` – directory, format and generated artifacts -- `runtime` – runtime paths and timeouts -- `performance` – caching and batching controls -- `types` – opt-in type mapping presets -- `development` – hot reloading and validation mode -- `debug` – enable verbose debug logging - -### Extension Hooks - -Plugins can extend tywrap via lifecycle hooks: - -- `beforeGeneration(options)` -- `afterGeneration(result)` -- `transformPythonType(type)` -- `transformTypescriptCode(code)` +See [Configuration Guide](./docs/configuration.md) for all options. -See the [Configuration Guide](./docs/configuration.md) for details. +## Supported Data Types -## Build Tool Integration +| Python | TypeScript | Notes | +|--------|-----------|-------| +| `numpy.ndarray` | `Uint8Array` / `array` | Arrow or JSON | +| `pandas.DataFrame` | Arrow Table / `object[]` | Arrow or JSON | +| `scipy.sparse.*` | `SparseMatrix` | CSR, CSC, COO | +| `torch.Tensor` | `TorchTensor` | CPU only | +| `sklearn estimator` | `SklearnEstimator` | Params only | +| `datetime`, `Decimal`, `UUID`, `Path` | `string` | Standard formats | -Build tool integrations are planned. For now, run `tywrap generate` in your build pipeline. +For Arrow encoding with numpy/pandas: -## Performance - -tywrap is designed for production use with enterprise-grade performance: - -- **30-50% faster** than runtime bridges -- **Zero runtime overhead** with build-time optimization -- **Smart bundling** with tree-shaking and code splitting -- **Intelligent caching** with automatic invalidation -- **Request batching** for optimal throughput - -## How It Works - -1. **Python IR Extraction** - `tywrap_ir` reflects Python modules and emits versioned JSON IR -2. **Type Mapping** - Converts Python IR annotations to TypeScript types -3. **Code Generation** - Generates TypeScript wrappers with runtime bridge hooks -4. **Runtime Execution** - Node.js (subprocess) MVP; others later - -## Arrow/Codec (optional) - -Numpy/Pandas results can be transported more efficiently using Arrow. tywrap emits structured envelopes from the Python side; on the JS side you can opt-in to Arrow decoding. - -Envelopes (from Python bridge): -- `{"__tywrap__":"dataframe","encoding":"arrow","b64":"..."}` (Feather/Arrow) -- `{"__tywrap__":"series","encoding":"arrow","b64":"..."}` or JSON fallback with `data` -- `{"__tywrap__":"ndarray","encoding":"arrow","b64":"...","shape":[...]} or JSON fallback with `data` -- `{"__tywrap__":"scipy.sparse","encoding":"json","format":"csr","shape":[...],"data":[...],...}` (sparse matrices) -- `{"__tywrap__":"torch.tensor","encoding":"ndarray","value":{...},"shape":[...],"dtype":"...","device":"cpu"}` (torch tensors) -- `{"__tywrap__":"sklearn.estimator","encoding":"json","className":"...","module":"...","params":{...}}` (estimator metadata) - -Enable decoding (Node/browser) when you have `apache-arrow` installed: - -```ts +```typescript import { registerArrowDecoder } from 'tywrap'; - -// If you have apache-arrow available, register a decoder once at app startup -import('apache-arrow').then(mod => { - const Table = (mod as { Table: { from: (i: Uint8Array | Iterable) => unknown } }).Table; - registerArrowDecoder(bytes => { - try { - return Table.from(bytes as Uint8Array); - } catch { - return Table.from([bytes as Uint8Array]); - } - }); -}); +import { tableFromIPC } from 'apache-arrow'; +registerArrowDecoder(bytes => tableFromIPC(bytes)); ``` -If you don't register a decoder, `decodeValue`/`decodeValueAsync` will throw for Arrow-encoded payloads. To accept raw bytes, register a decoder that returns the input `Uint8Array`. - -Fallback policy: By default, the Python bridge requires Arrow for DataFrame/Series/ndarray and will throw if unavailable. To opt into JSON fallback for development or constrained environments, set the environment variable `TYWRAP_CODEC_FALLBACK=json` when launching Python (e.g., `TYWRAP_CODEC_FALLBACK=json node app.js`). - -Torch tensors on GPU or non-contiguous tensors require an explicit opt-in: set `TYWRAP_TORCH_ALLOW_COPY=1` to allow CPU transfer/contiguous copies during serialization. +## Documentation -## Matrix quick run (optional) +- [Getting Started](./docs/getting-started.md) +- [Configuration](./docs/configuration.md) +- [API Reference](./docs/api/README.md) +- [Troubleshooting](./docs/troubleshooting/README.md) -Generate wrappers for a curated set of libraries to validate coverage locally. +## Contributing ```bash -npm run build -npm run matrix +npm install +npm test ``` -Notes: -- The harness creates `.tywrap/venv` and prefers `python3.12` for better wheel availability (e.g., pydantic-core). -- Results end up in `generated/`. You can tweak the list in `tools/matrix.js`. - -## Roadmap - -- [x] Core architecture and multi-runtime support -- [x] Python AST analysis and type extraction -- [x] TypeScript code generation -- [ ] Build tool integrations (Vite, Webpack, Rollup) -- [ ] Advanced optimizations (SharedArrayBuffer, streaming) -- [ ] IDE extensions and developer tools -- [ ] Enterprise features (security sandbox, monitoring) - -## Documentation - -- [Getting Started Guide](./docs/getting-started.md) - Get up and running in minutes -- [Configuration Reference](./docs/configuration.md) - Complete configuration options -- [Node.js Runtime](./docs/runtimes/nodejs.md) - Node.js integration guide -- [Browser Runtime](./docs/runtimes/browser.md) - Browser/Pyodide integration -- [API Reference](./docs/api/README.md) - Complete API documentation -- [Examples](./docs/examples/README.md) - Real-world usage examples -- [Troubleshooting](./docs/troubleshooting/README.md) - Common issues and solutions - -## Versioning - -tywrap follows [Semantic Versioning](https://semver.org/): - -- **0.x.x** - Experimental releases. Breaking changes may occur in any release -- **1.x.x** - Stable API. Breaking changes only in major versions -- **x.Y.x** - New features and improvements (backwards compatible) -- **x.x.Z** - Bug fixes and patches (backwards compatible) - -### Version 0.1.0 Status - -**Current State:** -- ✅ Core TypeScript generation working -- ✅ Node.js runtime bridge functional -- ✅ Multi-runtime support (Node.js, Deno, Bun, Browser) -- ✅ Type safety and IR extraction -- ⚠️ API surface may change significantly -- ⚠️ Limited real-world testing - -**Roadmap to 1.0:** -- Extensive testing with popular Python libraries -- API stabilization and documentation -- Performance optimization -- Production deployment guides - -## Contributing - -We welcome contributions! See [CONTRIBUTING](./CONTRIBUTING.md). +See [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines. ## License @@ -329,7 +127,6 @@ MIT © [tywrap contributors](LICENSE) ## Links -- [Documentation](https://tywrap.dev/docs) -- [API Reference](https://tywrap.dev/api) -- [Examples](https://github.com/tywrap/examples) -- [Discord Community](https://discord.gg/tywrap) +- [GitHub](https://github.com/bbopen/tywrap) +- [npm](https://www.npmjs.com/package/tywrap) +- [Issues](https://github.com/bbopen/tywrap/issues) diff --git a/docs/getting-started.md b/docs/getting-started.md index 5fdbfd8..9aa7124 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -336,5 +336,5 @@ const batch = await mathBatch([ ## Support - [Troubleshooting Guide](./troubleshooting/README.md) -- [GitHub Issues](https://github.com/tywrap/tywrap/issues) -- [GitHub Discussions](https://github.com/tywrap/tywrap/discussions) +- [GitHub Issues](https://github.com/bbopen/tywrap/issues) +- [GitHub Discussions](https://github.com/bbopen/tywrap/discussions) diff --git a/docs/release.md b/docs/release.md index bd43c47..b916a49 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,32 +1,22 @@ # Release -Use the release helper to bump versions, run checks, and optionally tag/publish. +## Creating a Release -## Typical flow +1. Ensure working tree is clean on main branch -1. Ensure your working tree is clean. -2. Run the release helper: +2. Bump version and tag: + ```sh + node scripts/release.mjs --commit --tag + ``` -```sh -node scripts/release.mjs --commit --tag -``` +3. Push: + ```sh + git push && git push --tags + ``` -3. Publish to npm when ready: +## Version Format -```sh -node scripts/release.mjs --publish -``` - -## Options - -- `--dry-run`: updates versions and runs checks, but skips git/npm side effects. -- `--allow-dirty`: allows running with uncommitted changes. -- `--commit`: creates a `release: vX.Y.Z` commit. -- `--tag`: creates a `vX.Y.Z` tag. -- `--publish`: runs `npm publish` (use after a successful tag/commit). - -After tagging, push tags with: - -```sh -git push --tags -``` +- `X.Y.Z` - stable release +- `X.Y.Z-alpha.N` - alpha +- `X.Y.Z-beta.N` - beta +- `X.Y.Z-rc.N` - release candidate diff --git a/docs/troubleshooting/README.md b/docs/troubleshooting/README.md index 810a443..b5d508e 100644 --- a/docs/troubleshooting/README.md +++ b/docs/troubleshooting/README.md @@ -597,8 +597,8 @@ npx tywrap generate ``` ### Community Resources -- **GitHub Issues**: [Report bugs](https://github.com/tywrap/tywrap/issues) -- **Discussions**: [Ask questions](https://github.com/tywrap/tywrap/discussions) +- **GitHub Issues**: [Report bugs](https://github.com/bbopen/tywrap/issues) +- **Discussions**: [Ask questions](https://github.com/bbopen/tywrap/discussions) - **Discord**: [Real-time help](https://discord.gg/tywrap) - **Stack Overflow**: Tag your questions with `tywrap` diff --git a/package.json b/package.json index 23a5afe..70d1bee 100644 --- a/package.json +++ b/package.json @@ -82,12 +82,16 @@ "license": "MIT", "repository": { "type": "git", - "url": "https://github.com/tywrap/tywrap.git" + "url": "https://github.com/bbopen/tywrap.git" }, "bugs": { - "url": "https://github.com/tywrap/tywrap/issues" + "url": "https://github.com/bbopen/tywrap/issues" + }, + "homepage": "https://github.com/bbopen/tywrap#readme", + "publishConfig": { + "access": "public", + "registry": "https://registry.npmjs.org/" }, - "homepage": "https://github.com/tywrap/tywrap#readme", "dependencies": { "@babel/parser": "^7.25.6", "@babel/types": "^7.25.6",