feat: inline Arrow IPC support + Python backend (appkit-py)

knip.json

@@ -3,6 +3,7 @@
   "ignoreWorkspaces": [
     "packages/shared",
     "packages/lakebase",
+    "packages/appkit-py",
     "apps/**",
     "docs"
   ],
@@ -15,7 +16,9 @@
     "**/*.example.tsx",
     "template/**",
     "tools/**",
-    "docs/**"
+    "docs/**",
+    "client/**",
+    "test-e2e-minimal.ts"
   ],
   "ignoreBinaries": ["tarball"]
 }

packages/appkit-py/.gitignore

@@ -0,0 +1,25 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+*.egg
+dist/
+build/
+.eggs/
+
+# Virtual environment
+.venv/
+venv/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+
+# Testing
+.pytest_cache/
+htmlcov/
+.coverage
+
+# OS
+.DS_Store

packages/appkit-py/pyproject.toml

@@ -0,0 +1,48 @@
+[project]
+name = "appkit-py"
+version = "0.1.0"
+description = "Python backend for Databricks AppKit — 100% API compatible with the TypeScript version"
+requires-python = ">=3.12"
+dependencies = [
+    "fastapi>=0.115",
+    "uvicorn[standard]>=0.30",
+    "starlette>=0.40",
+    "databricks-sdk>=0.30",
+    "pyarrow>=14.0",
+    "httpx>=0.27",
+    "pydantic>=2.0",
+    "cachetools>=5.3",
+    "python-dotenv>=1.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+    "httpx>=0.27",
+    "pytest-cov>=5.0",
+    "ruff>=0.5",
+    "mypy>=1.10",
+]
+
+[build-system]
+requires = ["setuptools>=68.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+markers = [
+    "integration: marks tests that require a running backend server",
+    "unit: marks unit tests that run in isolation",
+]
+
+[tool.ruff]
+target-version = "py312"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "W"]

packages/appkit-py/src/appkit_py/__init__.py

@@ -0,0 +1 @@
+"""Python backend for Databricks AppKit — 100% API compatible with the TypeScript version."""

packages/appkit-py/src/appkit_py/__main__.py

@@ -0,0 +1,25 @@
+"""Entry point for running the AppKit Python backend with `python -m appkit_py`."""
+
+import os
+
+from dotenv import load_dotenv
+
+
+def main() -> None:
+    load_dotenv()
+
+    import uvicorn
+
+    from appkit_py.server import create_server
+
+    # Match TS AppKit env vars for compatibility
+    host = os.environ.get("FLASK_RUN_HOST", os.environ.get("APPKIT_HOST", "0.0.0.0"))
+    port = int(os.environ.get("DATABRICKS_APP_PORT", "8000"))
+    log_level = os.environ.get("APPKIT_LOG_LEVEL", "info")
+
+    app = create_server()
+    uvicorn.run(app, host=host, port=port, log_level=log_level)
+
+
+if __name__ == "__main__":
+    main()

packages/appkit-py/src/appkit_py/cache/cache_manager.py

@@ -0,0 +1,86 @@
+"""CacheManager with TTL-based in-memory caching.
+
+Mirrors the TypeScript CacheManager from packages/appkit/src/cache/index.ts.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from typing import Any, Awaitable, Callable, TypeVar
+
+T = TypeVar("T")
+
+
+class CacheManager:
+    """In-memory TTL cache with SHA256 key generation."""
+
+    _instance: CacheManager | None = None
+
+    def __init__(self) -> None:
+        self._store: dict[str, tuple[Any, float]] = {}  # key -> (value, expires_at)
+
+    @classmethod
+    def get_instance(cls) -> CacheManager:
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    @classmethod
+    def get_instance_sync(cls) -> CacheManager:
+        return cls.get_instance()
+
+    @classmethod
+    def reset(cls) -> None:
+        cls._instance = None
+
+    def generate_key(self, parts: list[Any], user_key: str) -> str:
+        """Generate a SHA256 cache key from parts and user key."""
+        raw = json.dumps([user_key] + [str(p) for p in parts], sort_keys=True)
+        return hashlib.sha256(raw.encode()).hexdigest()
+
+    async def get_or_execute(
+        self,
+        key_parts: list[Any],
+        fn: Callable[[], Awaitable[T]],
+        user_key: str,
+        ttl: float = 300,
+    ) -> T:
+        """Get cached value or execute function and cache the result."""
+        cache_key = self.generate_key(key_parts, user_key)
+
+        # Check cache
+        if cache_key in self._store:
+            value, expires_at = self._store[cache_key]
+            if time.time() < expires_at:
+                return value
+            else:
+                del self._store[cache_key]
+
+        # Execute and cache
+        result = await fn()
+        self._store[cache_key] = (result, time.time() + ttl)
+        return result
+
+    def get(self, key: str) -> Any | None:
+        if key in self._store:
+            value, expires_at = self._store[key]
+            if time.time() < expires_at:
+                return value
+            del self._store[key]
+        return None
+
+    def set(self, key: str, value: Any, ttl: float = 300) -> None:
+        self._store[key] = (value, time.time() + ttl)
+
+    def delete(self, key: str) -> None:
+        self._store.pop(key, None)
+
+    def has(self, key: str) -> bool:
+        if key in self._store:
+            _, expires_at = self._store[key]
+            if time.time() < expires_at:
+                return True
+            del self._store[key]
+        return False

packages/appkit-py/src/appkit_py/connectors/files/client.py

@@ -0,0 +1,168 @@
+"""Files connector wrapping databricks.sdk.
+
+Mirrors packages/appkit/src/connectors/files/client.ts
+"""
+
+from __future__ import annotations
+
+import asyncio
+import io
+import logging
+import mimetypes
+from typing import Any
+
+from databricks.sdk import WorkspaceClient
+
+logger = logging.getLogger("appkit.connector.files")
+
+# Maximum path length (matching TS)
+MAX_PATH_LENGTH = 4096
+
+
+class FilesConnector:
+    """Perform file operations on Unity Catalog Volumes via Databricks SDK."""
+
+    def __init__(self, default_volume: str | None = None) -> None:
+        self.default_volume = default_volume or ""
+
+    def resolve_path(self, file_path: str) -> str:
+        """Resolve a relative path against the default volume.
+
+        Rejects path traversal sequences to prevent escaping the volume.
+        """
+        # Reject traversal sequences
+        if ".." in file_path:
+            raise ValueError(f"Path must not contain '..': {file_path}")
+
+        if file_path.startswith("/Volumes/"):
+            return file_path
+        # Strip leading slash and join with volume path
+        clean = file_path.lstrip("/")
+        return f"{self.default_volume.rstrip('/')}/{clean}"
+
+    async def list(
+        self, client: WorkspaceClient, directory_path: str | None = None
+    ) -> list[dict[str, Any]]:
+        """List directory contents."""
+        path = self.resolve_path(directory_path or "")
+        entries = await asyncio.to_thread(
+            lambda: list(client.files.list_directory_contents(path))
+        )
+        return [
+            {
+                "name": e.name,
+                "path": e.path,
+                "is_directory": e.is_directory,
+                "file_size": e.file_size,
+                "last_modified": e.last_modified,
+            }
+            for e in entries
+        ]
+
+    async def read(
+        self, client: WorkspaceClient, file_path: str, options: dict | None = None
+    ) -> str:
+        """Read file as text, enforcing optional maxSize limit."""
+        max_size = (options or {}).get("maxSize")
+        path = self.resolve_path(file_path)
+        response = await asyncio.to_thread(client.files.download, path)
+
+        if max_size:
+            content = response.contents.read(max_size + 1)
+            if isinstance(content, bytes) and len(content) > max_size:
+                raise ValueError(
+                    f"File exceeds maximum read size ({max_size} bytes)"
+                )
+        else:
+            content = response.contents.read()
+
+        if isinstance(content, bytes):
+            return content.decode("utf-8", errors="replace")
+        return content
+
+    async def download(
+        self, client: WorkspaceClient, file_path: str
+    ) -> dict[str, Any]:
+        """Download file as binary stream."""
+        path = self.resolve_path(file_path)
+        response = await asyncio.to_thread(client.files.download, path)
+        return {"contents": response.contents, "content_type": response.content_type}
+
+    async def exists(self, client: WorkspaceClient, file_path: str) -> bool:
+        """Check if a file exists."""
+        path = self.resolve_path(file_path)
+        try:
+            await asyncio.to_thread(client.files.get_metadata, path)
+            return True
+        except Exception:
+            return False
+
+    async def metadata(
+        self, client: WorkspaceClient, file_path: str
+    ) -> dict[str, Any]:
+        """Get file metadata."""
+        path = self.resolve_path(file_path)
+        meta = await asyncio.to_thread(client.files.get_metadata, path)
+        return {
+            "contentLength": meta.content_length,
+            "contentType": meta.content_type,
+            "lastModified": str(meta.last_modified) if meta.last_modified else None,
+        }
+
+    async def upload(
+        self,
+        client: WorkspaceClient,
+        file_path: str,
+        contents: bytes | io.IOBase,
+        options: dict | None = None,
+    ) -> None:
+        """Upload file contents."""
+        path = self.resolve_path(file_path)
+        overwrite = (options or {}).get("overwrite", True)
+        if isinstance(contents, bytes):
+            contents = io.BytesIO(contents)
+        await asyncio.to_thread(
+            client.files.upload, path, contents, overwrite=overwrite
+        )
+
+    async def create_directory(
+        self, client: WorkspaceClient, directory_path: str
+    ) -> None:
+        """Create a directory."""
+        path = self.resolve_path(directory_path)
+        await asyncio.to_thread(client.files.create_directory, path)
+
+    async def delete(self, client: WorkspaceClient, file_path: str) -> None:
+        """Delete a file."""
+        path = self.resolve_path(file_path)
+        await asyncio.to_thread(client.files.delete, path)
+
+    async def preview(
+        self, client: WorkspaceClient, file_path: str
+    ) -> dict[str, Any]:
+        """Get a preview of a file (metadata + text preview for text files)."""
+        path = self.resolve_path(file_path)
+        meta = await asyncio.to_thread(client.files.get_metadata, path)
+        content_type = meta.content_type or mimetypes.guess_type(file_path)[0] or ""
+        is_text = content_type.startswith("text/") or content_type in (
+            "application/json", "application/xml", "application/javascript",
+        )
+        is_image = content_type.startswith("image/")
+
+        text_preview = None
+        if is_text:
+            try:
+                response = await asyncio.to_thread(client.files.download, path)
+                raw = response.contents.read(1024)
+                text_preview = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
+            except Exception:
+                pass
+
+        return {
+            "contentLength": meta.content_length,
+            "contentType": meta.content_type,
+            "lastModified": str(meta.last_modified) if meta.last_modified else None,
+            "textPreview": text_preview,
+            "isText": is_text,
+            "isImage": is_image,
+        }