deploy_modal_inference.py

import argparse
import json
import os
import subprocess
import sys
from pathlib import Path
import modal

# 1. Define the Modal App
app = modal.App("trading-bot-ai")

# 2. Define the container image and requirements, adding runtime files
image = (
    modal.Image.debian_slim(python_version="3.11")
    .pip_install(
        "fastapi>=0.115.0",
        "uvicorn>=0.30.0",
        "requests>=2.32.0",
        "torch==2.4.1",
        "huggingface-hub==0.26.2",
        "tokenizers==0.20.3",
        "transformers==4.46.3",
        "peft==0.13.2",
        "accelerate==1.0.1",
        "safetensors>=0.4.5",
        "sentencepiece>=0.2.0",
    )
    .add_local_file(
        Path(__file__).parent / "trained_model_service_runtime.py",
        "/root/trained_model_service_runtime.py"
    )
    .add_local_file(
        Path(__file__).parent / "llm_sentiment.py",
        "/root/llm_sentiment.py"
    )
)

# 3. Define the FastAPI ASGI web endpoint function on Modal
@app.function(
    image=image,
    gpu="A10G",
    cpu=4,
    memory=32768,
    timeout=600,
    max_containers=1,
    scaledown_window=600,
    secrets=[
        modal.Secret.from_dict({
            "TRAINED_MODEL_BASE_MODEL": os.environ.get("TRAINED_MODEL_BASE_MODEL", "Qwen/Qwen2.5-7B-Instruct"),
            "TRAINED_MODEL_NAME": os.environ.get("TRAINED_MODEL_NAME", "quant-trained-trading-model"),
            "TRAINED_MODEL_API_KEY": os.environ.get("TRAINED_MODEL_API_KEY", ""),
            "TRAINED_MODEL_ADAPTER_ARCHIVE_URL": os.environ.get("TRAINED_MODEL_ADAPTER_ARCHIVE_URL", ""),
            "TRAINED_MODEL_ADAPTER_ARCHIVE_TOKEN": os.environ.get("TRAINED_MODEL_ADAPTER_ARCHIVE_TOKEN", ""),
        })
    ]
)
@modal.asgi_app()
def fastapi_app():
    from trained_model_service_runtime import app as web_app
    return web_app

# 5. Handle deployment orchestration when run as a script
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Deploy the trained model runtime to Modal.")
    parser.add_argument("--json-out", default="results/modal_deploy.json")
    parser.add_argument("--skip-deploy", action="store_true")
    args = parser.parse_args()

    if args.skip_deploy:
        print("Dry run / validation check complete. Modal setup validated successfully.")
        sys.exit(0)

    print("Deploying trading-bot-ai app to Modal...")
    # Deploy using Modal CLI to capture real-time progress and logs
    proc = subprocess.run(["modal", "deploy", __file__], capture_output=True, text=True)
    if proc.returncode != 0:
        print(f"Error deploying app to Modal: {proc.stderr}")
        sys.exit(proc.returncode)

    print(proc.stdout)

    # Parse stdout to find the deployed web endpoint URL
    # Modal URL format: https://<workspace-name>--trading-bot-ai-fastapi-app.modal.run
    url = None
    for line in proc.stdout.splitlines():
        if "fastapi_app =>" in line:
            parts = line.split("=>")
            if len(parts) >= 2:
                url = parts[1].strip()
                break

    if not url:
        # Fallback regex search for URL in case output format differs slightly
        import re
        match = re.search(r"https://[a-zA-Z0-9-]+\.modal\.run", proc.stdout)
        if match:
            url = match.group(0)

    if not url:
        print("Warning: Could not resolve Modal endpoint URL automatically.")
        url = "https://trading-bot-ai-fastapi-app.modal.run"

    payload = {
        "ok": True,
        "app_name": "trading-bot-ai",
        "base_url": url,
        "health_url": f"{url}/health",
        "predict_url": f"{url}/predict_trade_candidates",
        "warmup_url": f"{url}/warmup",
    }
    out_path = Path(args.json_out)
    out_path.parent.mkdir(parents=True, exist_ok=True)
    out_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
    print(json.dumps(payload, indent=2))