XPU Enablement for InternVLA-N1-DualVLN

README.md

@@ -63,6 +63,8 @@ The toolbox supports the most advanced high-quality navigation dataset, InternDa
 
 Please refer to the [documentation](https://internrobotics.github.io/user_guide/internnav/quick_start/index.html) for quick start with InternNav, from installation to training or evaluating supported models.
 
+For native Intel XPU inference, see [setup and validation limits](docs/xpu_inference.md). The existing CUDA installation route is unchanged.
+
 ## 📦 Overview
 
 ### 🧪 Supported Benchmarks

docs/xpu_inference.md

@@ -0,0 +1,76 @@
+# Optional Intel XPU eager inference
+
+Eager inference for InternVLA-N1-DualVLN on Intel GPUs.
+
+## Installation
+
+Install the Intel GPU driver and runtime per the
+[PyTorch Intel GPU prerequisites](https://docs.pytorch.org/docs/stable/notes/get_start_xpu.html).
+Then, from the repository root, in a dedicated virtual environment:
+
+```bash
+git submodule update --init internnav/model/basemodel/LongCLIP
+venv/bin/python -m pip install -r requirements/internvla_n1_xpu.txt
+venv/bin/python -m pip check
+venv/bin/python -c "import torch; assert torch.xpu.is_available(); print(torch.__version__); print(torch.xpu.get_device_properties(0)); print(torch.xpu.mem_get_info(0))"
+```
+
+Do not install `xformers` or `flash-attn` in this environment. Keep `diffusers==0.32.2`.
+
+## Auxiliary checkpoint
+
+Download the Depth Anything V2 checkpoint into `checkpoints/`:
+
+```bash
+curl -fL --retry 2 -o checkpoints/depth_anything_v2_metric_hypersim_vits.pth \
+  https://huggingface.co/depth-anything/Depth-Anything-V2-Metric-Hypersim-Small/resolve/main/depth_anything_v2_metric_hypersim_vits.pth
+sha256sum checkpoints/depth_anything_v2_metric_hypersim_vits.pth
+```
+
+## Configuration
+
+Set `device` and, optionally, `attn_implementation` (default `auto`) in `model_settings`
+of the evaluation config, e.g. `scripts/eval/configs/h1_internvla_n1_async_cfg.py`:
+
+```python
+model_settings={
+    ...
+    'model_path': 'checkpoints/InternVLA-N1-DualVLN',
+    'device': 'xpu:0',
+    'attn_implementation': 'auto',
+}
+```
+
+| Device | `auto` | Other selections |
+| --- | --- | --- |
+| CUDA | `flash_attention_2` | `sdpa`, `eager` |
+| XPU | `sdpa` | `eager`; `flash_attention_2` is not supported |
+
+The real-world HTTP server uses `auto`.
+
+## Launch
+
+```bash
+venv/bin/python -m scripts.realworld.http_internvla_server \
+  --model_path checkpoints/InternVLA-N1-DualVLN \
+  --device xpu:0 --plan_step_gap 4 --skip_warmup
+```
+
+The server listens on port 5801.
+
+## Memory
+
+Model weights alone need 15.62 GiB of free GPU memory; inference requires more.
+
+## Tests
+
+```bash
+venv/bin/python -m pip install pytest
+INTERNNAV_FULL_XPU_TEST=1 venv/bin/python -m pytest tests/unit_test/test_xpu_inference.py -q -s
+```
+
+CUDA equivalence tests, in a CUDA environment:
+
+```bash
+python -m pytest tests/unit_test/test_xpu_inference.py -q -rs -k "accelerator or cuda"
+```

internnav/agent/internvla_n1_agent_realworld.py

@@ -18,20 +18,21 @@
 from transformers import AutoProcessor
 
 from internnav.model.basemodel.internvla_n1.internvla_n1 import InternVLAN1ForCausalLM
+from internnav.model.utils.inference_device import resolve_inference_device
 from internnav.model.utils.vln_utils import S2Output, split_and_clean, traj_to_actions
 
 DEFAULT_IMAGE_TOKEN = "<image>"
 
 
 class InternVLAN1AsyncAgent:
     def __init__(self, args):
-        self.device = torch.device(args.device)
+        self.device, attention = resolve_inference_device(args.device, getattr(args, 'attn_implementation', 'auto'))
         self.save_dir = "test_data/" + datetime.now().strftime("%Y%m%d_%H%M%S")
         print(f"args.model_path{args.model_path}")
         self.model = InternVLAN1ForCausalLM.from_pretrained(
             args.model_path,
             torch_dtype=torch.bfloat16,
-            attn_implementation="flash_attention_2",
+            attn_implementation=attention,
             device_map={"": self.device},
         )
         self.model.eval()

internnav/model/basemodel/internvla_n1/internvla_n1_policy.py

@@ -14,6 +14,7 @@
     InternVLAN1ForCausalLM,
     InternVLAN1ModelConfig,
 )
+from internnav.model.utils.inference_device import resolve_inference_device
 from internnav.model.utils.vln_utils import (
     S1Output,
     S2Output,
@@ -30,11 +31,14 @@ def __init__(self, config: Union[InternVLAN1ModelConfig, ModelCfg]):
         super().__init__(config)
         self.model_config = ModelCfg(**config.model_cfg['model'])
 
+        device, attention = resolve_inference_device(
+            self.model_config.device, getattr(self.model_config, 'attn_implementation', 'auto')
+        )
         self.model = InternVLAN1ForCausalLM.from_pretrained(
             self.model_config.model_path,
             torch_dtype=torch.bfloat16,
-            attn_implementation="flash_attention_2",
-            device_map={"": self.model_config.device},
+            attn_implementation=attention,
+            device_map={"": device},
         )
 
         self.tokenizer = AutoTokenizer.from_pretrained(self.model_config.model_path, use_fast=True)

internnav/model/utils/inference_device.py

@@ -0,0 +1,20 @@
+import torch
+
+
+def resolve_inference_device(device, attn_implementation='auto'):
+    device = torch.device(device)
+    supported = ('auto', 'flash_attention_2', 'sdpa', 'eager')
+    if attn_implementation not in supported:
+        raise ValueError(f'Attention must be one of {supported}, got {attn_implementation!r}')
+    if device.type == 'xpu':
+        if attn_implementation == 'flash_attention_2':
+            raise ValueError('External FlashAttention 2 is not supported on XPU; use sdpa or eager')
+        if not hasattr(torch, 'xpu') or not torch.xpu.is_available():
+            raise RuntimeError(
+                'XPU was requested but is unavailable; check native PyTorch XPU wheels and Intel drivers'
+            )
+        if device.index is not None and device.index >= torch.xpu.device_count():
+            raise RuntimeError(f'Requested XPU device {device} does not exist')
+    if attn_implementation == 'auto':
+        attn_implementation = 'flash_attention_2' if device.type == 'cuda' else 'sdpa'
+    return device, attn_implementation

requirements/internvla_n1_xpu.txt

@@ -0,0 +1,19 @@
+--extra-index-url https://download.pytorch.org/whl/xpu
+torch==2.14.1+xpu
+torchvision==0.29.1+xpu
+accelerate==1.4.0
+diffusers==0.32.2
+transformers==4.51.0
+ftfy==6.3.1
+einops==0.8.1
+numpy==1.26.4
+opencv-python-headless==4.11.0.86
+pydantic>=2.11,<2.12
+flask>=3.1,<4
+gym==0.26.2
+imageio==2.37.0
+pillow>=11,<12
+numpy-quaternion==2024.0.13
+scipy>=1.15,<1.18
+fastapi>=0.110,<0.111
+uvicorn>=0.29,<0.31

scripts/realworld/http_internvla_server.py

@@ -85,18 +85,21 @@ def eval_dual():
     parser.add_argument("--resize_w", type=int, default=384)
     parser.add_argument("--resize_h", type=int, default=384)
     parser.add_argument("--num_history", type=int, default=8)
+    parser.add_argument("--plan_step_gap", type=int, default=4)
+    parser.add_argument("--skip_warmup", action="store_true")
     args = parser.parse_args()
 
     args.camera_intrinsic = np.array(
         [[386.5, 0.0, 328.9, 0.0], [0.0, 386.5, 244, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]
     )
     agent = InternVLAN1AsyncAgent(args)
-    agent.step(
-        np.zeros((480, 640, 3)),
-        np.zeros((480, 640)),
-        np.eye(4),
-        "hello",
-    )
+    if not args.skip_warmup:
+        agent.step(
+            np.zeros((480, 640, 3)),
+            np.zeros((480, 640)),
+            np.eye(4),
+            "hello",
+        )
     agent.reset()
 
     app.run(host='0.0.0.0', port=5801)