-
-
Prompt Tokens
-
${tokenTotals.prompt.toLocaleString()}
-
-
-
Completion Tokens
-
${tokenTotals.completion.toLocaleString()}
+
+
+
+
⚡ Performance
+
✅ Quality
+
🖼️ Vision
-
-
Total Tokens
-
${tokenTotals.total.toLocaleString()}
-
-
-
Throughput
-
${tokPerSec}
-
tokens/second
+
+
-
-${allResults.length > 1 ? `
Historical Comparison
-
- | Date | Model | LLM | LLM % | VLM | VLM % | Time | Tokens |
|---|
- ${historyRows}
-
` : ''}
-
-
System Configuration
-
-
OS${system.os || '?'}
-
CPU${system.cpu || '?'}
-
Cores${system.cpuCores || '?'}
-
RAM${system.totalMemoryGB || '?'} GB total
-
Free RAM${system.freeMemoryGB || '?'} GB
-
Node${system.nodeVersion || '?'}
-
Process RSS${system.processMemoryMB?.rss || '?'} MB
-
Heap Used${system.processMemoryMB?.heapUsed || '?'} MB
+
+
-
+
-
+
`;
}
@@ -288,4 +952,23 @@ if (require.main === module) {
generateReport();
}
+function buildLiveBanner(status) {
+ if (!status) {
+ return `
Benchmark starting\u2026
`;
+ }
+ const { suitesCompleted = 0, totalSuites = 0, currentSuite = '', currentTest = '', testsCompleted = 0, startedAt = '' } = status;
+ const pct = totalSuites > 0 ? Math.round((suitesCompleted / totalSuites) * 100) : 0;
+ const elapsed = startedAt ? Math.round((Date.now() - new Date(startedAt).getTime()) / 1000) : 0;
+ const elapsedStr = elapsed > 60 ? Math.floor(elapsed / 60) + 'm ' + (elapsed % 60) + 's' : elapsed + 's';
+ const testInfo = currentTest ? ` — ✅
${escHtml(currentTest)}` : '';
+ return `
+
+
LIVE — Suite ${suitesCompleted}/${totalSuites} (${pct}%)
+ ${currentSuite ? ' — 🔧
' + escHtml(currentSuite) + '' : ''}
+ ${testInfo}
+
${testsCompleted} tests · ${elapsedStr} elapsed
+
+
`;
+}
+
module.exports = { generateReport };
diff --git a/skills/analysis/home-security-benchmark/scripts/run-benchmark.cjs b/skills/analysis/home-security-benchmark/scripts/run-benchmark.cjs
index c0f32fa9..8598be17 100644
--- a/skills/analysis/home-security-benchmark/scripts/run-benchmark.cjs
+++ b/skills/analysis/home-security-benchmark/scripts/run-benchmark.cjs
@@ -85,7 +85,8 @@ const VLM_URL = process.env.AEGIS_VLM_URL || getArg('vlm', '');
const RESULTS_DIR = getArg('out', path.join(os.homedir(), '.aegis-ai', 'benchmarks'));
const IS_SKILL_MODE = !!process.env.AEGIS_SKILL_ID;
const NO_OPEN = args.includes('--no-open') || skillParams.noOpen || false;
-const TEST_MODE = skillParams.mode || 'full';
+// Auto-detect mode: if no VLM URL, default to 'llm' (skip VLM image-analysis tests)
+const TEST_MODE = skillParams.mode || (VLM_URL ? 'full' : 'llm');
const IDLE_TIMEOUT_MS = 30000; // Streaming idle timeout — resets on each received token
const FIXTURES_DIR = path.join(__dirname, '..', 'fixtures');
@@ -155,6 +156,8 @@ const results = {
suites: [],
totals: { passed: 0, failed: 0, skipped: 0, total: 0, timeMs: 0 },
tokenTotals: { prompt: 0, completion: 0, total: 0 },
+ perfTotals: { ttftMs: [], decodeTokensPerSec: [], prefillTokensPerSec: null, serverDecodeTokensPerSec: null },
+ resourceSamples: [], // GPU/memory snapshots taken after each suite
};
async function llmCall(messages, opts = {}) {
@@ -165,9 +168,10 @@ async function llmCall(messages, opts = {}) {
}
const model = opts.model || (opts.vlm ? VLM_MODEL : LLM_MODEL) || undefined;
- // For JSON-expected tests, disable thinking (Qwen3.5 doesn't support /no_think)
- // Method 1: Inject empty
assistant prefix to skip reasoning phase
- // Method 2: chat_template_kwargs via extra_body (works if server supports it)
+ // For JSON-expected tests, use low temperature + top_p to encourage
+ // direct JSON output without extended reasoning.
+ // NOTE: Do NOT inject assistant prefill — Qwen3.5 rejects prefill
+ // when enable_thinking is active (400 error).
if (opts.expectJSON) {
messages = [...messages];
// Remove any leftover /no_think from messages
@@ -177,20 +181,62 @@ async function llmCall(messages, opts = {}) {
}
return m;
});
- // Inject empty think block as assistant prefix (most portable method)
- messages.push({ role: 'assistant', content: '
\n\n' });
+ // Append JSON guidance to last user message for local models
+ const lastUser = messages.findLastIndex(m => m.role === 'user');
+ if (lastUser >= 0 && typeof messages[lastUser].content === 'string') {
+ messages[lastUser] = {
+ ...messages[lastUser],
+ content: messages[lastUser].content + '\n\nRespond with ONLY valid JSON, no explanation or markdown.',
+ };
+ }
}
+ // Sanitize messages for llama-server compatibility:
+ // - Replace null content with empty string (llama-server rejects null)
+ // - Convert tool_calls assistant messages to plain text (llama-server
+ // doesn't support OpenAI tool_calls format in conversation history)
+ // - Convert tool result messages to user messages
+ messages = messages.map(m => {
+ if (m.role === 'assistant' && m.tool_calls) {
+ // Convert tool call to text representation
+ const callDesc = m.tool_calls.map(tc => {
+ const argStr = typeof tc.function.arguments === 'string'
+ ? tc.function.arguments
+ : JSON.stringify(tc.function.arguments);
+ return `[Calling ${tc.function.name}(${argStr})]`;
+ }).join('\n');
+ return { role: 'assistant', content: callDesc };
+ }
+ if (m.role === 'tool') {
+ // Convert tool result to user message
+ return { role: 'user', content: `[Tool result]: ${m.content}` };
+ }
+ return {
+ ...m,
+ ...(m.content === null && { content: '' }),
+ };
+ });
+
+ // Determine the correct max-tokens parameter name:
+ // - OpenAI cloud (GPT-5.4+): requires 'max_completion_tokens', rejects 'max_tokens'
+ // - Local llama-server: requires 'max_tokens', may not understand 'max_completion_tokens'
+ const isCloudApi = !opts.vlm && (LLM_API_TYPE === 'openai' || LLM_BASE_URL.includes('openai.com') || LLM_BASE_URL.includes('api.anthropic'));
+
+ // No max_tokens for any API — the streaming loop's 2000-token hard cap is the safety net.
+ // Sending max_tokens to thinking models (Qwen3.5) starves actual output since
+ // reasoning_content counts against the limit.
+
// Build request params
const params = {
messages,
stream: true,
+ // Request token usage in streaming response (only supported by cloud APIs;
+ // llama-server crashes with "Failed to parse input" when stream_options is present)
+ ...(isCloudApi && { stream_options: { include_usage: true } }),
...(model && { model }),
...(opts.temperature !== undefined && { temperature: opts.temperature }),
- ...(opts.maxTokens && { max_completion_tokens: opts.maxTokens }),
- // Qwen3.5 non-thinking mode recommended params
...(opts.expectJSON && opts.temperature === undefined && { temperature: 0.7 }),
- ...(opts.expectJSON && { top_p: 0.8, presence_penalty: 1.5 }),
+ ...(opts.expectJSON && { top_p: 0.8 }),
...(opts.tools && { tools: opts.tools }),
};
@@ -228,6 +274,7 @@ async function llmCall(messages, opts = {}) {
}
}
+ const callStartTime = Date.now();
try {
const stream = await client.chat.completions.create(params, {
signal: controller.signal,
@@ -240,6 +287,7 @@ async function llmCall(messages, opts = {}) {
let usage = {};
let tokenCount = 0;
let tokenBuffer = '';
+ let firstTokenTime = null; // For TTFT measurement
for await (const chunk of stream) {
resetIdle();
@@ -251,6 +299,8 @@ async function llmCall(messages, opts = {}) {
if (delta?.reasoning_content) reasoningContent += delta.reasoning_content;
if (delta?.content || delta?.reasoning_content) {
tokenCount++;
+ // Capture TTFT on first content/reasoning token
+ if (!firstTokenTime) firstTokenTime = Date.now();
// Buffer and log tokens — tag with field source
const isContent = !!delta?.content;
const tok = delta?.content || delta?.reasoning_content || '';
@@ -266,10 +316,10 @@ async function llmCall(messages, opts = {}) {
}
// Smart early abort for JSON-expected tests:
- // If the model is producing reasoning_content (thinking) for a JSON test,
- // abort after 100 reasoning tokens — it should output JSON directly.
- if (opts.expectJSON && !isContent && tokenCount > 100) {
- log(` ⚠ Aborting: ${tokenCount} reasoning tokens for JSON test — model is thinking instead of outputting JSON`);
+ // Allow thinking models (Qwen3.5) up to 500 reasoning tokens before aborting.
+ // They legitimately need to reason before outputting JSON.
+ if (opts.expectJSON && !isContent && tokenCount > 500) {
+ log(` ⚠ Aborting: ${tokenCount} reasoning tokens for JSON test — model is thinking too long`);
controller.abort();
break;
}
@@ -304,7 +354,12 @@ async function llmCall(messages, opts = {}) {
toolCalls[idx] = { id: tc.id, type: tc.type || 'function', function: { name: '', arguments: '' } };
}
if (tc.function?.name) toolCalls[idx].function.name += tc.function.name;
- if (tc.function?.arguments) toolCalls[idx].function.arguments += tc.function.arguments;
+ if (tc.function?.arguments) {
+ const chunk = typeof tc.function.arguments === 'string'
+ ? tc.function.arguments
+ : JSON.stringify(tc.function.arguments);
+ toolCalls[idx].function.arguments += chunk;
+ }
}
}
@@ -316,14 +371,65 @@ async function llmCall(messages, opts = {}) {
// If the model only produced reasoning_content (thinking) with no content,
// use the reasoning output as the response content for evaluation purposes.
+ // Try to extract JSON from reasoning if this was a JSON-expected call.
if (!content && reasoningContent) {
- content = reasoningContent;
+ // Try to find JSON embedded in the reasoning output
+ try {
+ const jsonMatch = reasoningContent.match(/[{\[][\s\S]*[}\]]/);
+ if (jsonMatch) {
+ content = jsonMatch[0];
+ } else {
+ content = reasoningContent;
+ }
+ } catch {
+ content = reasoningContent;
+ }
+ }
+
+ // Build per-call token data:
+ // Prefer server-reported usage; fall back to chunk-counted completion tokens
+ const promptTokens = usage.prompt_tokens || 0;
+ const completionTokens = usage.completion_tokens || tokenCount; // tokenCount = chunks with content/reasoning
+ const totalTokens = usage.total_tokens || (promptTokens + completionTokens);
+ const callTokens = { prompt: promptTokens, completion: completionTokens, total: totalTokens };
+
+ // ─── Performance metrics ───
+ const callEndTime = Date.now();
+ const totalElapsedMs = callEndTime - callStartTime;
+ const ttftMs = firstTokenTime ? (firstTokenTime - callStartTime) : null;
+ // Decode throughput: tokens generated / time spent generating (after first token)
+ const decodeMs = firstTokenTime ? (callEndTime - firstTokenTime) : 0;
+ const decodeTokensPerSec = (decodeMs > 0 && tokenCount > 1)
+ ? ((tokenCount - 1) / (decodeMs / 1000)) // -1 because first token is the TTFT boundary
+ : null;
+
+ const callPerf = {
+ ttftMs,
+ decodeTokensPerSec: decodeTokensPerSec ? parseFloat(decodeTokensPerSec.toFixed(1)) : null,
+ totalElapsedMs,
+ };
+
+ // Track global token totals
+ results.tokenTotals.prompt += callTokens.prompt;
+ results.tokenTotals.completion += callTokens.completion;
+ results.tokenTotals.total += callTokens.total;
+
+ // Track per-test tokens (accumulated across multiple llmCall invocations within one test)
+ if (_currentTestTokens) {
+ _currentTestTokens.prompt += callTokens.prompt;
+ _currentTestTokens.completion += callTokens.completion;
+ _currentTestTokens.total += callTokens.total;
+ }
+
+ // Track per-test perf (accumulated across multiple llmCall invocations within one test)
+ if (_currentTestPerf) {
+ if (ttftMs !== null) _currentTestPerf.ttftMs.push(ttftMs);
+ if (decodeTokensPerSec !== null) _currentTestPerf.decodeTokensPerSec.push(decodeTokensPerSec);
}
- // Track token totals
- results.tokenTotals.prompt += usage.prompt_tokens || 0;
- results.tokenTotals.completion += usage.completion_tokens || 0;
- results.tokenTotals.total += usage.total_tokens || 0;
+ // Track global perf totals
+ if (ttftMs !== null) results.perfTotals.ttftMs.push(ttftMs);
+ if (decodeTokensPerSec !== null) results.perfTotals.decodeTokensPerSec.push(decodeTokensPerSec);
// Capture model name from first response
if (opts.vlm) {
@@ -332,7 +438,7 @@ async function llmCall(messages, opts = {}) {
if (!results.model.name && model) results.model.name = model;
}
- return { content, toolCalls, usage, model };
+ return { content, toolCalls, usage: callTokens, perf: callPerf, model };
} finally {
clearTimeout(idleTimer);
}
@@ -340,7 +446,12 @@ async function llmCall(messages, opts = {}) {
}
function stripThink(text) {
- return text.replace(/
[\s\S]*?<\/think>\s*/gi, '').trim();
+ // Strip standard ... tags
+ let cleaned = text.replace(/[\s\S]*?<\/think>\s*/gi, '').trim();
+ // Strip Qwen3.5 'Thinking Process:' blocks (outputs plain text reasoning
+ // instead of tags when enable_thinking is active)
+ cleaned = cleaned.replace(/^Thinking Process[:\s]*[\s\S]*?(?=\n\s*[{\[]|\n```|$)/i, '').trim();
+ return cleaned;
}
function parseJSON(text) {
@@ -351,7 +462,7 @@ function parseJSON(text) {
jsonStr = codeBlock[1];
} else {
// Find first { or [ and extract balanced JSON
- const startIdx = cleaned.search(/[{[]/);
+ const startIdx = cleaned.search(/[{\[]/);
if (startIdx >= 0) {
const opener = cleaned[startIdx];
const closer = opener === '{' ? '}' : ']';
@@ -370,15 +481,198 @@ function parseJSON(text) {
}
}
}
- return JSON.parse(jsonStr.trim());
+ // Clean common local model artifacts before parsing:
+ // - Replace literal "..." or "…" placeholders in arrays/values
+ // - Replace tags (model echoes prompt templates)
+ jsonStr = jsonStr
+ .replace(/,\s*\.{3,}\s*(?=[\]},])/g, '') // trailing ..., before ] } or ,
+ .replace(/\.{3,}/g, '"..."') // standalone ... → string
+ .replace(/…/g, '"..."') // ellipsis char
+ .replace(/<[^>]+>/g, '"placeholder"') // → "placeholder" (multi-word)
+ .replace(/,\s*([}\]])/g, '$1'); // trailing commas
+ try {
+ return JSON.parse(jsonStr.trim());
+ } catch (firstErr) {
+ // Aggressive retry: strip all non-JSON artifacts
+ const aggressive = jsonStr
+ .replace(/"placeholder"(\s*"placeholder")*/g, '"placeholder"') // collapse repeated placeholders
+ .replace(/\bplaceholder\b/g, '""') // placeholder → empty string
+ .replace(/,\s*([}\]])/g, '$1'); // re-clean trailing commas
+ return JSON.parse(aggressive.trim());
+ }
}
function assert(condition, msg) {
if (!condition) throw new Error(msg || 'Assertion failed');
}
+// ─── Resource Metrics (GPU/MPS + Memory) ─────────────────────────────────────
+
+/**
+ * Sample GPU (Apple Silicon MPS) utilization and system memory.
+ * Uses `ioreg` for GPU stats (no sudo needed).
+ */
+function sampleResourceMetrics() {
+ const os = require('os');
+ const sample = {
+ timestamp: new Date().toISOString(),
+ sys: {
+ totalGB: parseFloat((os.totalmem() / 1073741824).toFixed(1)),
+ freeGB: parseFloat((os.freemem() / 1073741824).toFixed(1)),
+ usedGB: parseFloat(((os.totalmem() - os.freemem()) / 1073741824).toFixed(1)),
+ },
+ process: {
+ rssMB: parseFloat((process.memoryUsage().rss / 1048576).toFixed(0)),
+ },
+ gpu: null,
+ };
+
+ // Apple Silicon GPU via ioreg (macOS only)
+ if (process.platform === 'darwin') {
+ try {
+ const out = execSync('ioreg -r -c AGXAccelerator 2>/dev/null', { encoding: 'utf8', timeout: 3000 });
+ const m = (key) => { const r = new RegExp('"' + key + '"=(\\d+)'); const match = out.match(r); return match ? parseInt(match[1]) : null; };
+ const deviceUtil = m('Device Utilization %');
+ const rendererUtil = m('Renderer Utilization %');
+ const tilerUtil = m('Tiler Utilization %');
+ const memUsed = m('In use system memory');
+ const memAlloc = m('Alloc system memory');
+ if (deviceUtil !== null) {
+ sample.gpu = {
+ util: deviceUtil,
+ renderer: rendererUtil,
+ tiler: tilerUtil,
+ memUsedGB: memUsed ? parseFloat((memUsed / 1073741824).toFixed(1)) : null,
+ memAllocGB: memAlloc ? parseFloat((memAlloc / 1073741824).toFixed(1)) : null,
+ };
+ }
+ } catch { /* ioreg not available or timed out */ }
+ }
+
+ return sample;
+}
+
+// ─── Live progress: intermediate saves + report regeneration ────────────────
+let _liveReportOpened = false;
+let _runStartedAt = null; // Set when runSuites() begins
+let _currentTestName = null; // Set during test execution for live banner
+let _currentSuiteIndex = 0; // Current suite index for live progress
+let _totalSuites = 0; // Total number of suites
+
+/**
+ * Save the current (in-progress) results to disk and regenerate the live report.
+ * Called after each test completes so the browser auto-refreshes with updated data.
+ */
+function saveLiveProgress(startedAt, suitesCompleted, totalSuites, nextSuiteName, currentTest) {
+ try {
+ fs.mkdirSync(RESULTS_DIR, { recursive: true });
+
+ // Save current results as a live file (will be overwritten each time)
+ const liveFile = path.join(RESULTS_DIR, '_live_progress.json');
+ // Include the in-progress suite so Quality/Vision tabs can render partial data
+ const liveSuites = [...results.suites];
+ if (currentSuite && currentSuite.tests.length > 0 && !results.suites.includes(currentSuite)) {
+ liveSuites.push(currentSuite);
+ }
+ const liveResults = {
+ ...results,
+ suites: liveSuites,
+ _live: true,
+ _progress: { suitesCompleted, totalSuites, startedAt, currentTest: currentTest || null },
+ };
+ fs.writeFileSync(liveFile, JSON.stringify(liveResults, null, 2));
+
+ // Build a temporary index with just the live file
+ const indexFile = path.join(RESULTS_DIR, 'index.json');
+
+ // Compute live performance summary from accumulated data
+ const ttftArr = [...results.perfTotals.ttftMs];
+ const decArr = [...results.perfTotals.decodeTokensPerSec];
+ const livePerfSummary = (ttftArr.length > 0 || decArr.length > 0) ? {
+ ttft: ttftArr.length > 0 ? {
+ avgMs: Math.round(ttftArr.reduce((a, b) => a + b, 0) / ttftArr.length),
+ p50Ms: [...ttftArr].sort((a, b) => a - b)[Math.floor(ttftArr.length * 0.5)],
+ p95Ms: [...ttftArr].sort((a, b) => a - b)[Math.floor(ttftArr.length * 0.95)],
+ samples: ttftArr.length,
+ } : null,
+ decode: decArr.length > 0 ? {
+ avgTokensPerSec: parseFloat((decArr.reduce((a, b) => a + b, 0) / decArr.length).toFixed(1)),
+ samples: decArr.length,
+ } : null,
+ server: {
+ prefillTokensPerSec: results.perfTotals.prefillTokensPerSec,
+ decodeTokensPerSec: results.perfTotals.serverDecodeTokensPerSec,
+ },
+ resource: results.resourceSamples.length > 0 ? results.resourceSamples[results.resourceSamples.length - 1] : null,
+ } : null;
+
+ // Preserve previous runs in index for comparison sidebar
+ let existingIndex = [];
+ try { existingIndex = JSON.parse(fs.readFileSync(indexFile, 'utf8')).filter(e => e.file !== '_live_progress.json'); } catch { }
+ const liveEntry = {
+ file: '_live_progress.json',
+ model: results.model.name || 'loading...',
+ vlm: results.model.vlm || null,
+ timestamp: results.timestamp,
+ passed: results.totals.passed,
+ failed: results.totals.failed,
+ total: results.totals.total,
+ llmPassed: results.totals.passed, // Simplified for live view
+ llmTotal: results.totals.total,
+ vlmPassed: 0, vlmTotal: 0,
+ timeMs: Date.now() - new Date(startedAt).getTime(),
+ tokens: results.tokenTotals.total,
+ perfSummary: livePerfSummary,
+ };
+ fs.writeFileSync(indexFile, JSON.stringify([...existingIndex, liveEntry], null, 2));
+
+ // Regenerate report in live mode
+ const reportScript = path.join(__dirname, 'generate-report.cjs');
+ // Clear require cache to pick up any code changes
+ delete require.cache[require.resolve(reportScript)];
+ const { generateReport } = require(reportScript);
+ const testsCompleted = liveSuites.reduce((n, s) => n + s.tests.length, 0);
+ const testsTotal = liveSuites.reduce((n, s) => n + s.tests.length, 0) + (currentTest ? 0 : 0);
+ const reportPath = generateReport(RESULTS_DIR, {
+ liveMode: true,
+ liveStatus: {
+ suitesCompleted,
+ totalSuites,
+ currentSuite: currentSuite?.name || nextSuiteName || 'Finishing...',
+ currentTest: currentTest || null,
+ testsCompleted,
+ startedAt,
+ },
+ });
+
+ // Open browser on first save (so user sees live progress from the start)
+ if (!_liveReportOpened && !NO_OPEN && reportPath) {
+ if (IS_SKILL_MODE) {
+ // Ask Aegis to open in its embedded browser window
+ emit({ event: 'open_report', reportPath });
+ log(' 📊 Requested Aegis to open live report');
+ } else {
+ // Standalone: open in system browser
+ try {
+ const openCmd = process.platform === 'darwin' ? 'open' : 'xdg-open';
+ execSync(`${openCmd} "${reportPath}"`, { stdio: 'ignore' });
+ log(' 📊 Live report opened in browser (auto-refreshes every 5s)');
+ } catch { }
+ }
+ _liveReportOpened = true;
+ }
+ } catch (err) {
+ // Non-fatal — live progress is a nice-to-have
+ log(` ⚠️ Live progress update failed: ${err.message}`);
+ }
+}
+
async function runSuites() {
- for (const s of suites) {
+ _runStartedAt = new Date().toISOString();
+ _totalSuites = suites.length;
+ for (let si = 0; si < suites.length; si++) {
+ const s = suites[si];
+ _currentSuiteIndex = si;
currentSuite = { name: s.name, tests: [], passed: 0, failed: 0, skipped: 0, timeMs: 0 };
log(`\n${'─'.repeat(60)}`);
log(` ${s.name}`);
@@ -394,28 +688,68 @@ async function runSuites() {
results.totals.total += currentSuite.tests.length;
emit({ event: 'suite_end', suite: s.name, passed: currentSuite.passed, failed: currentSuite.failed, skipped: currentSuite.skipped, timeMs: currentSuite.timeMs });
+
+ // Sample resource metrics (GPU + memory) after each suite
+ const resourceSample = sampleResourceMetrics();
+ resourceSample.suite = s.name;
+ results.resourceSamples.push(resourceSample);
+
+ // Scrape server metrics after each suite so live perf cards update
+ await scrapeServerMetrics();
+
+ // Live progress: save after suite (also saved per-test, but suite boundary is a clean checkpoint)
+ saveLiveProgress(_runStartedAt, si + 1, suites.length, si + 1 < suites.length ? suites[si + 1]?.name : null);
}
}
+// ─── Per-test token + perf accumulators (set by test(), read by llmCall) ──────
+let _currentTestTokens = null;
+let _currentTestPerf = null;
+let _vlmTestMeta = null; // VLM fixture metadata (set during VLM tests, read after test() completes)
+
async function test(name, fn) {
- const testResult = { name, status: 'pass', timeMs: 0, detail: '', tokens: {} };
+ const testResult = { name, status: 'pass', timeMs: 0, detail: '', tokens: { prompt: 0, completion: 0, total: 0 }, perf: {} };
+ _currentTestTokens = { prompt: 0, completion: 0, total: 0 };
+ _currentTestPerf = { ttftMs: [], decodeTokensPerSec: [] };
const start = Date.now();
try {
const detail = await fn();
testResult.timeMs = Date.now() - start;
testResult.detail = detail || '';
+ testResult.tokens = { ..._currentTestTokens };
+ // Compute aggregate perf for this test (may span multiple llmCall invocations)
+ testResult.perf = {
+ ttftMs: _currentTestPerf.ttftMs.length > 0 ? Math.round(_currentTestPerf.ttftMs.reduce((a, b) => a + b, 0) / _currentTestPerf.ttftMs.length) : null,
+ decodeTokensPerSec: _currentTestPerf.decodeTokensPerSec.length > 0 ? parseFloat((_currentTestPerf.decodeTokensPerSec.reduce((a, b) => a + b, 0) / _currentTestPerf.decodeTokensPerSec.length).toFixed(1)) : null,
+ };
currentSuite.passed++;
- log(` ✅ ${name} (${testResult.timeMs}ms)${detail ? ` — ${detail}` : ''}`);
+ const tokInfo = _currentTestTokens.total > 0 ? `, ${_currentTestTokens.total} tok` : '';
+ const perfInfo = testResult.perf.ttftMs !== null ? `, TTFT ${testResult.perf.ttftMs}ms` : '';
+ const tpsInfo = testResult.perf.decodeTokensPerSec !== null ? `, ${testResult.perf.decodeTokensPerSec} tok/s` : '';
+ log(` ✅ ${name} (${testResult.timeMs}ms${tokInfo}${perfInfo}${tpsInfo})${detail ? ` — ${detail}` : ''}`);
} catch (err) {
testResult.timeMs = Date.now() - start;
testResult.status = 'fail';
testResult.detail = err.message;
+ testResult.tokens = { ..._currentTestTokens };
+ testResult.perf = {
+ ttftMs: _currentTestPerf.ttftMs.length > 0 ? Math.round(_currentTestPerf.ttftMs.reduce((a, b) => a + b, 0) / _currentTestPerf.ttftMs.length) : null,
+ decodeTokensPerSec: _currentTestPerf.decodeTokensPerSec.length > 0 ? parseFloat((_currentTestPerf.decodeTokensPerSec.reduce((a, b) => a + b, 0) / _currentTestPerf.decodeTokensPerSec.length).toFixed(1)) : null,
+ };
currentSuite.failed++;
log(` ❌ ${name} (${testResult.timeMs}ms) — ${err.message}`);
}
+ _currentTestTokens = null;
+ _currentTestPerf = null;
currentSuite.timeMs += testResult.timeMs;
currentSuite.tests.push(testResult);
- emit({ event: 'test_result', suite: currentSuite.name, test: name, status: testResult.status, timeMs: testResult.timeMs, detail: testResult.detail.slice(0, 120) });
+ emit({ event: 'test_result', suite: currentSuite.name, test: name, status: testResult.status, timeMs: testResult.timeMs, detail: testResult.detail.slice(0, 120), tokens: testResult.tokens, perf: testResult.perf });
+
+ // Live progress: save after each test for real-time updates in commander center
+ if (_runStartedAt) {
+ _currentTestName = null; // Test just completed
+ saveLiveProgress(_runStartedAt, _currentSuiteIndex, _totalSuites, null, name);
+ }
}
function skip(name, reason) {
@@ -444,11 +778,7 @@ ${userMessage}
3. Always keep the last 2 user messages (most recent context)
4. Keep system messages (they contain tool results)
-## Response Format
-Respond with ONLY a valid JSON object, no other text:
-{"keep": [], "summary": ""}
-
-Example: if keeping messages at indices 0, 18, 22 → {"keep": [0, 18, 22], "summary": "Removed 4 duplicate 'what happened today' questions"}
+Respond with ONLY valid JSON: {"keep": [0, 18, 22], "summary": "Removed 4 duplicate questions"}
If nothing should be dropped, keep ALL indices and set summary to "".`;
}
@@ -1879,18 +2209,37 @@ suite('📸 VLM Scene Analysis', async () => {
const framePath = path.join(FIXTURES_DIR, 'frames', t.file);
if (!fs.existsSync(framePath)) { skip(t.name, `File missing: ${t.file}`); return; }
const desc = await vlmAnalyze(framePath, t.prompt);
- if (t.expect === null) {
- // Just check we got a meaningful response
- assert(desc.length > 20, `Response too short: ${desc.length} chars`);
- return `${desc.length} chars ✓`;
- }
- const lower = desc.toLowerCase();
- const matched = t.expect.some(term => lower.includes(term));
- assert(matched,
- `Expected one of [${t.expect.slice(0, 4).join(', ')}...] in: "${desc.slice(0, 80)}"`);
- const hits = t.expect.filter(term => lower.includes(term));
- return `${desc.length} chars, matched: ${hits.join(', ')} ✓`;
+
+ // Save fixture filename + VLM response for Vision tab in report
+ const lastTest = currentSuite.tests.length > 0 ? null : undefined; // will be set after push
+ // Attach after test() pushes — use a post-hook via the return
+ const result = (() => {
+ if (t.expect === null) {
+ assert(desc.length > 20, `Response too short: ${desc.length} chars`);
+ return `${desc.length} chars ✓`;
+ }
+ const lower = desc.toLowerCase();
+ const matched = t.expect.some(term => lower.includes(term));
+ assert(matched,
+ `Expected one of [${t.expect.slice(0, 4).join(', ')}...] in: "${desc.slice(0, 80)}"`);
+ const hits = t.expect.filter(term => lower.includes(term));
+ return `${desc.length} chars, matched: ${hits.join(', ')} ✓`;
+ })();
+
+ // Stash fixture + response on the test result (test() pushes to currentSuite.tests)
+ // We set it as a closure-accessible value; the test() function reads the return value.
+ // After test() completes, we patch the last test entry with VLM metadata.
+ _vlmTestMeta = { fixture: t.file, vlmResponse: desc.slice(0, 300), prompt: t.prompt };
+ return result;
});
+ // Patch the last pushed test with VLM metadata (fixture filename + response preview)
+ if (_vlmTestMeta && currentSuite.tests.length > 0) {
+ const lastTest = currentSuite.tests[currentSuite.tests.length - 1];
+ lastTest.fixture = _vlmTestMeta.fixture;
+ lastTest.vlmResponse = _vlmTestMeta.vlmResponse;
+ lastTest.vlmPrompt = _vlmTestMeta.prompt;
+ _vlmTestMeta = null;
+ }
}
});
@@ -1916,6 +2265,52 @@ function collectSystemInfo() {
};
}
+// ═══════════════════════════════════════════════════════════════════════════════
+// SERVER METRICS SCRAPER (llama-server Prometheus /metrics endpoint)
+// ═══════════════════════════════════════════════════════════════════════════════
+
+/**
+ * Scrape llama-server /metrics endpoint for server-side performance stats.
+ * Requires llama-server to be launched with --metrics flag.
+ * Extracts: prompt_tokens_seconds (prefill tok/s), predicted_tokens_seconds (decode tok/s)
+ */
+async function scrapeServerMetrics() {
+ // Try LLM server first, then VLM server
+ const ports = [
+ { name: 'LLM', url: LLM_URL || GATEWAY_URL },
+ ...(VLM_URL ? [{ name: 'VLM', url: VLM_URL }] : []),
+ ];
+
+ for (const { name, url } of ports) {
+ try {
+ const base = url.replace(/\/v1\/?$/, '');
+ const controller = new AbortController();
+ const timeout = setTimeout(() => controller.abort(), 3000);
+ const res = await fetch(`${base}/metrics`, { signal: controller.signal });
+ clearTimeout(timeout);
+
+ if (!res.ok) continue;
+ const text = await res.text();
+
+ // Parse Prometheus text format for our metrics
+ const prefillMatch = text.match(/llamacpp:prompt_tokens_seconds\s+([\d.]+)/);
+ const decodeMatch = text.match(/llamacpp:predicted_tokens_seconds\s+([\d.]+)/);
+
+ if (prefillMatch || decodeMatch) {
+ const prefill = prefillMatch ? parseFloat(parseFloat(prefillMatch[1]).toFixed(1)) : null;
+ const decode = decodeMatch ? parseFloat(parseFloat(decodeMatch[1]).toFixed(1)) : null;
+ results.perfTotals.prefillTokensPerSec = prefill;
+ results.perfTotals.serverDecodeTokensPerSec = decode;
+ log(` 📊 ${name} server metrics: prefill ${prefill || '?'} tok/s, decode ${decode || '?'} tok/s`);
+ return; // Got metrics from at least one server
+ }
+ } catch (_) {
+ // /metrics not available — server not started with --metrics flag
+ }
+ }
+ log(' ℹ️ Server /metrics not available (start with --metrics for server-side stats)');
+}
+
// ═══════════════════════════════════════════════════════════════════════════════
// MAIN RUNNER
// ═══════════════════════════════════════════════════════════════════════════════
@@ -1942,7 +2337,6 @@ async function main() {
const ping = await llmClient.chat.completions.create({
...(LLM_MODEL && { model: LLM_MODEL }),
messages: [{ role: 'user', content: 'ping' }],
- max_completion_tokens: 5,
});
results.model.name = ping.model || 'unknown';
log(` Model: ${results.model.name}`);
@@ -1951,7 +2345,7 @@ async function main() {
log(` Base URL: ${llmBaseUrl}`);
log(' Check that the LLM server is running.\n');
emit({ event: 'error', message: `Cannot reach LLM endpoint: ${err.message}` });
- process.exit(1);
+ process.exit(IS_SKILL_MODE ? 0 : 1);
}
// Collect system info
@@ -1991,14 +2385,44 @@ async function main() {
heapUsed: (postMem.heapUsed / 1048576).toFixed(1),
};
+ // Scrape llama-server /metrics for server-side prefill/decode stats
+ await scrapeServerMetrics();
+
// Summary
const { passed, failed, skipped, total, timeMs } = results.totals;
const tokPerSec = timeMs > 0 ? ((results.tokenTotals.total / (timeMs / 1000)).toFixed(1)) : '?';
+ // Compute aggregate perf stats
+ const ttftArr = results.perfTotals.ttftMs;
+ const avgTtft = ttftArr.length > 0 ? Math.round(ttftArr.reduce((a, b) => a + b, 0) / ttftArr.length) : null;
+ const p50Ttft = ttftArr.length > 0 ? ttftArr.sort((a, b) => a - b)[Math.floor(ttftArr.length * 0.5)] : null;
+ const p95Ttft = ttftArr.length > 0 ? ttftArr.sort((a, b) => a - b)[Math.floor(ttftArr.length * 0.95)] : null;
+ const decArr = results.perfTotals.decodeTokensPerSec;
+ const avgDecode = decArr.length > 0 ? parseFloat((decArr.reduce((a, b) => a + b, 0) / decArr.length).toFixed(1)) : null;
+
+ // Store computed aggregates
+ results.perfSummary = {
+ ttft: { avgMs: avgTtft, p50Ms: p50Ttft, p95Ms: p95Ttft, samples: ttftArr.length },
+ decode: { avgTokensPerSec: avgDecode, samples: decArr.length },
+ server: {
+ prefillTokensPerSec: results.perfTotals.prefillTokensPerSec,
+ decodeTokensPerSec: results.perfTotals.serverDecodeTokensPerSec,
+ },
+ };
+
log(`\n${'═'.repeat(66)}`);
log(` RESULTS: ${passed}/${total} passed, ${failed} failed, ${skipped} skipped (${(timeMs / 1000).toFixed(1)}s)`);
log(` TOKENS: ${results.tokenTotals.prompt} prompt + ${results.tokenTotals.completion} completion = ${results.tokenTotals.total} total (${tokPerSec} tok/s)`);
log(` MODEL: ${results.model.name}${results.model.vlm ? ' | VLM: ' + results.model.vlm : ''}`);
+ if (avgTtft !== null) {
+ log(` TTFT: avg ${avgTtft}ms | p50 ${p50Ttft}ms | p95 ${p95Ttft}ms (${ttftArr.length} samples)`);
+ }
+ if (avgDecode !== null) {
+ log(` DECODE: ${avgDecode} tok/s avg (${decArr.length} samples)`);
+ }
+ if (results.perfTotals.prefillTokensPerSec !== null) {
+ log(` SERVER: prefill ${results.perfTotals.prefillTokensPerSec} tok/s | decode ${results.perfTotals.serverDecodeTokensPerSec} tok/s (from /metrics)`);
+ }
log(`${'═'.repeat(66)}`);
if (failed > 0) {
@@ -2012,20 +2436,23 @@ async function main() {
// Save results
fs.mkdirSync(RESULTS_DIR, { recursive: true });
+ // Clean up live progress file (replaced by final results)
+ try { fs.unlinkSync(path.join(RESULTS_DIR, '_live_progress.json')); } catch { }
const modelSlug = (results.model.name || 'unknown').replace(/[^a-zA-Z0-9_.-]/g, '_');
const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
const resultFile = path.join(RESULTS_DIR, `${modelSlug}_${ts}.json`);
fs.writeFileSync(resultFile, JSON.stringify(results, null, 2));
log(`\n Results saved: ${resultFile}`);
- // Update index
+ // Update index (filter out any live progress entries)
const indexFile = path.join(RESULTS_DIR, 'index.json');
let index = [];
- try { index = JSON.parse(fs.readFileSync(indexFile, 'utf8')); } catch { }
- // Compute LLM vs VLM split
- const vlmSuite = results.suites.find(s => s.name.includes('VLM'));
- const vlmPassed = vlmSuite ? vlmSuite.tests.filter(t => t.status === 'pass').length : 0;
- const vlmTotal = vlmSuite ? vlmSuite.tests.length : 0;
+ try { index = JSON.parse(fs.readFileSync(indexFile, 'utf8')).filter(e => e.file !== '_live_progress.json'); } catch { }
+ // Compute LLM vs VLM split (only count image analysis suites as VLM)
+ const isVlmImageSuite = (name) => name.includes('VLM Scene') || name.includes('📸');
+ const vlmSuites = results.suites.filter(s => isVlmImageSuite(s.name));
+ const vlmPassed = vlmSuites.reduce((n, s) => n + s.tests.filter(t => t.status === 'pass').length, 0);
+ const vlmTotal = vlmSuites.reduce((n, s) => n + s.tests.length, 0);
const llmPassed = passed - vlmPassed;
const llmTotal = total - vlmTotal;
@@ -2039,19 +2466,26 @@ async function main() {
vlmPassed, vlmTotal,
timeMs,
tokens: results.tokenTotals.total,
+ perfSummary: {
+ ...(results.perfSummary || {}),
+ resource: results.resourceSamples?.length > 0 ? results.resourceSamples[results.resourceSamples.length - 1] : null,
+ },
});
fs.writeFileSync(indexFile, JSON.stringify(index, null, 2));
- // Always generate report (skip only on explicit --no-open with no --report flag)
+ // Always generate final report (without live mode)
let reportPath = null;
log('\n Generating HTML report...');
try {
const reportScript = path.join(__dirname, 'generate-report.cjs');
+ // Clear require cache to get latest version
+ delete require.cache[require.resolve(reportScript)];
reportPath = require(reportScript).generateReport(RESULTS_DIR);
log(` ✅ Report: ${reportPath}`);
// Auto-open in browser — only in standalone mode (Aegis handles its own opening)
- if (!NO_OPEN && !IS_SKILL_MODE && reportPath) {
+ // Skip if live mode already opened the browser earlier
+ if (!_liveReportOpened && !NO_OPEN && !IS_SKILL_MODE && reportPath) {
try {
const openCmd = process.platform === 'darwin' ? 'open' : 'xdg-open';
execSync(`${openCmd} "${reportPath}"`, { stdio: 'ignore' });
@@ -2077,7 +2511,10 @@ async function main() {
});
log('');
- process.exit(failed > 0 ? 1 : 0);
+ // When running as Aegis skill, always exit 0 — test results are reported
+ // via JSON events (pass/fail is a result, not an error). Exit 1 only for
+ // standalone CLI usage where CI/CD pipelines expect non-zero on failures.
+ process.exit(IS_SKILL_MODE ? 0 : (failed > 0 ? 1 : 0));
}
// Run when executed directly — supports both plain Node and Electron spawn.
@@ -2090,7 +2527,7 @@ if (isDirectRun) {
main().catch(err => {
log(`Fatal: ${err.message}`);
emit({ event: 'error', message: err.message });
- process.exit(1);
+ process.exit(IS_SKILL_MODE ? 0 : 1);
});
}
diff --git a/skills/annotation/dataset-management/SKILL.md b/skills/annotation/dataset-management/SKILL.md
new file mode 100644
index 00000000..02e6455c
--- /dev/null
+++ b/skills/annotation/dataset-management/SKILL.md
@@ -0,0 +1,51 @@
+---
+name: annotation-data
+description: "Dataset annotation management — COCO labels, sequences, export, and Kaggle upload"
+version: 1.0.0
+entry: scripts/annotation_manager.py
+deploy: deploy.sh
+
+parameters:
+ - name: datasets_dir
+ label: "Datasets Directory"
+ type: string
+ default: ""
+ description: "Root directory for annotation datasets (auto-detected if empty)"
+ group: Storage
+
+capabilities:
+ live_transform:
+ script: scripts/annotation_manager.py
+ description: "Dataset CRUD, annotation save/load, COCO export"
+
+ui_unlocks:
+ - annotation_studio
+---
+
+# Annotation Data Management
+
+Manages annotation datasets for Aegis Annotation Studio. Handles dataset CRUD, label management, COCO-format export, and Kaggle upload.
+
+## Protocol (stdin/stdout JSONL)
+
+### Aegis → Skill
+```jsonl
+{"command": "list_datasets", "request_id": "req_001"}
+{"command": "get_dataset", "name": "my_dataset", "request_id": "req_002"}
+{"command": "save_dataset", "name": "my_dataset", "labels": [...], "request_id": "req_003"}
+{"command": "delete_dataset", "name": "my_dataset", "request_id": "req_004"}
+{"command": "save_annotation", "dataset": "my_dataset", "frame_id": "f1", "annotations": [...], "request_id": "req_005"}
+{"command": "list_labels", "dataset": "my_dataset", "request_id": "req_006"}
+{"command": "export_coco", "dataset": "my_dataset", "request_id": "req_007"}
+{"command": "get_stats", "dataset": "my_dataset", "request_id": "req_008"}
+{"command": "stop"}
+```
+
+### Skill → Aegis
+```jsonl
+{"event": "annotation", "type": "ready", "request_id": "", "data": {"version": "1.0.0"}}
+{"event": "annotation", "type": "datasets", "request_id": "req_001", "data": [...]}
+{"event": "annotation", "type": "dataset", "request_id": "req_002", "data": {...}}
+{"event": "annotation", "type": "saved", "request_id": "req_005", "data": {"frame_id": "f1", "count": 3}}
+{"event": "annotation", "type": "exported", "request_id": "req_007", "data": {"path": "/path/to/coco.json"}}
+```
diff --git a/skills/annotation/dataset-management/deploy.bat b/skills/annotation/dataset-management/deploy.bat
new file mode 100644
index 00000000..16c81462
--- /dev/null
+++ b/skills/annotation/dataset-management/deploy.bat
@@ -0,0 +1,52 @@
+@echo off
+REM deploy.bat — Bootstrapper for Annotation Data Management Skill (Windows)
+REM Lightweight — no GPU needed, stdlib-only Python.
+
+setlocal enabledelayedexpansion
+
+set "SKILL_DIR=%~dp0"
+if "%SKILL_DIR:~-1%"=="\" set "SKILL_DIR=%SKILL_DIR:~0,-1%"
+set "VENV_DIR=%SKILL_DIR%\.venv"
+set "LOG_PREFIX=[annotation-data-deploy]"
+
+REM ─── Find Python ───────────────────────────────────────────────────────
+set "PYTHON_CMD="
+for %%V in (3.12 3.11 3.10 3.9) do (
+ if not defined PYTHON_CMD (
+ py -%%V --version >nul 2>&1
+ if !errorlevel! equ 0 set "PYTHON_CMD=py -%%V"
+ )
+)
+if not defined PYTHON_CMD (
+ python3 --version >nul 2>&1
+ if !errorlevel! equ 0 set "PYTHON_CMD=python3"
+)
+if not defined PYTHON_CMD (
+ python --version >nul 2>&1
+ if !errorlevel! equ 0 set "PYTHON_CMD=python"
+)
+if not defined PYTHON_CMD (
+ echo %LOG_PREFIX% ERROR: No Python found>&2
+ echo {"event": "error", "stage": "python", "message": "No Python found"}
+ exit /b 1
+)
+
+for /f "tokens=*" %%A in ('!PYTHON_CMD! --version 2^>^&1') do set "PY_VERSION=%%A"
+echo %LOG_PREFIX% Using Python: %PYTHON_CMD% (%PY_VERSION%)>&2
+echo {"event": "progress", "stage": "python", "message": "Found %PY_VERSION%"}
+
+REM ─── Create venv ───────────────────────────────────────────────────────
+if not exist "%VENV_DIR%\Scripts\python.exe" (
+ %PYTHON_CMD% -m venv "%VENV_DIR%"
+)
+
+echo {"event": "progress", "stage": "venv", "message": "Virtual environment ready"}
+
+REM ─── Verify ────────────────────────────────────────────────────────────
+"%VENV_DIR%\Scripts\python.exe" -c "import json, pathlib; print('Annotation data skill ready')" 2>&1
+
+echo {"event": "complete", "backend": "cpu", "message": "Annotation data skill installed"}
+echo %LOG_PREFIX% Done!>&2
+
+endlocal
+exit /b 0
diff --git a/skills/annotation/dataset-management/deploy.sh b/skills/annotation/dataset-management/deploy.sh
new file mode 100755
index 00000000..c18bc3c4
--- /dev/null
+++ b/skills/annotation/dataset-management/deploy.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# deploy.sh — Bootstrapper for Annotation Data Management Skill
+# Lightweight — no GPU needed, stdlib-only Python.
+
+set -euo pipefail
+
+SKILL_DIR="$(cd "$(dirname "$0")" && pwd)"
+VENV_DIR="$SKILL_DIR/.venv"
+LOG_PREFIX="[annotation-data-deploy]"
+
+log() { echo "$LOG_PREFIX $*" >&2; }
+emit() { echo "$1"; }
+
+# ─── Find Python ──────────────────────────────────────────────────────────
+find_python() {
+ for cmd in python3.12 python3.11 python3.10 python3.9 python3; do
+ if command -v "$cmd" &>/dev/null; then
+ local ver
+ ver="$("$cmd" --version 2>&1 | grep -oE '[0-9]+\.[0-9]+')"
+ local major minor
+ major=$(echo "$ver" | cut -d. -f1)
+ minor=$(echo "$ver" | cut -d. -f2)
+ if [ "$major" -ge 3 ] && [ "$minor" -ge 9 ]; then
+ echo "$cmd"
+ return 0
+ fi
+ fi
+ done
+ return 1
+}
+
+PYTHON_CMD=$(find_python) || {
+ log "ERROR: No Python >=3.9 found."
+ emit '{"event": "error", "stage": "python", "message": "No Python >=3.9 found"}'
+ exit 1
+}
+
+log "Using Python: $PYTHON_CMD ($($PYTHON_CMD --version 2>&1))"
+emit "{\"event\": \"progress\", \"stage\": \"python\", \"message\": \"Found $($PYTHON_CMD --version 2>&1)\"}"
+
+# ─── Create venv ──────────────────────────────────────────────────────────
+if [ ! -d "$VENV_DIR" ]; then
+ "$PYTHON_CMD" -m venv "$VENV_DIR"
+fi
+
+emit '{"event": "progress", "stage": "venv", "message": "Virtual environment ready"}'
+
+# ─── Verify ───────────────────────────────────────────────────────────────
+"$VENV_DIR/bin/python" -c "import json, pathlib; print('Annotation data skill ready')" 2>&1 | while read -r line; do log "$line"; done
+
+emit '{"event": "complete", "backend": "cpu", "message": "Annotation data skill installed"}'
+log "Done!"
diff --git a/skills/annotation/dataset-management/requirements.txt b/skills/annotation/dataset-management/requirements.txt
new file mode 100644
index 00000000..941cfc21
--- /dev/null
+++ b/skills/annotation/dataset-management/requirements.txt
@@ -0,0 +1,2 @@
+# Annotation Data Management — minimal deps (stdlib only)
+# No external packages needed — all Python stdlib
diff --git a/skills/annotation/dataset-management/scripts/annotation_manager.py b/skills/annotation/dataset-management/scripts/annotation_manager.py
new file mode 100644
index 00000000..9ffed8af
--- /dev/null
+++ b/skills/annotation/dataset-management/scripts/annotation_manager.py
@@ -0,0 +1,350 @@
+#!/usr/bin/env python3
+"""
+Annotation Data Management Skill — Dataset CRUD via JSONL protocol.
+
+Manages annotation datasets, labels, sequences, COCO export.
+Replaces the REST-based annotation_dataset_api.py.
+
+Protocol (JSONL over stdin/stdout):
+ stdin: {"command": "list_datasets|get_dataset|save_annotation|...", ...}
+ stdout: {"event": "annotation", "type": "...", "request_id": "...", "data": ...}
+"""
+
+import sys
+import json
+import os
+import time
+import shutil
+import argparse
+import signal
+from pathlib import Path
+from datetime import datetime
+
+
+# ───────────────────────────────────────────────────────────────────────────────
+# Stdout protocol
+# ───────────────────────────────────────────────────────────────────────────────
+
+def emit(obj):
+ """Write a JSON object to stdout for Aegis to parse."""
+ sys.stdout.write(json.dumps(obj, default=str) + "\n")
+ sys.stdout.flush()
+
+def log(msg):
+ """Write a log message to stderr."""
+ sys.stderr.write(f"[annotation-data] {msg}\n")
+ sys.stderr.flush()
+
+def emit_result(type_: str, request_id: str, data=None, error=None):
+ """Emit an annotation event."""
+ event = {
+ "event": "annotation",
+ "type": type_,
+ "request_id": request_id,
+ }
+ if data is not None:
+ event["data"] = data
+ if error is not None:
+ event["error"] = error
+ emit(event)
+
+
+# ───────────────────────────────────────────────────────────────────────────────
+# Dataset manager
+# ───────────────────────────────────────────────────────────────────────────────
+
+class DatasetManager:
+ """Manages JSONL-based annotation datasets on disk."""
+
+ def __init__(self, root_dir: Path):
+ self.root = root_dir
+ self.root.mkdir(parents=True, exist_ok=True)
+ log(f"Dataset root: {self.root}")
+
+ def list_datasets(self) -> list:
+ """Return list of dataset metadata."""
+ datasets = []
+ for d in sorted(self.root.iterdir()):
+ if d.is_dir() and (d / "meta.json").exists():
+ try:
+ meta = json.loads((d / "meta.json").read_text())
+ meta["name"] = d.name
+ # Count annotations
+ annot_file = d / "annotations.jsonl"
+ meta["annotation_count"] = sum(1 for _ in open(annot_file)) if annot_file.exists() else 0
+ datasets.append(meta)
+ except Exception as e:
+ log(f"Skipping {d.name}: {e}")
+ return datasets
+
+ def get_dataset(self, name: str) -> dict:
+ """Get full dataset details + annotations."""
+ ds_dir = self.root / name
+ if not ds_dir.exists():
+ raise FileNotFoundError(f"Dataset '{name}' not found")
+ meta = json.loads((ds_dir / "meta.json").read_text())
+ meta["name"] = name
+ # Load annotations
+ annot_file = ds_dir / "annotations.jsonl"
+ annotations = []
+ if annot_file.exists():
+ with open(annot_file) as f:
+ for line in f:
+ line = line.strip()
+ if line:
+ annotations.append(json.loads(line))
+ meta["annotations"] = annotations
+ return meta
+
+ def save_dataset(self, name: str, labels: list = None, description: str = "") -> dict:
+ """Create or update dataset metadata."""
+ ds_dir = self.root / name
+ ds_dir.mkdir(parents=True, exist_ok=True)
+ meta_file = ds_dir / "meta.json"
+ if meta_file.exists():
+ meta = json.loads(meta_file.read_text())
+ else:
+ meta = {
+ "created": datetime.now().isoformat(),
+ "format": "jsonl",
+ }
+ meta["updated"] = datetime.now().isoformat()
+ if labels is not None:
+ meta["labels"] = labels
+ if description:
+ meta["description"] = description
+ meta_file.write_text(json.dumps(meta, indent=2, default=str))
+ return {"name": name, "updated": meta["updated"]}
+
+ def delete_dataset(self, name: str) -> dict:
+ """Delete a dataset directory."""
+ ds_dir = self.root / name
+ if ds_dir.exists():
+ shutil.rmtree(ds_dir)
+ return {"name": name, "deleted": True}
+ raise FileNotFoundError(f"Dataset '{name}' not found")
+
+ def save_annotation(self, dataset: str, frame_id: str, annotations: list) -> dict:
+ """Append annotations for a frame (JSONL append)."""
+ ds_dir = self.root / dataset
+ if not ds_dir.exists():
+ raise FileNotFoundError(f"Dataset '{dataset}' not found")
+ annot_file = ds_dir / "annotations.jsonl"
+ record = {
+ "frame_id": frame_id,
+ "timestamp": datetime.now().isoformat(),
+ "annotations": annotations,
+ }
+ with open(annot_file, "a") as f:
+ f.write(json.dumps(record, default=str) + "\n")
+ return {"frame_id": frame_id, "count": len(annotations)}
+
+ def list_labels(self, dataset: str) -> list:
+ """Get labels for a dataset."""
+ ds_dir = self.root / dataset
+ if not ds_dir.exists():
+ raise FileNotFoundError(f"Dataset '{dataset}' not found")
+ meta = json.loads((ds_dir / "meta.json").read_text())
+ return meta.get("labels", [])
+
+ def get_stats(self, dataset: str) -> dict:
+ """Get annotation statistics for a dataset."""
+ ds_dir = self.root / dataset
+ if not ds_dir.exists():
+ raise FileNotFoundError(f"Dataset '{dataset}' not found")
+ annot_file = ds_dir / "annotations.jsonl"
+ total_frames = 0
+ total_annotations = 0
+ label_counts = {}
+ if annot_file.exists():
+ with open(annot_file) as f:
+ for line in f:
+ line = line.strip()
+ if not line:
+ continue
+ record = json.loads(line)
+ total_frames += 1
+ for ann in record.get("annotations", []):
+ total_annotations += 1
+ label = ann.get("label", "unknown")
+ label_counts[label] = label_counts.get(label, 0) + 1
+ return {
+ "total_frames": total_frames,
+ "total_annotations": total_annotations,
+ "label_counts": label_counts,
+ }
+
+ def export_coco(self, dataset: str) -> dict:
+ """Export dataset to COCO JSON format."""
+ ds_dir = self.root / dataset
+ if not ds_dir.exists():
+ raise FileNotFoundError(f"Dataset '{dataset}' not found")
+ meta = json.loads((ds_dir / "meta.json").read_text())
+ labels = meta.get("labels", [])
+ # Build COCO structure
+ coco = {
+ "info": {
+ "description": meta.get("description", dataset),
+ "version": "1.0",
+ "year": datetime.now().year,
+ "date_created": datetime.now().isoformat(),
+ },
+ "categories": [
+ {"id": i + 1, "name": label, "supercategory": ""}
+ for i, label in enumerate(labels)
+ ],
+ "images": [],
+ "annotations": [],
+ }
+ label_to_id = {label: i + 1 for i, label in enumerate(labels)}
+ image_id = 0
+ ann_id = 0
+ annot_file = ds_dir / "annotations.jsonl"
+ if annot_file.exists():
+ with open(annot_file) as f:
+ for line in f:
+ line = line.strip()
+ if not line:
+ continue
+ record = json.loads(line)
+ image_id += 1
+ coco["images"].append({
+ "id": image_id,
+ "file_name": record.get("frame_id", f"frame_{image_id}"),
+ "width": record.get("width", 0),
+ "height": record.get("height", 0),
+ })
+ for ann in record.get("annotations", []):
+ ann_id += 1
+ bbox = ann.get("bbox", [0, 0, 0, 0])
+ coco["annotations"].append({
+ "id": ann_id,
+ "image_id": image_id,
+ "category_id": label_to_id.get(ann.get("label", ""), 0),
+ "bbox": bbox,
+ "area": bbox[2] * bbox[3] if len(bbox) == 4 else 0,
+ "segmentation": ann.get("segmentation", []),
+ "iscrowd": 0,
+ })
+ export_path = str(ds_dir / "coco_export.json")
+ with open(export_path, "w") as f:
+ json.dump(coco, f, indent=2, default=str)
+ return {
+ "path": export_path,
+ "images": len(coco["images"]),
+ "annotations": len(coco["annotations"]),
+ "categories": len(coco["categories"]),
+ }
+
+
+# ───────────────────────────────────────────────────────────────────────────────
+# Main loop
+# ───────────────────────────────────────────────────────────────────────────────
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Annotation Data Management")
+ parser.add_argument("--config", type=str)
+ parser.add_argument("--datasets-dir", type=str, default="")
+ return parser.parse_args()
+
+
+def main():
+ args = parse_args()
+
+ # Determine datasets directory
+ datasets_dir = args.datasets_dir
+ if not datasets_dir:
+ env_params = os.environ.get("AEGIS_SKILL_PARAMS")
+ if env_params:
+ try:
+ params = json.loads(env_params)
+ datasets_dir = params.get("datasets_dir", "")
+ except json.JSONDecodeError:
+ pass
+ if not datasets_dir:
+ # Default: ~/.aegis/datasets
+ datasets_dir = str(Path.home() / ".aegis" / "datasets")
+
+ manager = DatasetManager(Path(datasets_dir))
+
+ # Handle graceful shutdown
+ signal.signal(signal.SIGINT, lambda *_: sys.exit(0))
+ signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
+
+ # Emit ready
+ emit_result("ready", "", data={
+ "version": "1.0.0",
+ "datasets_dir": datasets_dir,
+ })
+ log("Ready")
+
+ # Main JSONL command loop
+ for raw_line in sys.stdin:
+ line = raw_line.strip()
+ if not line:
+ continue
+ try:
+ msg = json.loads(line)
+ except json.JSONDecodeError:
+ log(f"Invalid JSON: {line[:100]}")
+ continue
+
+ cmd = msg.get("command", "")
+ req_id = msg.get("request_id", "")
+
+ if cmd == "stop":
+ break
+
+ try:
+ if cmd == "list_datasets":
+ data = manager.list_datasets()
+ emit_result("datasets", req_id, data=data)
+
+ elif cmd == "get_dataset":
+ data = manager.get_dataset(msg["name"])
+ emit_result("dataset", req_id, data=data)
+
+ elif cmd == "save_dataset":
+ data = manager.save_dataset(
+ msg["name"],
+ labels=msg.get("labels"),
+ description=msg.get("description", ""),
+ )
+ emit_result("dataset_saved", req_id, data=data)
+
+ elif cmd == "delete_dataset":
+ data = manager.delete_dataset(msg["name"])
+ emit_result("dataset_deleted", req_id, data=data)
+
+ elif cmd == "save_annotation":
+ data = manager.save_annotation(
+ msg["dataset"],
+ msg["frame_id"],
+ msg.get("annotations", []),
+ )
+ emit_result("annotation_saved", req_id, data=data)
+
+ elif cmd == "list_labels":
+ data = manager.list_labels(msg["dataset"])
+ emit_result("labels", req_id, data=data)
+
+ elif cmd == "get_stats":
+ data = manager.get_stats(msg["dataset"])
+ emit_result("stats", req_id, data=data)
+
+ elif cmd == "export_coco":
+ data = manager.export_coco(msg["dataset"])
+ emit_result("exported", req_id, data=data)
+
+ else:
+ emit_result("error", req_id, error=f"Unknown command: {cmd}")
+
+ except FileNotFoundError as e:
+ emit_result("error", req_id, error=str(e))
+ except Exception as e:
+ log(f"Error handling {cmd}: {e}")
+ emit_result("error", req_id, error=str(e))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/skills/annotation/sam2-segmentation/SKILL.md b/skills/annotation/sam2-segmentation/SKILL.md
deleted file mode 100644
index dbdb6e0d..00000000
--- a/skills/annotation/sam2-segmentation/SKILL.md
+++ /dev/null
@@ -1,60 +0,0 @@
----
-name: sam2-segmentation
-description: "Interactive click-to-segment using Segment Anything 2"
-version: 1.0.0
-
-parameters:
- - name: model
- label: "SAM2 Model"
- type: select
- options: ["sam2-tiny", "sam2-small", "sam2-base", "sam2-large"]
- default: "sam2-small"
- group: Model
-
- - name: device
- label: "Device"
- type: select
- options: ["auto", "cpu", "cuda", "mps"]
- default: "auto"
- group: Performance
-
-capabilities:
- live_transform:
- script: scripts/segment.py
- description: "Interactive segmentation on frames"
----
-
-# SAM2 Interactive Segmentation
-
-Click anywhere on a video frame to segment objects using Meta's Segment Anything 2. Generates pixel-perfect masks for annotation, tracking, and video compositing.
-
-## What You Get
-
-- **Click-to-segment** — click on any object to get its mask
-- **Video propagation** — segment in one frame, track through the video
-- **Annotation** — export masks for dataset creation (COCO format)
-- **Background removal** — isolate objects from scenes
-
-## Protocol
-
-### Aegis → Skill (stdin)
-```jsonl
-{"event": "frame", "camera_id": "front_door", "frame_path": "/tmp/frame.jpg", "timestamp": "..."}
-{"event": "click", "x": 450, "y": 320, "label": 1}
-{"event": "propagate", "direction": "forward", "num_frames": 30}
-```
-
-### Skill → Aegis (stdout)
-```jsonl
-{"event": "ready", "model": "sam2-small", "device": "mps"}
-{"event": "segmentation", "frame_number": 0, "mask_path": "/tmp/mask_001.png", "score": 0.95, "bbox": [100, 50, 350, 420]}
-{"event": "propagation_complete", "frames_processed": 30, "masks_dir": "/tmp/masks/"}
-```
-
-## Setup
-
-```bash
-python3 -m venv .venv && source .venv/bin/activate
-pip install -r requirements.txt
-python scripts/download_model.py --model sam2-small
-```
diff --git a/skills/annotation/sam2-segmentation/scripts/segment.py b/skills/annotation/sam2-segmentation/scripts/segment.py
deleted file mode 100644
index cb96af67..00000000
--- a/skills/annotation/sam2-segmentation/scripts/segment.py
+++ /dev/null
@@ -1,149 +0,0 @@
-#!/usr/bin/env python3
-"""
-SAM2 Segmentation Skill — Interactive click-to-segment.
-
-Generates pixel-perfect masks from point/box prompts using Segment Anything 2.
-"""
-
-import sys
-import json
-import argparse
-import signal
-import tempfile
-from pathlib import Path
-
-
-def parse_args():
- parser = argparse.ArgumentParser(description="SAM2 Segmentation Skill")
- parser.add_argument("--config", type=str)
- parser.add_argument("--model", type=str, default="sam2-small")
- parser.add_argument("--device", type=str, default="auto")
- return parser.parse_args()
-
-
-def load_config(args):
- if args.config and Path(args.config).exists():
- with open(args.config) as f:
- return json.load(f)
- return {"model": args.model, "device": args.device}
-
-
-def select_device(pref):
- if pref != "auto":
- return pref
- try:
- import torch
- if torch.cuda.is_available(): return "cuda"
- if hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): return "mps"
- except ImportError:
- pass
- return "cpu"
-
-
-def emit(event):
- print(json.dumps(event), flush=True)
-
-
-def main():
- args = parse_args()
- config = load_config(args)
- device = select_device(config.get("device", "auto"))
-
- try:
- import torch
- import numpy as np
- import cv2
- from sam2.build_sam import build_sam2
- from sam2.sam2_image_predictor import SAM2ImagePredictor
-
- model_cfg = {
- "sam2-tiny": "sam2_hiera_t.yaml",
- "sam2-small": "sam2_hiera_s.yaml",
- "sam2-base": "sam2_hiera_b+.yaml",
- "sam2-large": "sam2_hiera_l.yaml",
- }
-
- model_name = config.get("model", "sam2-small")
- checkpoint = f"models/{model_name}.pt"
-
- sam2 = build_sam2(model_cfg.get(model_name, "sam2_hiera_s.yaml"), checkpoint)
- predictor = SAM2ImagePredictor(sam2)
- predictor.model.to(device)
-
- emit({"event": "ready", "model": model_name, "device": device})
- except Exception as e:
- emit({"event": "error", "message": f"Failed to load SAM2: {e}", "retriable": False})
- sys.exit(1)
-
- running = True
- current_image = None
-
- def handle_signal(s, f):
- nonlocal running
- running = False
- signal.signal(signal.SIGTERM, handle_signal)
- signal.signal(signal.SIGINT, handle_signal)
-
- for line in sys.stdin:
- if not running:
- break
- line = line.strip()
- if not line:
- continue
- try:
- msg = json.loads(line)
- except json.JSONDecodeError:
- continue
-
- if msg.get("command") == "stop":
- break
-
- event = msg.get("event")
-
- if event == "frame":
- frame_path = msg.get("frame_path")
- if frame_path and Path(frame_path).exists():
- current_image = cv2.imread(frame_path)
- current_image = cv2.cvtColor(current_image, cv2.COLOR_BGR2RGB)
- predictor.set_image(current_image)
-
- elif event == "click" and current_image is not None:
- x, y = msg.get("x", 0), msg.get("y", 0)
- label = msg.get("label", 1) # 1=foreground, 0=background
-
- try:
- point = np.array([[x, y]])
- point_label = np.array([label])
-
- masks, scores, _ = predictor.predict(
- point_coords=point,
- point_labels=point_label,
- multimask_output=True,
- )
-
- # Use highest-scoring mask
- best_idx = np.argmax(scores)
- mask = masks[best_idx]
- score = float(scores[best_idx])
-
- # Save mask
- mask_path = tempfile.mktemp(suffix=".png", dir="/tmp")
- cv2.imwrite(mask_path, (mask * 255).astype(np.uint8))
-
- # Compute bbox from mask
- ys, xs = np.where(mask)
- bbox = [int(xs.min()), int(ys.min()), int(xs.max()), int(ys.max())]
-
- emit({
- "event": "segmentation",
- "frame_number": msg.get("frame_number", 0),
- "mask_path": mask_path,
- "score": round(score, 3),
- "bbox": bbox,
- })
- except Exception as e:
- emit({"event": "error", "message": f"Segmentation error: {e}", "retriable": True})
-
-
-if __name__ == "__main__":
- main()
diff --git a/skills/detection/yolo-detection-2026/config.yaml b/skills/detection/yolo-detection-2026/config.yaml
index 62f82256..d84fc4ca 100644
--- a/skills/detection/yolo-detection-2026/config.yaml
+++ b/skills/detection/yolo-detection-2026/config.yaml
@@ -6,7 +6,7 @@ params:
- key: auto_start
label: Auto Start
type: boolean
- default: false
+ default: true
description: "Start this skill automatically when Aegis launches"
- key: model_size
diff --git a/skills/detection/yolo-detection-2026/requirements_mps.txt b/skills/detection/yolo-detection-2026/requirements_mps.txt
index a9e282fa..822288d1 100644
--- a/skills/detection/yolo-detection-2026/requirements_mps.txt
+++ b/skills/detection/yolo-detection-2026/requirements_mps.txt
@@ -1,10 +1,8 @@
# YOLO 2026 — MPS (Apple Silicon) requirements
-# Standard PyTorch — MPS backend is included by default on macOS
-torch>=2.4.0
-torchvision>=0.19.0
-ultralytics>=8.3.0
-coremltools>=8.0
+# Uses ONNX Runtime + CoreML EP for GPU/ANE acceleration.
+# Pre-built yolo26n.onnx is shipped in the repo, so torch/ultralytics
+# are NOT needed at runtime — only onnxruntime for inference.
+onnxruntime>=1.19.0
numpy>=1.24.0,<2.0.0
opencv-python-headless>=4.8.0
Pillow>=10.0.0
-
diff --git a/skills/detection/yolo-detection-2026/scripts/env_config.py b/skills/detection/yolo-detection-2026/scripts/env_config.py
index 7c46c05b..10797702 100644
--- a/skills/detection/yolo-detection-2026/scripts/env_config.py
+++ b/skills/detection/yolo-detection-2026/scripts/env_config.py
@@ -58,11 +58,12 @@ class BackendSpec:
),
"mps": BackendSpec(
name="mps",
- export_format="coreml",
- model_suffix=".mlpackage",
- half=True,
- extra_export_args={"nms": False},
- compute_units="cpu_and_ne", # Route to Neural Engine, leave GPU free for LLM/VLM
+ export_format="onnx",
+ model_suffix=".onnx",
+ half=False, # ONNX Runtime handles precision internally
+ # ONNX Runtime + CoreMLExecutionProvider bypasses the broken
+ # MPSGraphExecutable MLIR pipeline on macOS 26.x while still
+ # leveraging GPU/ANE via CoreML under the hood.
),
"intel": BackendSpec(
name="intel",
@@ -78,6 +79,116 @@ class BackendSpec:
),
}
+# ─── ONNX + CoreML EP wrapper ────────────────────────────────────────────────
+# Provides an ultralytics-compatible model interface using onnxruntime directly
+# with CoreMLExecutionProvider for ~6ms inference on Apple Silicon (vs 21ms when
+# ultralytics defaults to CPUExecutionProvider).
+
+class _BoxResult:
+ """Minimal replacement for ultralytics Boxes result."""
+ __slots__ = ('xyxy', 'conf', 'cls')
+
+ def __init__(self, xyxy, conf, cls):
+ self.xyxy = xyxy # [[x1,y1,x2,y2]]
+ self.conf = conf # [conf]
+ self.cls = cls # [cls_id]
+
+
+class _DetResult:
+ """Minimal replacement for ultralytics Results."""
+ __slots__ = ('boxes',)
+
+ def __init__(self, boxes: list):
+ self.boxes = boxes
+
+
+class _OnnxCoreMLModel:
+ """ONNX Runtime model with CoreML EP, compatible with ultralytics API.
+
+ Supports: model(image_path_or_pil, conf=0.5, verbose=False)
+ Returns: list of _DetResult with .boxes iterable of _BoxResult
+ """
+
+ def __init__(self, session, class_names: dict):
+ self.session = session
+ self.names = class_names
+ self._input_name = session.get_inputs()[0].name
+ # Expected input shape: [1, 3, H, W]
+ shape = session.get_inputs()[0].shape
+ self._input_h = shape[2] if isinstance(shape[2], int) else 640
+ self._input_w = shape[3] if isinstance(shape[3], int) else 640
+
+ def __call__(self, source, conf: float = 0.25, verbose: bool = True, **kwargs):
+ """Run inference on an image path or PIL Image.
+
+ All models use onnx-community HuggingFace format:
+ outputs[0] = logits [1, 300, 80] (raw, pre-sigmoid)
+ outputs[1] = pred_boxes [1, 300, 4] (cx, cy, w, h normalized 0..1)
+ """
+ import numpy as np
+ from PIL import Image
+
+ # Load image
+ if isinstance(source, str):
+ img = Image.open(source).convert("RGB")
+ elif isinstance(source, Image.Image):
+ img = source.convert("RGB")
+ else:
+ img = Image.fromarray(source).convert("RGB")
+
+ orig_w, orig_h = img.size
+
+ # Letterbox resize to input size
+ scale = min(self._input_w / orig_w, self._input_h / orig_h)
+ new_w, new_h = int(orig_w * scale), int(orig_h * scale)
+ img_resized = img.resize((new_w, new_h), Image.BILINEAR)
+
+ # Pad to input size (center)
+ pad_x = (self._input_w - new_w) // 2
+ pad_y = (self._input_h - new_h) // 2
+ canvas = np.full((self._input_h, self._input_w, 3), 114, dtype=np.uint8)
+ canvas[pad_y:pad_y + new_h, pad_x:pad_x + new_w] = np.array(img_resized)
+
+ # HWC→CHW, normalize, add batch dim
+ blob = canvas.transpose(2, 0, 1).astype(np.float32) / 255.0
+ blob = np.expand_dims(blob, 0)
+
+ # Run inference
+ outputs = self.session.run(None, {self._input_name: blob})
+ logits = outputs[0][0] # [300, 80] raw class logits
+ pred_boxes = outputs[1][0] # [300, 4] cx, cy, w, h (normalized 0..1)
+
+ # Sigmoid → class probabilities
+ probs = 1.0 / (1.0 + np.exp(-logits))
+
+ # Parse detections
+ boxes = []
+ for i in range(len(pred_boxes)):
+ cls_id = int(np.argmax(probs[i]))
+ det_conf = float(probs[i][cls_id])
+ if det_conf < conf:
+ continue
+
+ # cx,cy,w,h (normalized) → x1,y1,x2,y2 (original image pixels)
+ cx, cy, bw, bh = pred_boxes[i]
+ px_cx = cx * self._input_w
+ px_cy = cy * self._input_h
+ px_w = bw * self._input_w
+ px_h = bh * self._input_h
+
+ x1 = max(0, min((px_cx - px_w / 2 - pad_x) / scale, orig_w))
+ y1 = max(0, min((px_cy - px_h / 2 - pad_y) / scale, orig_h))
+ x2 = max(0, min((px_cx + px_w / 2 - pad_x) / scale, orig_w))
+ y2 = max(0, min((px_cy + px_h / 2 - pad_y) / scale, orig_h))
+
+ boxes.append(_BoxResult(
+ xyxy=np.array([[x1, y1, x2, y2]]),
+ conf=np.array([det_conf]),
+ cls=np.array([cls_id]),
+ ))
+
+ return [_DetResult(boxes)]
+
# ─── Hardware detection ──────────────────────────────────────────────────────
@@ -133,31 +244,79 @@ def detect() -> "HardwareEnv":
return env
def _try_cuda(self) -> bool:
- """Detect NVIDIA GPU via nvidia-smi and torch."""
- if not shutil.which("nvidia-smi"):
- return False
+ """Detect NVIDIA GPU via nvidia-smi (with Windows path search) and WMI fallback."""
+ nvidia_smi = shutil.which("nvidia-smi")
+
+ # Windows: check well-known paths if not on PATH
+ if not nvidia_smi and platform.system() == "Windows":
+ for candidate in [
+ Path(os.environ.get("PROGRAMFILES", r"C:\Program Files"))
+ / "NVIDIA Corporation" / "NVSMI" / "nvidia-smi.exe",
+ Path(os.environ.get("WINDIR", r"C:\Windows"))
+ / "System32" / "nvidia-smi.exe",
+ ]:
+ if candidate.is_file():
+ nvidia_smi = str(candidate)
+ _log(f"Found nvidia-smi at {nvidia_smi}")
+ break
+
+ if nvidia_smi:
+ try:
+ result = subprocess.run(
+ [nvidia_smi, "--query-gpu=name,memory.total,driver_version",
+ "--format=csv,noheader,nounits"],
+ capture_output=True, text=True, timeout=10,
+ )
+ if result.returncode == 0:
+ line = result.stdout.strip().split("\n")[0]
+ parts = [p.strip() for p in line.split(",")]
+ if len(parts) >= 3:
+ self.backend = "cuda"
+ self.device = "cuda"
+ self.gpu_name = parts[0]
+ self.gpu_memory_mb = int(float(parts[1]))
+ self.driver_version = parts[2]
+ self.detection_details["nvidia_smi"] = line
+ _log(f"NVIDIA GPU: {self.gpu_name} ({self.gpu_memory_mb}MB, driver {self.driver_version})")
+ return True
+ except (subprocess.TimeoutExpired, FileNotFoundError, ValueError) as e:
+ _log(f"nvidia-smi probe failed: {e}")
+
+ # Windows WMI fallback: detect NVIDIA GPU even without nvidia-smi on PATH
+ if platform.system() == "Windows":
+ return self._try_cuda_wmi()
+
+ return False
+
+ def _try_cuda_wmi(self) -> bool:
+ """Windows-only: detect NVIDIA GPU via WMI (wmic)."""
try:
result = subprocess.run(
- ["nvidia-smi", "--query-gpu=name,memory.total,driver_version",
- "--format=csv,noheader,nounits"],
+ ["wmic", "path", "win32_VideoController", "get",
+ "Name,AdapterRAM,DriverVersion", "/format:csv"],
capture_output=True, text=True, timeout=10,
)
if result.returncode != 0:
return False
- line = result.stdout.strip().split("\n")[0]
- parts = [p.strip() for p in line.split(",")]
- if len(parts) >= 3:
- self.backend = "cuda"
- self.device = "cuda"
- self.gpu_name = parts[0]
- self.gpu_memory_mb = int(float(parts[1]))
- self.driver_version = parts[2]
- self.detection_details["nvidia_smi"] = line
- _log(f"NVIDIA GPU: {self.gpu_name} ({self.gpu_memory_mb}MB, driver {self.driver_version})")
- return True
+ for line in result.stdout.strip().split("\n"):
+ if "NVIDIA" in line.upper():
+ parts = [p.strip() for p in line.split(",")]
+ # CSV format: Node,AdapterRAM,DriverVersion,Name
+ if len(parts) >= 4:
+ self.backend = "cuda"
+ self.device = "cuda"
+ self.gpu_name = parts[3]
+ try:
+ self.gpu_memory_mb = int(int(parts[1]) / (1024 * 1024))
+ except (ValueError, IndexError):
+ pass
+ self.driver_version = parts[2] if len(parts) > 2 else ""
+ self.detection_details["wmi"] = line
+ _log(f"NVIDIA GPU (WMI): {self.gpu_name} ({self.gpu_memory_mb}MB)")
+ return True
except (subprocess.TimeoutExpired, FileNotFoundError, ValueError) as e:
- _log(f"nvidia-smi probe failed: {e}")
+ _log(f"WMI probe failed: {e}")
return False
def _try_rocm(self) -> bool:
@@ -363,12 +522,28 @@ def _check_rocm_runtime(self):
_log("Fix: pip uninstall onnxruntime && pip install onnxruntime-rocm")
raise ImportError("ROCmExecutionProvider not available")
+ def _check_mps_runtime(self):
+ """Verify onnxruntime has CoreML provider for Apple GPU/ANE acceleration.
+
+ ONNX Runtime + CoreMLExecutionProvider bypasses the broken
+ MPSGraphExecutable MLIR pipeline (macOS 26.x) while still routing
+ inference through CoreML to leverage GPU and Neural Engine.
+ """
+ import onnxruntime
+ providers = onnxruntime.get_available_providers()
+ if "CoreMLExecutionProvider" in providers:
+ _log(f"onnxruntime CoreML provider available: {providers}")
+ return True
+ _log(f"onnxruntime providers: {providers} — CoreMLExecutionProvider not found")
+ _log("Fix: pip install onnxruntime (arm64 macOS wheel includes CoreML EP)")
+ raise ImportError("CoreMLExecutionProvider not available")
+
def _check_framework(self) -> bool:
- """Check if the optimized inference runtime is importable."""
+ """Check if the optimized inference runtime is importable and compatible."""
checks = {
"cuda": lambda: __import__("tensorrt"),
"rocm": lambda: self._check_rocm_runtime(),
- "mps": lambda: __import__("coremltools"),
+ "mps": lambda: self._check_mps_runtime(),
"intel": lambda: __import__("openvino"),
"cpu": lambda: __import__("onnxruntime"),
}
@@ -496,6 +671,109 @@ def __init__(self, *args, **kwargs):
_log("coremltools not available, loading without compute_units")
return YOLO(model_path)
+ # ── ONNX model download from HuggingFace ──────────────────────────
+
+ # Maps model base name → onnx-community HuggingFace repo
+ _ONNX_HF_REPOS = {
+ "yolo26n": "onnx-community/yolo26n-ONNX",
+ "yolo26s": "onnx-community/yolo26s-ONNX",
+ "yolo26m": "onnx-community/yolo26m-ONNX",
+ "yolo26l": "onnx-community/yolo26l-ONNX",
+ }
+
+ def _download_onnx_from_hf(self, model_name: str, dest_path: Path) -> bool:
+ """Download pre-built ONNX model from onnx-community on HuggingFace.
+
+ Uses urllib (no extra dependencies). Downloads to dest_path.
+ Returns True on success, False on failure.
+ """
+ repo = self._ONNX_HF_REPOS.get(model_name)
+ if not repo:
+ _log(f"No HuggingFace repo for {model_name}")
+ return False
+
+ url = f"https://huggingface.co/{repo}/resolve/main/onnx/model.onnx"
+ names_url = None # class names not available on HF, use bundled nano names
+
+ _log(f"Downloading {model_name}.onnx from {repo}...")
+ try:
+ import urllib.request
+ import shutil
+
+ # Download ONNX model
+ tmp_path = str(dest_path) + ".download"
+ with urllib.request.urlopen(url) as resp, open(tmp_path, 'wb') as f:
+ shutil.copyfileobj(resp, f)
+
+ # Rename to final path
+ Path(tmp_path).rename(dest_path)
+ size_mb = dest_path.stat().st_size / 1e6
+ _log(f"Downloaded {model_name}.onnx ({size_mb:.1f} MB)")
+
+ # Create class names JSON if missing (COCO 80 — same for all YOLO models)
+ names_path = Path(str(dest_path).replace('.onnx', '_names.json'))
+ if not names_path.exists():
+ # Try copying from nano (which is shipped in the repo)
+ nano_names = dest_path.parent / "yolo26n_names.json"
+ if nano_names.exists():
+ shutil.copy2(str(nano_names), str(names_path))
+ _log(f"Copied class names from yolo26n_names.json")
+ else:
+ # Generate default COCO names
+ import json
+ coco_names = {str(i): f"class_{i}" for i in range(80)}
+ with open(str(names_path), 'w') as f:
+ json.dump(coco_names, f)
+ _log("Generated default class names")
+
+ return True
+ except Exception as e:
+ _log(f"HuggingFace download failed: {e}")
+ # Clean up partial download
+ for p in [str(dest_path) + ".download", str(dest_path)]:
+ try:
+ Path(p).unlink(missing_ok=True)
+ except Exception:
+ pass
+ return False
+
+ def _load_onnx_coreml(self, onnx_path: str):
+ """Load ONNX model with CoreMLExecutionProvider for fast GPU/ANE inference.
+
+ Returns an OnnxCoreMLModel wrapper that is compatible with the
+ ultralytics model(frame_path, conf=...) call pattern.
+ """
+ import onnxruntime as ort
+
+ providers = ['CoreMLExecutionProvider', 'CPUExecutionProvider']
+ session = ort.InferenceSession(onnx_path, providers=providers)
+ active = session.get_providers()
+ _log(f"ONNX+CoreML session: {active}")
+
+ # Load class names from companion JSON (avoids torch/ultralytics dep)
+ import json
+ names_path = onnx_path.replace('.onnx', '_names.json')
+ try:
+ with open(names_path) as f:
+ raw = json.load(f)
+ # JSON keys are strings; convert to int-keyed dict
+ class_names = {int(k): v for k, v in raw.items()}
+ _log(f"Loaded {len(class_names)} class names from {Path(names_path).name}")
+ except FileNotFoundError:
+ # Fallback: try loading from .pt if JSON doesn't exist
+ try:
+ from ultralytics import YOLO
+ pt_path = onnx_path.replace('.onnx', '.pt')
+ pt_model = YOLO(pt_path)
+ class_names = pt_model.names
+ _log(f"Loaded class names from {Path(pt_path).name} (fallback)")
+ except Exception:
+ # Last resort: use COCO 80-class defaults
+ _log("WARNING: No class names found, using generic labels")
+ class_names = {i: f"class_{i}" for i in range(80)}
+
+ return _OnnxCoreMLModel(session, class_names)
+
def load_optimized(self, model_name: str, use_optimized: bool = True):
"""
Load the best available model for this hardware.
@@ -512,10 +790,9 @@ def load_optimized(self, model_name: str, use_optimized: bool = True):
optimized_path = self.get_optimized_path(model_name)
if optimized_path.exists():
try:
- # On Apple Silicon: route CoreML to Neural Engine
- if self.backend == "mps" and self.compute_units != "all":
- model = self._load_coreml_with_compute_units(
- str(optimized_path))
+ # MPS: use ONNX Runtime + CoreML EP for fast inference
+ if self.backend == "mps":
+ model = self._load_onnx_coreml(str(optimized_path))
else:
model = YOLO(str(optimized_path))
self.load_ms = (time.perf_counter() - t0) * 1000
@@ -524,15 +801,27 @@ def load_optimized(self, model_name: str, use_optimized: bool = True):
except Exception as e:
_log(f"Failed to load cached model: {e}")
+ # Try downloading pre-built ONNX from HuggingFace (no torch needed)
+ if self.export_format == "onnx" and self._download_onnx_from_hf(model_name, optimized_path):
+ try:
+ if self.backend == "mps":
+ model = self._load_onnx_coreml(str(optimized_path))
+ else:
+ model = YOLO(str(optimized_path))
+ self.load_ms = (time.perf_counter() - t0) * 1000
+ _log(f"Loaded HuggingFace ONNX model ({self.load_ms:.0f}ms)")
+ return model, self.export_format
+ except Exception as e:
+ _log(f"Failed to load HF-downloaded model: {e}")
+
# Try exporting then loading
pt_model = YOLO(f"{model_name}.pt")
exported = self.export_model(pt_model, model_name)
if exported:
try:
- # On Apple Silicon: route CoreML to Neural Engine
- if self.backend == "mps" and self.compute_units != "all":
- model = self._load_coreml_with_compute_units(
- str(exported))
+ # MPS: use ONNX Runtime + CoreML EP for fast inference
+ if self.backend == "mps":
+ model = self._load_onnx_coreml(str(exported))
else:
model = YOLO(str(exported))
self.load_ms = (time.perf_counter() - t0) * 1000
diff --git a/skills/detection/yolo-detection-2026/yolo26n.onnx b/skills/detection/yolo-detection-2026/yolo26n.onnx
new file mode 100644
index 00000000..1b015a02
Binary files /dev/null and b/skills/detection/yolo-detection-2026/yolo26n.onnx differ
diff --git a/skills/detection/yolo-detection-2026/yolo26n_names.json b/skills/detection/yolo-detection-2026/yolo26n_names.json
new file mode 100644
index 00000000..67db67b1
--- /dev/null
+++ b/skills/detection/yolo-detection-2026/yolo26n_names.json
@@ -0,0 +1,82 @@
+{
+ "0": "person",
+ "1": "bicycle",
+ "2": "car",
+ "3": "motorcycle",
+ "4": "airplane",
+ "5": "bus",
+ "6": "train",
+ "7": "truck",
+ "8": "boat",
+ "9": "traffic light",
+ "10": "fire hydrant",
+ "11": "stop sign",
+ "12": "parking meter",
+ "13": "bench",
+ "14": "bird",
+ "15": "cat",
+ "16": "dog",
+ "17": "horse",
+ "18": "sheep",
+ "19": "cow",
+ "20": "elephant",
+ "21": "bear",
+ "22": "zebra",
+ "23": "giraffe",
+ "24": "backpack",
+ "25": "umbrella",
+ "26": "handbag",
+ "27": "tie",
+ "28": "suitcase",
+ "29": "frisbee",
+ "30": "skis",
+ "31": "snowboard",
+ "32": "sports ball",
+ "33": "kite",
+ "34": "baseball bat",
+ "35": "baseball glove",
+ "36": "skateboard",
+ "37": "surfboard",
+ "38": "tennis racket",
+ "39": "bottle",
+ "40": "wine glass",
+ "41": "cup",
+ "42": "fork",
+ "43": "knife",
+ "44": "spoon",
+ "45": "bowl",
+ "46": "banana",
+ "47": "apple",
+ "48": "sandwich",
+ "49": "orange",
+ "50": "broccoli",
+ "51": "carrot",
+ "52": "hot dog",
+ "53": "pizza",
+ "54": "donut",
+ "55": "cake",
+ "56": "chair",
+ "57": "couch",
+ "58": "potted plant",
+ "59": "bed",
+ "60": "dining table",
+ "61": "toilet",
+ "62": "tv",
+ "63": "laptop",
+ "64": "mouse",
+ "65": "remote",
+ "66": "keyboard",
+ "67": "cell phone",
+ "68": "microwave",
+ "69": "oven",
+ "70": "toaster",
+ "71": "sink",
+ "72": "refrigerator",
+ "73": "book",
+ "74": "clock",
+ "75": "vase",
+ "76": "scissors",
+ "77": "teddy bear",
+ "78": "hair drier",
+ "79": "toothbrush"
+}
\ No newline at end of file
diff --git a/skills/lib/env_config.py b/skills/lib/env_config.py
index 1669f03c..10797702 100644
--- a/skills/lib/env_config.py
+++ b/skills/lib/env_config.py
@@ -58,11 +58,12 @@ class BackendSpec:
),
"mps": BackendSpec(
name="mps",
- export_format="coreml",
- model_suffix=".mlpackage",
- half=True,
- extra_export_args={"nms": False},
- compute_units="cpu_and_ne", # Route to Neural Engine, leave GPU free for LLM/VLM
+ export_format="onnx",
+ model_suffix=".onnx",
+ half=False, # ONNX Runtime handles precision internally
+ # ONNX Runtime + CoreMLExecutionProvider bypasses the broken
+ # MPSGraphExecutable MLIR pipeline on macOS 26.x while still
+ # leveraging GPU/ANE via CoreML under the hood.
),
"intel": BackendSpec(
name="intel",
@@ -78,6 +79,116 @@ class BackendSpec:
),
}
+# ─── ONNX + CoreML EP wrapper ────────────────────────────────────────────────
+# Provides an ultralytics-compatible model interface using onnxruntime directly
+# with CoreMLExecutionProvider for ~6ms inference on Apple Silicon (vs 21ms when
+# ultralytics defaults to CPUExecutionProvider).
+
+class _BoxResult:
+ """Minimal replacement for ultralytics Boxes result."""
+ __slots__ = ('xyxy', 'conf', 'cls')
+
+ def __init__(self, xyxy, conf, cls):
+ self.xyxy = xyxy # [[x1,y1,x2,y2]]
+ self.conf = conf # [conf]
+ self.cls = cls # [cls_id]
+
+
+class _DetResult:
+ """Minimal replacement for ultralytics Results."""
+ __slots__ = ('boxes',)
+
+ def __init__(self, boxes: list):
+ self.boxes = boxes
+
+
+class _OnnxCoreMLModel:
+ """ONNX Runtime model with CoreML EP, compatible with ultralytics API.
+
+ Supports: model(image_path_or_pil, conf=0.5, verbose=False)
+ Returns: list of _DetResult with .boxes iterable of _BoxResult
+ """
+
+ def __init__(self, session, class_names: dict):
+ self.session = session
+ self.names = class_names
+ self._input_name = session.get_inputs()[0].name
+ # Expected input shape: [1, 3, H, W]
+ shape = session.get_inputs()[0].shape
+ self._input_h = shape[2] if isinstance(shape[2], int) else 640
+ self._input_w = shape[3] if isinstance(shape[3], int) else 640
+
+ def __call__(self, source, conf: float = 0.25, verbose: bool = True, **kwargs):
+ """Run inference on an image path or PIL Image.
+
+ All models use onnx-community HuggingFace format:
+ outputs[0] = logits [1, 300, 80] (raw, pre-sigmoid)
+ outputs[1] = pred_boxes [1, 300, 4] (cx, cy, w, h normalized 0..1)
+ """
+ import numpy as np
+ from PIL import Image
+
+ # Load image
+ if isinstance(source, str):
+ img = Image.open(source).convert("RGB")
+ elif isinstance(source, Image.Image):
+ img = source.convert("RGB")
+ else:
+ img = Image.fromarray(source).convert("RGB")
+
+ orig_w, orig_h = img.size
+
+ # Letterbox resize to input size
+ scale = min(self._input_w / orig_w, self._input_h / orig_h)
+ new_w, new_h = int(orig_w * scale), int(orig_h * scale)
+ img_resized = img.resize((new_w, new_h), Image.BILINEAR)
+
+ # Pad to input size (center)
+ pad_x = (self._input_w - new_w) // 2
+ pad_y = (self._input_h - new_h) // 2
+ canvas = np.full((self._input_h, self._input_w, 3), 114, dtype=np.uint8)
+ canvas[pad_y:pad_y + new_h, pad_x:pad_x + new_w] = np.array(img_resized)
+
+ # HWC→CHW, normalize, add batch dim
+ blob = canvas.transpose(2, 0, 1).astype(np.float32) / 255.0
+ blob = np.expand_dims(blob, 0)
+
+ # Run inference
+ outputs = self.session.run(None, {self._input_name: blob})
+ logits = outputs[0][0] # [300, 80] raw class logits
+ pred_boxes = outputs[1][0] # [300, 4] cx, cy, w, h (normalized 0..1)
+
+ # Sigmoid → class probabilities
+ probs = 1.0 / (1.0 + np.exp(-logits))
+
+ # Parse detections
+ boxes = []
+ for i in range(len(pred_boxes)):
+ cls_id = int(np.argmax(probs[i]))
+ det_conf = float(probs[i][cls_id])
+ if det_conf < conf:
+ continue
+
+ # cx,cy,w,h (normalized) → x1,y1,x2,y2 (original image pixels)
+ cx, cy, bw, bh = pred_boxes[i]
+ px_cx = cx * self._input_w
+ px_cy = cy * self._input_h
+ px_w = bw * self._input_w
+ px_h = bh * self._input_h
+
+ x1 = max(0, min((px_cx - px_w / 2 - pad_x) / scale, orig_w))
+ y1 = max(0, min((px_cy - px_h / 2 - pad_y) / scale, orig_h))
+ x2 = max(0, min((px_cx + px_w / 2 - pad_x) / scale, orig_w))
+ y2 = max(0, min((px_cy + px_h / 2 - pad_y) / scale, orig_h))
+
+ boxes.append(_BoxResult(
+ xyxy=np.array([[x1, y1, x2, y2]]),
+ conf=np.array([det_conf]),
+ cls=np.array([cls_id]),
+ ))
+
+ return [_DetResult(boxes)]
+
# ─── Hardware detection ──────────────────────────────────────────────────────
@@ -411,12 +522,28 @@ def _check_rocm_runtime(self):
_log("Fix: pip uninstall onnxruntime && pip install onnxruntime-rocm")
raise ImportError("ROCmExecutionProvider not available")
+ def _check_mps_runtime(self):
+ """Verify onnxruntime has CoreML provider for Apple GPU/ANE acceleration.
+
+ ONNX Runtime + CoreMLExecutionProvider bypasses the broken
+ MPSGraphExecutable MLIR pipeline (macOS 26.x) while still routing
+ inference through CoreML to leverage GPU and Neural Engine.
+ """
+ import onnxruntime
+ providers = onnxruntime.get_available_providers()
+ if "CoreMLExecutionProvider" in providers:
+ _log(f"onnxruntime CoreML provider available: {providers}")
+ return True
+ _log(f"onnxruntime providers: {providers} — CoreMLExecutionProvider not found")
+ _log("Fix: pip install onnxruntime (arm64 macOS wheel includes CoreML EP)")
+ raise ImportError("CoreMLExecutionProvider not available")
+
def _check_framework(self) -> bool:
- """Check if the optimized inference runtime is importable."""
+ """Check if the optimized inference runtime is importable and compatible."""
checks = {
"cuda": lambda: __import__("tensorrt"),
"rocm": lambda: self._check_rocm_runtime(),
- "mps": lambda: __import__("coremltools"),
+ "mps": lambda: self._check_mps_runtime(),
"intel": lambda: __import__("openvino"),
"cpu": lambda: __import__("onnxruntime"),
}
@@ -544,6 +671,109 @@ def __init__(self, *args, **kwargs):
_log("coremltools not available, loading without compute_units")
return YOLO(model_path)
+ # ── ONNX model download from HuggingFace ──────────────────────────
+
+ # Maps model base name → onnx-community HuggingFace repo
+ _ONNX_HF_REPOS = {
+ "yolo26n": "onnx-community/yolo26n-ONNX",
+ "yolo26s": "onnx-community/yolo26s-ONNX",
+ "yolo26m": "onnx-community/yolo26m-ONNX",
+ "yolo26l": "onnx-community/yolo26l-ONNX",
+ }
+
+ def _download_onnx_from_hf(self, model_name: str, dest_path: Path) -> bool:
+ """Download pre-built ONNX model from onnx-community on HuggingFace.
+
+ Uses urllib (no extra dependencies). Downloads to dest_path.
+ Returns True on success, False on failure.
+ """
+ repo = self._ONNX_HF_REPOS.get(model_name)
+ if not repo:
+ _log(f"No HuggingFace repo for {model_name}")
+ return False
+
+ url = f"https://huggingface.co/{repo}/resolve/main/onnx/model.onnx"
+ names_url = None # class names not available on HF, use bundled nano names
+
+ _log(f"Downloading {model_name}.onnx from {repo}...")
+ try:
+ import urllib.request
+ import shutil
+
+ # Download ONNX model
+ tmp_path = str(dest_path) + ".download"
+ with urllib.request.urlopen(url) as resp, open(tmp_path, 'wb') as f:
+ shutil.copyfileobj(resp, f)
+
+ # Rename to final path
+ Path(tmp_path).rename(dest_path)
+ size_mb = dest_path.stat().st_size / 1e6
+ _log(f"Downloaded {model_name}.onnx ({size_mb:.1f} MB)")
+
+ # Create class names JSON if missing (COCO 80 — same for all YOLO models)
+ names_path = Path(str(dest_path).replace('.onnx', '_names.json'))
+ if not names_path.exists():
+ # Try copying from nano (which is shipped in the repo)
+ nano_names = dest_path.parent / "yolo26n_names.json"
+ if nano_names.exists():
+ shutil.copy2(str(nano_names), str(names_path))
+ _log(f"Copied class names from yolo26n_names.json")
+ else:
+ # Generate default COCO names
+ import json
+ coco_names = {str(i): f"class_{i}" for i in range(80)}
+ with open(str(names_path), 'w') as f:
+ json.dump(coco_names, f)
+ _log("Generated default class names")
+
+ return True
+ except Exception as e:
+ _log(f"HuggingFace download failed: {e}")
+ # Clean up partial download
+ for p in [str(dest_path) + ".download", str(dest_path)]:
+ try:
+ Path(p).unlink(missing_ok=True)
+ except Exception:
+ pass
+ return False
+
+ def _load_onnx_coreml(self, onnx_path: str):
+ """Load ONNX model with CoreMLExecutionProvider for fast GPU/ANE inference.
+
+ Returns an OnnxCoreMLModel wrapper that is compatible with the
+ ultralytics model(frame_path, conf=...) call pattern.
+ """
+ import onnxruntime as ort
+
+ providers = ['CoreMLExecutionProvider', 'CPUExecutionProvider']
+ session = ort.InferenceSession(onnx_path, providers=providers)
+ active = session.get_providers()
+ _log(f"ONNX+CoreML session: {active}")
+
+ # Load class names from companion JSON (avoids torch/ultralytics dep)
+ import json
+ names_path = onnx_path.replace('.onnx', '_names.json')
+ try:
+ with open(names_path) as f:
+ raw = json.load(f)
+ # JSON keys are strings; convert to int-keyed dict
+ class_names = {int(k): v for k, v in raw.items()}
+ _log(f"Loaded {len(class_names)} class names from {Path(names_path).name}")
+ except FileNotFoundError:
+ # Fallback: try loading from .pt if JSON doesn't exist
+ try:
+ from ultralytics import YOLO
+ pt_path = onnx_path.replace('.onnx', '.pt')
+ pt_model = YOLO(pt_path)
+ class_names = pt_model.names
+ _log(f"Loaded class names from {Path(pt_path).name} (fallback)")
+ except Exception:
+ # Last resort: use COCO 80-class defaults
+ _log("WARNING: No class names found, using generic labels")
+ class_names = {i: f"class_{i}" for i in range(80)}
+
+ return _OnnxCoreMLModel(session, class_names)
+
def load_optimized(self, model_name: str, use_optimized: bool = True):
"""
Load the best available model for this hardware.
@@ -560,10 +790,9 @@ def load_optimized(self, model_name: str, use_optimized: bool = True):
optimized_path = self.get_optimized_path(model_name)
if optimized_path.exists():
try:
- # On Apple Silicon: route CoreML to Neural Engine
- if self.backend == "mps" and self.compute_units != "all":
- model = self._load_coreml_with_compute_units(
- str(optimized_path))
+ # MPS: use ONNX Runtime + CoreML EP for fast inference
+ if self.backend == "mps":
+ model = self._load_onnx_coreml(str(optimized_path))
else:
model = YOLO(str(optimized_path))
self.load_ms = (time.perf_counter() - t0) * 1000
@@ -572,15 +801,27 @@ def load_optimized(self, model_name: str, use_optimized: bool = True):
except Exception as e:
_log(f"Failed to load cached model: {e}")
+ # Try downloading pre-built ONNX from HuggingFace (no torch needed)
+ if self.export_format == "onnx" and self._download_onnx_from_hf(model_name, optimized_path):
+ try:
+ if self.backend == "mps":
+ model = self._load_onnx_coreml(str(optimized_path))
+ else:
+ model = YOLO(str(optimized_path))
+ self.load_ms = (time.perf_counter() - t0) * 1000
+ _log(f"Loaded HuggingFace ONNX model ({self.load_ms:.0f}ms)")
+ return model, self.export_format
+ except Exception as e:
+ _log(f"Failed to load HF-downloaded model: {e}")
+
# Try exporting then loading
pt_model = YOLO(f"{model_name}.pt")
exported = self.export_model(pt_model, model_name)
if exported:
try:
- # On Apple Silicon: route CoreML to Neural Engine
- if self.backend == "mps" and self.compute_units != "all":
- model = self._load_coreml_with_compute_units(
- str(exported))
+ # MPS: use ONNX Runtime + CoreML EP for fast inference
+ if self.backend == "mps":
+ model = self._load_onnx_coreml(str(exported))
else:
model = YOLO(str(exported))
self.load_ms = (time.perf_counter() - t0) * 1000
diff --git a/skills/segmentation/sam2-segmentation/SKILL.md b/skills/segmentation/sam2-segmentation/SKILL.md
new file mode 100644
index 00000000..818f9b68
--- /dev/null
+++ b/skills/segmentation/sam2-segmentation/SKILL.md
@@ -0,0 +1,67 @@
+---
+name: segmentation-sam2
+description: "Interactive click-to-segment using Segment Anything 2 — AI-assisted labeling for Annotation Studio"
+version: 1.0.0
+entry: scripts/segment.py
+deploy: deploy.sh
+
+parameters:
+ - name: model
+ label: "SAM2 Model"
+ type: select
+ options: ["sam2-tiny", "sam2-small", "sam2-base", "sam2-large"]
+ default: "sam2-small"
+ group: Model
+
+ - name: device
+ label: "Device"
+ type: select
+ options: ["auto", "cpu", "cuda", "mps"]
+ default: "auto"
+ group: Performance
+
+capabilities:
+ live_transform:
+ script: scripts/segment.py
+ description: "Interactive segmentation on frames"
+
+---
+
+# SAM2 Interactive Segmentation
+
+Click anywhere on a video frame to segment objects using Meta's Segment Anything 2. Generates pixel-perfect masks for annotation, tracking, and dataset creation.
+
+## What You Get
+
+- **Click-to-segment** — click on any object to get its mask
+- **Point & box prompts** — positive/negative points and bounding box selection
+- **Video tracking** — segment in one frame, propagate across the clip
+- **Annotation Studio** — full integration with sidebar Annotation Studio
+
+## Protocol
+
+Communicates via **JSON lines** over stdin/stdout.
+
+### Aegis → Skill (stdin)
+```jsonl
+{"event": "frame", "frame_path": "/tmp/frame.jpg", "frame_id": "frame_1", "request_id": "req_001"}
+{"command": "segment", "points": [{"x": 450, "y": 320, "label": 1}], "request_id": "req_002"}
+{"command": "track", "frame_path": "/tmp/frame2.jpg", "frame_id": "frame_2", "request_id": "req_003"}
+{"command": "stop"}
+```
+
+### Skill → Aegis (stdout)
+```jsonl
+{"event": "segmentation", "type": "ready", "request_id": "", "data": {"model": "sam2-small", "device": "mps"}}
+{"event": "segmentation", "type": "encoded", "request_id": "req_001", "data": {"frame_id": "frame_1", "width": 1920, "height": 1080}}
+{"event": "segmentation", "type": "segmented", "request_id": "req_002", "data": {"mask_path": "/tmp/mask.png", "mask_b64": "...", "score": 0.95, "bbox": [100, 50, 350, 420]}}
+{"event": "segmentation", "type": "tracked", "request_id": "req_003", "data": {"frame_id": "frame_2", "mask_path": "/tmp/track.png", "score": 0.93}}
+```
+
+## Installation
+
+The `deploy.sh` bootstrapper handles everything — Python environment, GPU detection, dependency installation, and model download. No manual setup required.
+
+```bash
+./deploy.sh
+```
diff --git a/skills/segmentation/sam2-segmentation/deploy.bat b/skills/segmentation/sam2-segmentation/deploy.bat
new file mode 100644
index 00000000..95fdc557
--- /dev/null
+++ b/skills/segmentation/sam2-segmentation/deploy.bat
@@ -0,0 +1,158 @@
+@echo off
+REM deploy.bat — Bootstrapper for SAM2 Segmentation Skill (Windows)
+REM
+REM Creates venv, installs dependencies, downloads model checkpoint.
+REM Called by Aegis skill-runtime-manager during installation.
+REM
+REM Exit codes:
+REM 0 = success
+REM 1 = fatal error
+
+setlocal enabledelayedexpansion
+
+set "SKILL_DIR=%~dp0"
+REM Remove trailing backslash
+if "%SKILL_DIR:~-1%"=="\" set "SKILL_DIR=%SKILL_DIR:~0,-1%"
+set "VENV_DIR=%SKILL_DIR%\.venv"
+set "MODELS_DIR=%SKILL_DIR%\models"
+set "LOG_PREFIX=[SAM2-deploy]"
+
+REM ─── Step 1: Find Python ───────────────────────────────────────────────────
+
+echo %LOG_PREFIX% Searching for Python...>&2
+
+set "PYTHON_CMD="
+
+REM Try the Windows Python launcher (py.exe) first
+for %%V in (3.12 3.11 3.10 3.9) do (
+ if not defined PYTHON_CMD (
+ py -%%V --version >nul 2>&1
+ if !errorlevel! equ 0 (
+ set "PYTHON_CMD=py -%%V"
+ )
+ )
+)
+
+REM Fallback: bare python3 / python on PATH
+if not defined PYTHON_CMD (
+ python3 --version >nul 2>&1
+ if !errorlevel! equ 0 (
+ for /f "tokens=2 delims= " %%A in ('python3 --version 2^>^&1') do set "_pyver=%%A"
+ for /f "tokens=1,2 delims=." %%A in ("!_pyver!") do (
+ if %%A geq 3 if %%B geq 9 set "PYTHON_CMD=python3"
+ )
+ )
+)
+
+if not defined PYTHON_CMD (
+ python --version >nul 2>&1
+ if !errorlevel! equ 0 (
+ for /f "tokens=2 delims= " %%A in ('python --version 2^>^&1') do set "_pyver=%%A"
+ for /f "tokens=1,2 delims=." %%A in ("!_pyver!") do (
+ if %%A geq 3 if %%B geq 9 set "PYTHON_CMD=python"
+ )
+ )
+)
+
+if not defined PYTHON_CMD (
+ echo %LOG_PREFIX% ERROR: No Python ^>=3.9 found. Install Python 3.9+ and retry.>&2
+ echo {"event": "error", "stage": "python", "message": "No Python >=3.9 found"}
+ exit /b 1
+)
+
+for /f "tokens=*" %%A in ('!PYTHON_CMD! --version 2^>^&1') do set "PY_VERSION=%%A"
+echo %LOG_PREFIX% Using Python: %PYTHON_CMD% (%PY_VERSION%)>&2
+echo {"event": "progress", "stage": "python", "message": "Found %PY_VERSION%"}
+
+REM ─── Step 2: Create virtual environment ────────────────────────────────────
+
+if not exist "%VENV_DIR%\Scripts\python.exe" (
+ echo %LOG_PREFIX% Creating virtual environment...>&2
+ %PYTHON_CMD% -m venv "%VENV_DIR%"
+ if !errorlevel! neq 0 (
+ echo %LOG_PREFIX% ERROR: Failed to create virtual environment>&2
+ echo {"event": "error", "stage": "venv", "message": "Failed to create venv"}
+ exit /b 1
+ )
+)
+
+set "PIP=%VENV_DIR%\Scripts\pip.exe"
+set "VPYTHON=%VENV_DIR%\Scripts\python.exe"
+
+"%PIP%" install --upgrade pip -q >nul 2>&1
+
+echo {"event": "progress", "stage": "venv", "message": "Virtual environment ready"}
+
+REM ─── Step 3: Detect GPU and install dependencies ───────────────────────────
+
+set "BACKEND=cpu"
+
+REM Check for NVIDIA GPU
+where nvidia-smi >nul 2>&1
+if !errorlevel! equ 0 (
+ for /f "tokens=*" %%G in ('nvidia-smi --query-gpu^=driver_version --format^=csv^,noheader 2^>nul') do (
+ if not "%%G"=="" (
+ set "BACKEND=cuda"
+ echo %LOG_PREFIX% Detected NVIDIA GPU ^(driver: %%G^)>&2
+ )
+ )
+)
+
+echo {"event": "progress", "stage": "gpu", "backend": "!BACKEND!", "message": "Compute backend: !BACKEND!"}
+
+echo %LOG_PREFIX% Installing dependencies...>&2
+echo {"event": "progress", "stage": "install", "message": "Installing SAM2 dependencies..."}
+
+REM Install PyTorch first (platform-specific)
+if "!BACKEND!"=="cuda" (
+ "%PIP%" install torch torchvision --index-url https://download.pytorch.org/whl/cu124 -q 2>&1 | findstr /V "^$" >nul
+ if !errorlevel! neq 0 (
+ echo %LOG_PREFIX% WARNING: cu124 failed, trying cu121...>&2
+ "%PIP%" install torch torchvision --index-url https://download.pytorch.org/whl/cu121 -q 2>&1 | findstr /V "^$" >nul
+ )
+) else (
+ "%PIP%" install torch torchvision --index-url https://download.pytorch.org/whl/cpu -q 2>&1 | findstr /V "^$" >nul
+)
+
+REM Install remaining deps
+"%PIP%" install -r "%SKILL_DIR%\requirements.txt" -q 2>&1 | findstr /V "^$" >nul
+
+echo {"event": "progress", "stage": "install", "message": "Dependencies installed"}
+
+REM ─── Step 4: Download default model checkpoint ────────────────────────────
+
+if not exist "%MODELS_DIR%" mkdir "%MODELS_DIR%"
+
+set "CHECKPOINT_FILE=%MODELS_DIR%\sam2-small.pt"
+set "CHECKPOINT_URL=https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_small.pt"
+
+if not exist "%CHECKPOINT_FILE%" (
+ echo %LOG_PREFIX% Downloading SAM2 model checkpoint...>&2
+ echo {"event": "progress", "stage": "model", "message": "Downloading SAM2 model (~180MB)..."}
+
+ REM Try PowerShell download (available on all modern Windows)
+ powershell -NoProfile -Command "Invoke-WebRequest -Uri '%CHECKPOINT_URL%' -OutFile '%CHECKPOINT_FILE%'" 2>&1
+
+ if exist "%CHECKPOINT_FILE%" (
+ echo %LOG_PREFIX% Model downloaded: %CHECKPOINT_FILE%>&2
+ echo {"event": "progress", "stage": "model", "message": "Model downloaded"}
+ ) else (
+ echo %LOG_PREFIX% ERROR: Model download failed>&2
+ echo {"event": "error", "stage": "model", "message": "Model download failed"}
+ exit /b 1
+ )
+) else (
+ echo %LOG_PREFIX% Model checkpoint already exists>&2
+ echo {"event": "progress", "stage": "model", "message": "Model already downloaded"}
+)
+
+REM ─── Step 5: Verify installation ───────────────────────────────────────────
+
+echo %LOG_PREFIX% Verifying installation...>&2
+"%VPYTHON%" -c "import torch, numpy, cv2; print(f'PyTorch {torch.__version__}'); print(f'CUDA: {torch.cuda.get_device_name(0)}' if torch.cuda.is_available() else 'Device: CPU')" 2>&1
+
+echo {"event": "complete", "backend": "!BACKEND!", "message": "SAM2 segmentation skill installed (!BACKEND! backend)"}
+echo %LOG_PREFIX% Done! Backend: !BACKEND!>&2
+
+endlocal
+exit /b 0
diff --git a/skills/segmentation/sam2-segmentation/deploy.sh b/skills/segmentation/sam2-segmentation/deploy.sh
new file mode 100755
index 00000000..20f07ed2
--- /dev/null
+++ b/skills/segmentation/sam2-segmentation/deploy.sh
@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+# deploy.sh — Bootstrapper for SAM2 Segmentation Skill
+#
+# Creates venv, installs dependencies, downloads model checkpoint.
+# Called by Aegis skill-runtime-manager during installation.
+#
+# Exit codes:
+# 0 = success
+# 1 = fatal error
+
+set -euo pipefail
+
+SKILL_DIR="$(cd "$(dirname "$0")" && pwd)"
+VENV_DIR="$SKILL_DIR/.venv"
+MODELS_DIR="$SKILL_DIR/models"
+LOG_PREFIX="[SAM2-deploy]"
+
+log() { echo "$LOG_PREFIX $*" >&2; }
+emit() { echo "$1"; } # JSON to stdout for Aegis to parse
+
+# ─── Step 1: Find Python ──────────────────────────────────────────────────
+
+find_python() {
+ for cmd in python3.12 python3.11 python3.10 python3.9 python3; do
+ if command -v "$cmd" &>/dev/null; then
+ local ver
+ ver="$("$cmd" --version 2>&1 | grep -oE '[0-9]+\.[0-9]+')"
+ local major minor
+ major=$(echo "$ver" | cut -d. -f1)
+ minor=$(echo "$ver" | cut -d. -f2)
+ if [ "$major" -ge 3 ] && [ "$minor" -ge 9 ]; then
+ echo "$cmd"
+ return 0
+ fi
+ fi
+ done
+ return 1
+}
+
+PYTHON_CMD=$(find_python) || {
+ log "ERROR: No Python >=3.9 found. Install Python 3.9+ and retry."
+ emit '{"event": "error", "stage": "python", "message": "No Python >=3.9 found"}'
+ exit 1
+}
+
+log "Using Python: $PYTHON_CMD ($($PYTHON_CMD --version 2>&1))"
+emit "{\"event\": \"progress\", \"stage\": \"python\", \"message\": \"Found $($PYTHON_CMD --version 2>&1)\"}"
+
+# ─── Step 2: Create virtual environment ──────────────────────────────────
+
+if [ ! -d "$VENV_DIR" ]; then
+ log "Creating virtual environment..."
+ "$PYTHON_CMD" -m venv "$VENV_DIR"
+fi
+
+# shellcheck disable=SC1091
+source "$VENV_DIR/bin/activate"
+PIP="$VENV_DIR/bin/pip"
+
+"$PIP" install --upgrade pip -q 2>/dev/null || true
+
+emit '{"event": "progress", "stage": "venv", "message": "Virtual environment ready"}'
+
+# ─── Step 3: Detect hardware and install deps ───────────────────────────
+
+BACKEND="cpu"
+if [ "$(uname)" = "Darwin" ] && [ "$(uname -m)" = "arm64" ]; then
+ BACKEND="mps"
+ log "Detected Apple Silicon (MPS)"
+elif command -v nvidia-smi &>/dev/null; then
+ BACKEND="cuda"
+ log "Detected NVIDIA GPU (CUDA)"
+fi
+
+emit "{\"event\": \"progress\", \"stage\": \"gpu\", \"backend\": \"$BACKEND\", \"message\": \"Compute backend: $BACKEND\"}"
+
+log "Installing dependencies..."
+emit '{"event": "progress", "stage": "install", "message": "Installing SAM2 dependencies..."}'
+
+# Install PyTorch first (platform-specific)
+if [ "$BACKEND" = "cuda" ]; then
+ "$PIP" install torch torchvision --index-url https://download.pytorch.org/whl/cu124 -q 2>&1 | tail -3 >&2
+elif [ "$BACKEND" = "mps" ]; then
+ "$PIP" install torch torchvision -q 2>&1 | tail -3 >&2
+else
+ "$PIP" install torch torchvision --index-url https://download.pytorch.org/whl/cpu -q 2>&1 | tail -3 >&2
+fi
+
+# Install remaining deps
+"$PIP" install -r "$SKILL_DIR/requirements.txt" -q 2>&1 | tail -5 >&2
+
+emit '{"event": "progress", "stage": "install", "message": "Dependencies installed"}'
+
+# ─── Step 4: Download default model checkpoint ─────────────────────────
+
+DEFAULT_MODEL="sam2.1-hiera-small"
+CHECKPOINT_URL="https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_small.pt"
+CHECKPOINT_FILE="$MODELS_DIR/sam2-small.pt"
+
+mkdir -p "$MODELS_DIR"
+
+if [ ! -f "$CHECKPOINT_FILE" ]; then
+ log "Downloading SAM2 model checkpoint ($DEFAULT_MODEL)..."
+ emit '{"event": "progress", "stage": "model", "message": "Downloading SAM2 model (~180MB)..."}'
+
+ if command -v curl &>/dev/null; then
+ curl -L -o "$CHECKPOINT_FILE" "$CHECKPOINT_URL" 2>&1 | tail -1 >&2
+ elif command -v wget &>/dev/null; then
+ wget -O "$CHECKPOINT_FILE" "$CHECKPOINT_URL" 2>&1 | tail -1 >&2
+ else
+ log "ERROR: Neither curl nor wget found. Cannot download model."
+ emit '{"event": "error", "stage": "model", "message": "No download tool available"}'
+ exit 1
+ fi
+
+ if [ -f "$CHECKPOINT_FILE" ]; then
+ SIZE=$(du -h "$CHECKPOINT_FILE" | cut -f1)
+ log "Model downloaded: $CHECKPOINT_FILE ($SIZE)"
+ emit "{\"event\": \"progress\", \"stage\": \"model\", \"message\": \"Model downloaded ($SIZE)\"}"
+ else
+ log "ERROR: Model download failed"
+ emit '{"event": "error", "stage": "model", "message": "Model download failed"}'
+ exit 1
+ fi
+else
+ log "Model checkpoint already exists: $CHECKPOINT_FILE"
+ emit '{"event": "progress", "stage": "model", "message": "Model already downloaded"}'
+fi
+
+# ─── Step 5: Verify installation ──────────────────────────────────────────
+
+log "Verifying installation..."
+"$VENV_DIR/bin/python" -c "
+import torch
+import numpy
+import cv2
+print(f'PyTorch {torch.__version__}')
+print(f'NumPy {numpy.__version__}')
+print(f'OpenCV {cv2.__version__}')
+if torch.cuda.is_available():
+ print(f'CUDA: {torch.cuda.get_device_name(0)}')
+elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():
+ print('MPS: Apple Silicon')
+else:
+ print('Device: CPU')
+" 2>&1 | while read -r line; do log "$line"; done
+
+emit "{\"event\": \"complete\", \"backend\": \"$BACKEND\", \"message\": \"SAM2 segmentation skill installed ($BACKEND backend)\"}"
+log "Done! Backend: $BACKEND"
diff --git a/skills/annotation/sam2-segmentation/requirements.txt b/skills/segmentation/sam2-segmentation/requirements.txt
similarity index 100%
rename from skills/annotation/sam2-segmentation/requirements.txt
rename to skills/segmentation/sam2-segmentation/requirements.txt
diff --git a/skills/segmentation/sam2-segmentation/scripts/segment.py b/skills/segmentation/sam2-segmentation/scripts/segment.py
new file mode 100644
index 00000000..26257fe8
--- /dev/null
+++ b/skills/segmentation/sam2-segmentation/scripts/segment.py
@@ -0,0 +1,430 @@
+#!/usr/bin/env python3
+"""
+SAM2 Annotation Skill — Interactive segmentation for Aegis Annotation Studio.
+
+Protocol (JSONL over stdin/stdout):
+ stdin: {"command": "encode", "frame_path": "...", "frame_id": "...", "request_id": "..."}
+ {"command": "segment", "points": [...], "boxes": [...], "request_id": "..."}
+ {"command": "track", "frame_id": "...", "request_id": "..."}
+ {"command": "stop"}
+ stdout: {"event": "segmentation", "type": "encoded"|"segmented"|"tracked"|"ready", ...}
+"""
+
+import sys
+import json
+import argparse
+import signal
+import time
+import tempfile
+import base64
+from pathlib import Path
+
+
+# ───────────────────────────────────────────────────────────────────────────────
+# Helpers
+# ───────────────────────────────────────────────────────────────────────────────
+
+def emit(event: dict):
+ """Send a JSONL event to stdout (Aegis picks this up)."""
+ print(json.dumps(event), flush=True)
+
+
+def log(msg: str):
+ """Log to stderr (visible in skill console, not parsed as protocol)."""
+ print(f"[SAM2] {msg}", file=sys.stderr, flush=True)
+
+
+def emit_segmentation(type_: str, request_id: str, data: dict = None, error: str = None):
+ """Emit a segmentation event in the format skill-runtime-manager.cjs expects."""
+ event = {
+ "event": "segmentation",
+ "type": type_,
+ "request_id": request_id or "",
+ "data": data or {},
+ }
+ if error:
+ event["error"] = error
+ emit(event)
+
+
+# ───────────────────────────────────────────────────────────────────────────────
+# Performance tracker
+# ───────────────────────────────────────────────────────────────────────────────
+
+PERF_INTERVAL = 20
+
+
+class PerfTracker:
+ def __init__(self):
+ self.frame_count = 0
+ self.total_encodes = 0
+ self.total_segments = 0
+ self.total_tracks = 0
+ self._timings: dict[str, list[float]] = {
+ "encode": [], "segment": [], "track": [],
+ }
+
+ def record(self, stage: str, ms: float):
+ if stage in self._timings:
+ self._timings[stage].append(ms)
+
+ def tick(self):
+ self.frame_count += 1
+ if self.frame_count >= PERF_INTERVAL:
+ self._emit()
+ self.frame_count = 0
+
+ def _emit(self):
+ stats = {"event": "perf_stats", "total_encodes": self.total_encodes,
+ "total_segments": self.total_segments, "total_tracks": self.total_tracks,
+ "timings_ms": {}}
+ for stage, vals in self._timings.items():
+ if vals:
+ stats["timings_ms"][stage] = {
+ "avg": round(sum(vals) / len(vals), 1),
+ "min": round(min(vals), 1),
+ "max": round(max(vals), 1),
+ }
+ emit(stats)
+ for k in self._timings:
+ self._timings[k].clear()
+
+ def emit_final(self):
+ if any(self._timings.values()):
+ self._emit()
+
+
+# ───────────────────────────────────────────────────────────────────────────────
+# Config & device
+# ───────────────────────────────────────────────────────────────────────────────
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="SAM2 Annotation Skill")
+ parser.add_argument("--config", type=str)
+ parser.add_argument("--model", type=str, default="sam2-small")
+ parser.add_argument("--device", type=str, default="auto")
+ parser.add_argument("--mock", action="store_true", help="Mock mode — no model, synthetic responses")
+ return parser.parse_args()
+
+
+def load_config(args):
+ import os
+ env_params = os.environ.get("AEGIS_SKILL_PARAMS")
+ if env_params:
+ try:
+ return json.loads(env_params)
+ except json.JSONDecodeError:
+ pass
+ if args.config and Path(args.config).exists():
+ with open(args.config) as f:
+ return json.load(f)
+ return {"model": args.model, "device": args.device}
+
+
+def select_device(pref):
+ if pref != "auto":
+ return pref
+ try:
+ import torch
+ if torch.cuda.is_available():
+ return "cuda"
+ if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+ return "mps"
+ except ImportError:
+ pass
+ return "cpu"
+
+
+# ───────────────────────────────────────────────────────────────────────────────
+# Model config mapping
+# ───────────────────────────────────────────────────────────────────────────────
+
+MODEL_CFG = {
+ "sam2-tiny": "sam2_hiera_t.yaml",
+ "sam2-small": "sam2_hiera_s.yaml",
+ "sam2-base": "sam2_hiera_b+.yaml",
+ "sam2-large": "sam2_hiera_l.yaml",
+}
+
+
+# ───────────────────────────────────────────────────────────────────────────────
+# Main
+# ───────────────────────────────────────────────────────────────────────────────
+
+def main():
+ args = parse_args()
+ config = load_config(args)
+ device = select_device(config.get("device", "auto"))
+ model_name = config.get("model", "sam2-small")
+ perf = PerfTracker()
+
+ mock_mode = args.mock or config.get("mock", False)
+ predictor = None
+
+ if mock_mode:
+ log("Running in MOCK mode — no model loaded, synthetic responses")
+ emit_segmentation("ready", "", {
+ "model": f"{model_name} (mock)",
+ "device": "mock",
+ "available_models": list(MODEL_CFG.keys()),
+ "mock": True,
+ })
+ else:
+ # ── Load model ──
+ emit({"event": "progress", "stage": "init", "message": f"Loading SAM2 ({model_name}) on {device}..."})
+
+ try:
+ import torch
+ import numpy as np
+ import cv2
+ from sam2.build_sam import build_sam2
+ from sam2.sam2_image_predictor import SAM2ImagePredictor
+
+ cfg_file = MODEL_CFG.get(model_name, "sam2_hiera_s.yaml")
+ checkpoint = f"models/{model_name}.pt"
+
+ sam2 = build_sam2(cfg_file, checkpoint)
+ predictor = SAM2ImagePredictor(sam2)
+ predictor.model.to(device)
+
+ emit_segmentation("ready", "", {
+ "model": model_name,
+ "device": device,
+ "available_models": list(MODEL_CFG.keys()),
+ })
+ log(f"Model loaded: {model_name} on {device}")
+ except Exception as e:
+ emit_segmentation("ready", "", error=f"Failed to load SAM2: {e}")
+ emit({"event": "error", "message": f"Failed to load SAM2: {e}", "retriable": False})
+ sys.exit(1)
+
+ # ── State ──
+ current_image = None
+ current_frame_id = None
+ masks_dir = Path(tempfile.mkdtemp(prefix="sam2_masks_"))
+
+ # ── Signal handling ──
+ def handle_signal(signum, frame):
+ sig = "SIGTERM" if signum == signal.SIGTERM else "SIGINT"
+ log(f"Received {sig}, shutting down")
+ perf.emit_final()
+ sys.exit(0)
+ signal.signal(signal.SIGTERM, handle_signal)
+ signal.signal(signal.SIGINT, handle_signal)
+
+ # ── Main stdin loop ──
+ for line in sys.stdin:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ msg = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+
+ cmd = msg.get("command")
+ req_id = msg.get("request_id", "")
+
+ if cmd == "stop":
+ break
+
+ # ── Mock mode: return synthetic responses immediately ──
+ if mock_mode:
+ if cmd == "encode":
+ frame_id = msg.get("frame_id", "mock_frame")
+ current_frame_id = frame_id
+ emit_segmentation("encoded", req_id, {
+ "frame_id": frame_id, "width": 1920, "height": 1080, "encode_ms": 1.0,
+ })
+ log(f"[MOCK] Encoded {frame_id}")
+ elif cmd == "segment":
+ # Generate a small synthetic 100x100 mock mask PNG
+ import io
+ mock_w, mock_h = 100, 80
+ # Create a simple 1-pixel header PNG-like base64 (white rectangle)
+ mock_mask_bytes = bytes([255] * (mock_w * mock_h))
+ mock_b64 = base64.b64encode(mock_mask_bytes).decode()
+ emit_segmentation("segmented", req_id, {
+ "frame_id": current_frame_id or "mock",
+ "mask_path": "/tmp/mock_mask.png",
+ "mask_b64": mock_b64,
+ "score": 0.95,
+ "bbox": [100, 50, 350, 420],
+ "segment_ms": 2.0,
+ "num_masks": 3,
+ })
+ log(f"[MOCK] Segmented")
+ elif cmd == "track":
+ frame_id = msg.get("frame_id", "mock_track")
+ emit_segmentation("tracked", req_id, {
+ "frame_id": frame_id,
+ "mask_path": "/tmp/mock_track.png",
+ "score": 0.92,
+ "bbox": [110, 55, 360, 430],
+ "track_ms": 3.0,
+ })
+ log(f"[MOCK] Tracked {frame_id}")
+ else:
+ log(f"[MOCK] Unknown command: {cmd}")
+ continue
+
+ elif cmd == "encode":
+ # ── Encode: load image and set in predictor ──
+ t0 = time.perf_counter()
+ frame_path = msg.get("frame_path")
+ frame_id = msg.get("frame_id", f"frame_{int(time.time())}")
+
+ if not frame_path or not Path(frame_path).exists():
+ emit_segmentation("encoded", req_id, error=f"Frame not found: {frame_path}")
+ continue
+
+ try:
+ img = cv2.imread(frame_path)
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+ predictor.set_image(img)
+ current_image = img
+ current_frame_id = frame_id
+
+ ms = (time.perf_counter() - t0) * 1000
+ perf.record("encode", ms)
+ perf.total_encodes += 1
+ perf.tick()
+
+ emit_segmentation("encoded", req_id, {
+ "frame_id": frame_id,
+ "width": img.shape[1],
+ "height": img.shape[0],
+ "encode_ms": round(ms, 1),
+ })
+ log(f"Encoded frame {frame_id} ({img.shape[1]}x{img.shape[0]}) in {ms:.0f}ms")
+ except Exception as e:
+ emit_segmentation("encoded", req_id, error=f"Encode error: {e}")
+
+ elif cmd == "segment":
+ # ── Segment: run point/box prompts to get masks ──
+ t0 = time.perf_counter()
+ if current_image is None:
+ emit_segmentation("segmented", req_id, error="No image encoded — send encode first")
+ continue
+
+ try:
+ points_raw = msg.get("points", [])
+ boxes_raw = msg.get("boxes", [])
+
+ point_coords = None
+ point_labels = None
+ input_box = None
+
+ if points_raw:
+ point_coords = np.array([[p["x"], p["y"]] for p in points_raw])
+ point_labels = np.array([p.get("label", 1) for p in points_raw])
+
+ if boxes_raw:
+ b = boxes_raw[0]
+ input_box = np.array([b["x1"], b["y1"], b["x2"], b["y2"]])
+
+ masks, scores, logits = predictor.predict(
+ point_coords=point_coords,
+ point_labels=point_labels,
+ box=input_box,
+ multimask_output=True,
+ )
+
+ # Use best mask
+ best_idx = np.argmax(scores)
+ mask = masks[best_idx]
+ score = float(scores[best_idx])
+
+ # Save mask as PNG
+ mask_filename = f"mask_{current_frame_id}_{int(time.time()*1000)}.png"
+ mask_path = str(masks_dir / mask_filename)
+ cv2.imwrite(mask_path, (mask * 255).astype(np.uint8))
+
+ # Compute bbox from mask
+ ys, xs = np.where(mask)
+ bbox = [int(xs.min()), int(ys.min()), int(xs.max()), int(ys.max())] if len(xs) > 0 else [0, 0, 0, 0]
+
+ ms = (time.perf_counter() - t0) * 1000
+ perf.record("segment", ms)
+ perf.total_segments += 1
+ perf.tick()
+
+ # Encode mask as base64 for frontend canvas rendering
+ mask_png = cv2.imencode('.png', (mask * 255).astype(np.uint8))[1]
+ mask_b64 = base64.b64encode(mask_png.tobytes()).decode()
+
+ emit_segmentation("segmented", req_id, {
+ "frame_id": current_frame_id,
+ "mask_path": mask_path,
+ "mask_b64": mask_b64,
+ "score": round(score, 3),
+ "bbox": bbox,
+ "segment_ms": round(ms, 1),
+ "num_masks": len(masks),
+ })
+ log(f"Segmented frame {current_frame_id}: score={score:.3f} bbox={bbox} in {ms:.0f}ms")
+ except Exception as e:
+ emit_segmentation("segmented", req_id, error=f"Segment error: {e}")
+
+ elif cmd == "track":
+ # ── Track: encode a new frame and propagate the last mask ──
+ t0 = time.perf_counter()
+ frame_path = msg.get("frame_path")
+ frame_id = msg.get("frame_id", f"track_{int(time.time())}")
+
+ if not frame_path or not Path(frame_path).exists():
+ emit_segmentation("tracked", req_id, error=f"Frame not found: {frame_path}")
+ continue
+
+ try:
+ img = cv2.imread(frame_path)
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+ predictor.set_image(img)
+ current_image = img
+ current_frame_id = frame_id
+
+ # Re-predict with same prompts (simple propagation)
+ # For full video tracking, SAM2VideoPredictor is needed
+ masks, scores, _ = predictor.predict(
+ point_coords=None,
+ point_labels=None,
+ multimask_output=True,
+ )
+
+ best_idx = np.argmax(scores)
+ mask = masks[best_idx]
+ score = float(scores[best_idx])
+
+ mask_filename = f"track_{frame_id}_{int(time.time()*1000)}.png"
+ mask_path = str(masks_dir / mask_filename)
+ cv2.imwrite(mask_path, (mask * 255).astype(np.uint8))
+
+ ys, xs = np.where(mask)
+ bbox = [int(xs.min()), int(ys.min()), int(xs.max()), int(ys.max())] if len(xs) > 0 else [0, 0, 0, 0]
+
+ ms = (time.perf_counter() - t0) * 1000
+ perf.record("track", ms)
+ perf.total_tracks += 1
+ perf.tick()
+
+ emit_segmentation("tracked", req_id, {
+ "frame_id": frame_id,
+ "mask_path": mask_path,
+ "score": round(score, 3),
+ "bbox": bbox,
+ "track_ms": round(ms, 1),
+ })
+ log(f"Tracked frame {frame_id}: score={score:.3f} in {ms:.0f}ms")
+ except Exception as e:
+ emit_segmentation("tracked", req_id, error=f"Track error: {e}")
+
+ else:
+ # Unknown command — echo back for debugging
+ log(f"Unknown command: {cmd}")
+
+ perf.emit_final()
+ log("Skill exiting cleanly")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/skills/transformation/depth-estimation/config.yaml b/skills/transformation/depth-estimation/config.yaml
new file mode 100644
index 00000000..e100e54b
--- /dev/null
+++ b/skills/transformation/depth-estimation/config.yaml
@@ -0,0 +1,72 @@
+# Depth Estimation Skill — Configuration Schema
+# Parsed by Aegis skill-registry-service.cjs → parseConfigYaml()
+# Format: params[] with key, type, label, default, description, options
+
+params:
+ - key: auto_start
+ label: Auto Start
+ type: boolean
+ default: true
+ description: "Start this skill automatically when Aegis launches"
+
+ - key: model
+ label: Depth Model
+ type: select
+ default: depth-anything-v2-small
+ description: "Depth Anything v2 model size — larger = more accurate but slower"
+ options:
+ - { value: depth-anything-v2-small, label: "Small (fastest)" }
+ - { value: depth-anything-v2-base, label: "Base (balanced)" }
+ - { value: depth-anything-v2-large, label: "Large (most accurate)" }
+
+ - key: variant
+ label: CoreML Variant (macOS)
+ type: select
+ default: DepthAnythingV2SmallF16
+ description: "CoreML model format — F16 recommended for Apple Neural Engine"
+ options:
+ - { value: DepthAnythingV2SmallF16, label: "Small F16 (recommended)" }
+ - { value: DepthAnythingV2SmallF16INT8, label: "Small F16+INT8 (faster)" }
+ - { value: DepthAnythingV2SmallF32, label: "Small F32 (highest precision)" }
+
+ - key: blend_mode
+ label: Display Mode
+ type: select
+ default: depth_only
+ description: "How the depth map is displayed over the camera feed"
+ options:
+ - { value: depth_only, label: "Depth Only (privacy)" }
+ - { value: overlay, label: "Overlay (semi-transparent)" }
+ - { value: side_by_side, label: "Side-by-Side" }
+
+ - key: opacity
+ label: Overlay Opacity
+ type: number
+ default: 0.5
+ description: "Overlay transparency when using overlay blend mode (0.0–1.0)"
+
+ - key: colormap
+ label: Depth Colormap
+ type: select
+ default: viridis
+ description: "Color scheme for depth visualization"
+ options:
+ - { value: inferno, label: "Inferno (warm)" }
+ - { value: viridis, label: "Viridis (green-blue)" }
+ - { value: plasma, label: "Plasma (purple-yellow)" }
+ - { value: magma, label: "Magma (dark-hot)" }
+ - { value: jet, label: "Jet (rainbow)" }
+ - { value: turbo, label: "Turbo (improved rainbow)" }
+ - { value: hot, label: "Hot (black-red-yellow)" }
+ - { value: cool, label: "Cool (cyan-magenta)" }
+
+ - key: device
+ label: Inference Device
+ type: select
+ default: auto
+ description: "Compute backend for inference"
+ options:
+ - { value: auto, label: "Auto-detect" }
+ - { value: cpu, label: "CPU" }
+ - { value: cuda, label: "NVIDIA CUDA" }
+ - { value: mps, label: "Apple Silicon (MPS)" }
diff --git a/skills/transformation/depth-estimation/deploy.bat b/skills/transformation/depth-estimation/deploy.bat
new file mode 100644
index 00000000..679c2d07
--- /dev/null
+++ b/skills/transformation/depth-estimation/deploy.bat
@@ -0,0 +1,130 @@
+@echo off
+setlocal enabledelayedexpansion
+REM ═══════════════════════════════════════════════════════════════════
+REM Depth Estimation Skill — Windows Deployment (ONNX Runtime)
+REM
+REM GPU detection cascade:
+REM 1. nvidia-smi found → onnxruntime-gpu (CUDA + TensorRT EPs)
+REM 2. Non-NVIDIA GPU found (WMI) → onnxruntime-directml
+REM 3. No GPU → onnxruntime (CPU)
+REM
+REM Then downloads ONNX model from HuggingFace.
+REM ═══════════════════════════════════════════════════════════════════
+
+echo [DepthDeploy] Starting depth-estimation skill deployment...
+echo [DepthDeploy] Platform: Windows (%PROCESSOR_ARCHITECTURE%)
+
+REM ── 1. Find Python ─────────────────────────────────────────────────
+set "PYTHON_CMD="
+
+REM Try py launcher first (most reliable on Windows)
+py --version >nul 2>&1
+if %ERRORLEVEL% equ 0 (
+ set "PYTHON_CMD=py"
+ goto :found_python
+)
+
+REM Try python (could be Python 3 on PATH)
+python --version >nul 2>&1
+if %ERRORLEVEL% equ 0 (
+ set "PYTHON_CMD=python"
+ goto :found_python
+)
+
+echo [DepthDeploy] ERROR: Python not found. Install Python 3.9+ from python.org
+exit /b 1
+
+:found_python
+echo [DepthDeploy] Using Python: %PYTHON_CMD%
+%PYTHON_CMD% --version
+
+REM ── 2. Create venv ─────────────────────────────────────────────────
+if not exist ".venv\Scripts\python.exe" (
+ echo [DepthDeploy] Creating virtual environment...
+ %PYTHON_CMD% -m venv .venv
+ if %ERRORLEVEL% neq 0 (
+ echo [DepthDeploy] ERROR: Failed to create venv
+ exit /b 1
+ )
+)
+
+set "VENV_PIP=.venv\Scripts\pip.exe"
+set "VENV_PYTHON=.venv\Scripts\python.exe"
+
+echo [DepthDeploy] Upgrading pip...
+%VENV_PYTHON% -m pip install --upgrade pip >nul 2>&1
+
+REM ── 3. Detect GPU ──────────────────────────────────────────────────
+echo [DepthDeploy] Detecting GPU hardware...
+
+set "GPU_BACKEND=cpu"
+set "REQUIREMENTS_FILE=requirements_cpu.txt"
+
+REM 3a. Check for NVIDIA GPU via nvidia-smi
+nvidia-smi --query-gpu=name --format=csv,noheader,nounits >nul 2>&1
+if %ERRORLEVEL% equ 0 (
+ echo [DepthDeploy] NVIDIA GPU detected:
+ nvidia-smi --query-gpu=name,memory.total --format=csv,noheader,nounits
+ set "GPU_BACKEND=cuda"
+ set "REQUIREMENTS_FILE=requirements_cuda.txt"
+ goto :gpu_detected
+)
+
+REM 3b. Check for any GPU via WMI (AMD, Intel, Qualcomm)
+for /f "tokens=*" %%G in ('powershell -NoProfile -Command "Get-CimInstance Win32_VideoController | Where-Object { $_.Name -notlike '*Microsoft*' -and $_.Name -notlike '*Remote*' } | Select-Object -ExpandProperty Name" 2^>nul') do (
+ echo [DepthDeploy] GPU found: %%G
+ set "GPU_BACKEND=directml"
+ set "REQUIREMENTS_FILE=requirements_directml.txt"
+)
+
+:gpu_detected
+echo [DepthDeploy] Selected backend: %GPU_BACKEND%
+echo [DepthDeploy] Requirements: %REQUIREMENTS_FILE%
+
+REM ── 4. Install dependencies ────────────────────────────────────────
+if not exist "%REQUIREMENTS_FILE%" (
+ echo [DepthDeploy] WARNING: %REQUIREMENTS_FILE% not found, falling back to requirements_cpu.txt
+ set "REQUIREMENTS_FILE=requirements_cpu.txt"
+)
+
+echo [DepthDeploy] Installing %REQUIREMENTS_FILE%...
+%VENV_PIP% install -r %REQUIREMENTS_FILE%
+if %ERRORLEVEL% neq 0 (
+ echo [DepthDeploy] WARNING: Install failed for %REQUIREMENTS_FILE%
+ if not "%GPU_BACKEND%"=="cpu" (
+ echo [DepthDeploy] Falling back to CPU requirements...
+ %VENV_PIP% install -r requirements_cpu.txt
+ )
+)
+
+REM ── 5. Download ONNX model ─────────────────────────────────────────
+echo [DepthDeploy] Downloading ONNX model from HuggingFace...
+
+set "MODELS_DIR=%USERPROFILE%\.aegis-ai\models\feature-extraction"
+if not exist "%MODELS_DIR%" mkdir "%MODELS_DIR%"
+
+if exist "%MODELS_DIR%\model.onnx" (
+ echo [DepthDeploy] ONNX model already exists at %MODELS_DIR%\model.onnx
+) else (
+ %VENV_PYTHON% -c "from huggingface_hub import hf_hub_download; import shutil, os; p = hf_hub_download('onnx-community/depth-anything-v2-small', 'onnx/model.onnx'); d = os.path.join(os.path.expanduser('~'), '.aegis-ai', 'models', 'feature-extraction', 'model.onnx'); shutil.copy2(p, d); print(f'[DepthDeploy] Model copied to {d}')"
+ if %ERRORLEVEL% equ 0 (
+ echo [DepthDeploy] ONNX model downloaded successfully
+ ) else (
+ echo [DepthDeploy] WARNING: Model download failed — will retry on first run
+ )
+)
+
+REM ── 6. Verify installation ─────────────────────────────────────────
+echo [DepthDeploy] Verifying ONNX Runtime installation...
+
+%VENV_PYTHON% -c "import onnxruntime as ort; eps = ort.get_available_providers(); print(f'[DepthDeploy] Available EPs: {eps}')"
+if %ERRORLEVEL% neq 0 (
+ echo [DepthDeploy] ERROR: ONNX Runtime import failed
+ exit /b 1
+)
+
+REM Log detected execution providers
+%VENV_PYTHON% -c "import onnxruntime as ort; eps = ort.get_available_providers(); cuda = 'CUDAExecutionProvider' in eps; trt = 'TensorrtExecutionProvider' in eps; dml = 'DmlExecutionProvider' in eps; print(f'[DepthDeploy] CUDA EP: {cuda}, TensorRT EP: {trt}, DirectML EP: {dml}')"
+
+echo [DepthDeploy] Deployment complete (%GPU_BACKEND% backend)
+exit /b 0
diff --git a/skills/transformation/depth-estimation/deploy.sh b/skills/transformation/depth-estimation/deploy.sh
index abfb23af..86a0e4fe 100755
--- a/skills/transformation/depth-estimation/deploy.sh
+++ b/skills/transformation/depth-estimation/deploy.sh
@@ -1,39 +1,136 @@
#!/bin/bash
-# deploy.sh — Platform-aware dependency install for Depth Estimation
+# deploy.sh — Zero-assumption bootstrapper for Depth Estimation Skill
#
-# macOS: CoreML only (fast ~10s install, Neural Engine inference)
-# Other: Full PyTorch stack (torch + torchvision + depth-anything-v2)
+# Probes the system for Python, GPU backends, and installs the minimum
+# viable stack. Called by Aegis skill-runtime-manager during installation.
#
-# The Aegis deployment agent calls this instead of raw pip install.
+# Uses skills/lib/env_config.py for hardware detection.
+#
+# Exit codes:
+# 0 = success
+# 1 = fatal error (no Python found)
+# 2 = partial success (CPU-only fallback)
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
VENV_DIR="$SCRIPT_DIR/.venv"
+LIB_DIR="$(cd "$SCRIPT_DIR/../../lib" 2>/dev/null && pwd || echo "")"
MODELS_DIR="$HOME/.aegis-ai/models/feature-extraction"
-COREML_VARIANT="DepthAnythingV2SmallF16"
-COREML_HF_REPO="apple/coreml-depth-anything-v2-small"
+LOG_PREFIX="[Depth-deploy]"
+
+log() { echo "$LOG_PREFIX $*" >&2; }
+emit() { echo "$1"; } # JSON to stdout for Aegis to parse
+
+# ─── Step 1: Find Python ────────────────────────────────────────────────────
+
+find_python() {
+ for cmd in python3.12 python3.11 python3.10 python3.9 python3; do
+ if command -v "$cmd" &>/dev/null; then
+ local ver
+ ver="$("$cmd" --version 2>&1 | grep -oE '[0-9]+\.[0-9]+')"
+ local major minor
+ major=$(echo "$ver" | cut -d. -f1)
+ minor=$(echo "$ver" | cut -d. -f2)
+ if [ "$major" -ge 3 ] && [ "$minor" -ge 9 ]; then
+ echo "$cmd"
+ return 0
+ fi
+ fi
+ done
+ return 1
+}
+
+PYTHON_CMD=$(find_python) || {
+ log "ERROR: No Python >=3.9 found. Install Python 3.9+ and retry."
+ emit '{"event": "error", "stage": "python", "message": "No Python >=3.9 found"}'
+ exit 1
+}
-echo "=== Depth Estimation (Privacy) — Setup ==="
-echo "Platform: $(uname -s) / $(uname -m)"
+log "Using Python: $PYTHON_CMD ($($PYTHON_CMD --version 2>&1))"
+emit "{\"event\": \"progress\", \"stage\": \"python\", \"message\": \"Found $($PYTHON_CMD --version 2>&1)\"}"
+
+# ─── Step 2: Create virtual environment ─────────────────────────────────────
-# ── Create venv ──────────────────────────────────────────────────────
if [ ! -d "$VENV_DIR" ]; then
- echo "Creating virtual environment..."
- python3 -m venv "$VENV_DIR"
+ log "Creating virtual environment..."
+ "$PYTHON_CMD" -m venv "$VENV_DIR"
fi
PIP="$VENV_DIR/bin/pip"
-PYTHON="$VENV_DIR/bin/python"
+VPYTHON="$VENV_DIR/bin/python"
+
+"$PIP" install --upgrade pip -q 2>/dev/null || true
+
+emit '{"event": "progress", "stage": "venv", "message": "Virtual environment ready"}'
+
+# ─── Step 2.5: Bundle env_config.py alongside transform.py ──────────────────
+
+if [ -n "$LIB_DIR" ] && [ -f "$LIB_DIR/env_config.py" ]; then
+ cp "$LIB_DIR/env_config.py" "$SCRIPT_DIR/scripts/env_config.py"
+ log "Bundled env_config.py into scripts/"
+fi
+
+# ─── Step 3: Detect hardware via env_config ──────────────────────────────────
+
+BACKEND="cpu"
+
+# Find env_config.py — bundled copy or repo lib/
+ENV_CONFIG_DIR=""
+if [ -f "$SCRIPT_DIR/scripts/env_config.py" ]; then
+ ENV_CONFIG_DIR="$SCRIPT_DIR/scripts"
+elif [ -n "$LIB_DIR" ] && [ -f "$LIB_DIR/env_config.py" ]; then
+ ENV_CONFIG_DIR="$LIB_DIR"
+fi
+
+if [ -n "$ENV_CONFIG_DIR" ]; then
+ log "Detecting hardware via env_config.py..."
+ DETECT_OUTPUT=$("$VPYTHON" -c "
+import sys
+sys.path.insert(0, '$ENV_CONFIG_DIR')
+from env_config import HardwareEnv
+env = HardwareEnv.detect()
+print(env.backend)
+" 2>&1) || true
+
+ # The last line of output is the backend name
+ BACKEND=$(echo "$DETECT_OUTPUT" | tail -1)
+
+ # Validate backend value
+ case "$BACKEND" in
+ cuda|rocm|mps|intel|cpu) ;;
+ *)
+ log "env_config returned unexpected backend '$BACKEND', falling back to cpu"
+ BACKEND="cpu"
+ ;;
+ esac
+
+ log "env_config detected backend: $BACKEND"
+else
+ log "env_config.py not found, using heuristic detection..."
-# Upgrade pip
-"$PIP" install --upgrade pip --quiet
+ # Fallback: inline GPU detection
+ if command -v nvidia-smi &>/dev/null; then
+ cuda_ver=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1)
+ if [ -n "$cuda_ver" ]; then
+ BACKEND="cuda"
+ log "Detected NVIDIA GPU (driver: $cuda_ver)"
+ fi
+ elif [ "$(uname)" = "Darwin" ] && [ "$(uname -m)" = "arm64" ]; then
+ BACKEND="mps"
+ log "Detected Apple Silicon (MPS)"
+ fi
+fi
+
+emit "{\"event\": \"progress\", \"stage\": \"gpu\", \"backend\": \"$BACKEND\", \"message\": \"Compute backend: $BACKEND\"}"
+
+# ─── Step 4: Install requirements ────────────────────────────────────────────
-# ── Platform detection ───────────────────────────────────────────────
if [ "$(uname -s)" = "Darwin" ]; then
- echo ""
- echo "=== macOS detected — CoreML backend (Neural Engine) ==="
- echo "Installing CoreML dependencies only (fast)..."
+ # macOS: CoreML backend — lightweight install
+ log "macOS detected — installing CoreML + common dependencies"
+ emit '{"event": "progress", "stage": "install", "message": "Installing CoreML dependencies..."}'
+
"$PIP" install --quiet \
"coremltools>=8.0" \
"huggingface_hub>=0.20.0" \
@@ -42,50 +139,75 @@ if [ "$(uname -s)" = "Darwin" ]; then
"Pillow>=10.0.0" \
"matplotlib>=3.7.0"
- echo "✅ CoreML dependencies installed"
+ log "CoreML dependencies installed"
- # ── Download CoreML model if not present ─────────────────────────
+ # Download CoreML model if not present
+ COREML_VARIANT="DepthAnythingV2SmallF16"
+ COREML_HF_REPO="apple/coreml-depth-anything-v2-small"
MODEL_PATH="$MODELS_DIR/$COREML_VARIANT.mlpackage"
+
if [ -d "$MODEL_PATH" ]; then
- echo "✅ CoreML model already present: $MODEL_PATH"
+ log "CoreML model already present: $MODEL_PATH"
else
- echo "Downloading CoreML model: $COREML_VARIANT from $COREML_HF_REPO..."
+ log "Downloading CoreML model: $COREML_VARIANT from $COREML_HF_REPO..."
mkdir -p "$MODELS_DIR"
- "$PYTHON" -c "
+ "$VPYTHON" -c "
from huggingface_hub import snapshot_download
snapshot_download(
'$COREML_HF_REPO',
local_dir='$MODELS_DIR',
allow_patterns=['$COREML_VARIANT.mlpackage/**'],
)
-print('✅ CoreML model downloaded')
+print('CoreML model downloaded')
"
fi
+else
+ # Non-macOS: use per-backend requirements files
+ REQ_FILE="$SCRIPT_DIR/requirements_${BACKEND}.txt"
+
+ if [ ! -f "$REQ_FILE" ]; then
+ log "WARNING: $REQ_FILE not found, falling back to CPU"
+ REQ_FILE="$SCRIPT_DIR/requirements_cpu.txt"
+ BACKEND="cpu"
+ fi
+
+ log "Installing dependencies from $REQ_FILE ..."
+ emit "{\"event\": \"progress\", \"stage\": \"install\", \"message\": \"Installing $BACKEND dependencies...\"}"
- # Verify
- "$PYTHON" -c "
+ "$PIP" install -r "$REQ_FILE" -q 2>&1 | tail -5 >&2
+fi
+
+# ─── Step 5: Verify installation ────────────────────────────────────────────
+
+log "Verifying installation..."
+
+if [ "$(uname -s)" = "Darwin" ]; then
+ "$VPYTHON" -c "
import coremltools, cv2, numpy, PIL
from pathlib import Path
-model_path = Path('$MODEL_PATH')
-assert model_path.exists(), f'Model not found: {model_path}'
-print(f'✅ Verified: coremltools={coremltools.__version__}, model={model_path.name}')
+model_path = Path('$MODEL_PATH') if '${MODEL_PATH:-}' else None
+if model_path and model_path.exists():
+ print(f'Verified: coremltools={coremltools.__version__}, model={model_path.name}')
+else:
+ print(f'Verified: coremltools={coremltools.__version__} (no model downloaded yet)')
"
-
else
- echo ""
- echo "=== Non-macOS — PyTorch backend ==="
- echo "Installing full PyTorch dependencies..."
- "$PIP" install --quiet -r "$SCRIPT_DIR/requirements.txt"
-
- echo "✅ PyTorch dependencies installed"
-
- # Verify
- "$PYTHON" -c "
+ if [ -n "$ENV_CONFIG_DIR" ]; then
+ "$VPYTHON" -c "
+import sys, json
+sys.path.insert(0, '$ENV_CONFIG_DIR')
+from env_config import HardwareEnv
+env = HardwareEnv.detect()
+print(json.dumps(env.to_dict(), indent=2))
+" 2>&1 | while read -r line; do log "$line"; done
+ else
+ "$VPYTHON" -c "
import torch, cv2, numpy, PIL
from depth_anything_v2.dpt import DepthAnythingV2
-print(f'✅ Verified: torch={torch.__version__}, CUDA={torch.cuda.is_available()}')
+print(f'Verified: torch={torch.__version__}, CUDA={torch.cuda.is_available()}')
"
+ fi
fi
-echo ""
-echo "=== Setup complete ==="
+emit "{\"event\": \"complete\", \"backend\": \"$BACKEND\", \"message\": \"Depth Estimation skill installed ($BACKEND backend)\"}"
+log "Done! Backend: $BACKEND"
diff --git a/skills/transformation/depth-estimation/models.json b/skills/transformation/depth-estimation/models.json
index 27ee043f..bde60dd8 100644
--- a/skills/transformation/depth-estimation/models.json
+++ b/skills/transformation/depth-estimation/models.json
@@ -59,24 +59,34 @@
}
},
"linux": {
- "repository": "depth-anything/Depth-Anything-V2-Small",
- "format": "pth",
+ "repository": "onnx-community/depth-anything-v2-small",
+ "format": "onnx",
"variants": {
- "depth_anything_v2_vits": {
+ "model": {
"precision": "float32",
- "size_mb": 99.0,
- "description": "PyTorch ViT-S — CUDA/CPU"
+ "size_mb": 98.0,
+ "description": "ONNX — CUDA/TensorRT/CPU"
+ },
+ "model_quantized": {
+ "precision": "int8",
+ "size_mb": 25.0,
+ "description": "ONNX INT8 quantized — smallest, fastest"
}
}
},
"win32": {
- "repository": "depth-anything/Depth-Anything-V2-Small",
- "format": "pth",
+ "repository": "onnx-community/depth-anything-v2-small",
+ "format": "onnx",
"variants": {
- "depth_anything_v2_vits": {
+ "model": {
"precision": "float32",
- "size_mb": 99.0,
- "description": "PyTorch ViT-S — CUDA/CPU"
+ "size_mb": 98.0,
+ "description": "ONNX — CUDA/TensorRT/DirectML/CPU"
+ },
+ "model_quantized": {
+ "precision": "int8",
+ "size_mb": 25.0,
+ "description": "ONNX INT8 quantized — smallest, fastest"
}
}
}
@@ -89,24 +99,24 @@
"input_size": [518, 392],
"platforms": {
"linux": {
- "repository": "depth-anything/Depth-Anything-V2-Base",
- "format": "pth",
+ "repository": "onnx-community/depth-anything-v2-base",
+ "format": "onnx",
"variants": {
- "depth_anything_v2_vitb": {
+ "model": {
"precision": "float32",
"size_mb": 390.0,
- "description": "PyTorch ViT-B — CUDA/CPU"
+ "description": "ONNX — CUDA/TensorRT/CPU"
}
}
},
"win32": {
- "repository": "depth-anything/Depth-Anything-V2-Base",
- "format": "pth",
+ "repository": "onnx-community/depth-anything-v2-base",
+ "format": "onnx",
"variants": {
- "depth_anything_v2_vitb": {
+ "model": {
"precision": "float32",
"size_mb": 390.0,
- "description": "PyTorch ViT-B — CUDA/CPU"
+ "description": "ONNX — CUDA/TensorRT/DirectML/CPU"
}
}
}
@@ -119,24 +129,24 @@
"input_size": [518, 392],
"platforms": {
"linux": {
- "repository": "depth-anything/Depth-Anything-V2-Large",
- "format": "pth",
+ "repository": "onnx-community/depth-anything-v2-large",
+ "format": "onnx",
"variants": {
- "depth_anything_v2_vitl": {
+ "model": {
"precision": "float32",
"size_mb": 1280.0,
- "description": "PyTorch ViT-L — CUDA recommended"
+ "description": "ONNX — CUDA/TensorRT/CPU"
}
}
},
"win32": {
- "repository": "depth-anything/Depth-Anything-V2-Large",
- "format": "pth",
+ "repository": "onnx-community/depth-anything-v2-large",
+ "format": "onnx",
"variants": {
- "depth_anything_v2_vitl": {
+ "model": {
"precision": "float32",
"size_mb": 1280.0,
- "description": "PyTorch ViT-L — CUDA recommended"
+ "description": "ONNX — CUDA/TensorRT/DirectML/CPU"
}
}
}
diff --git a/skills/transformation/depth-estimation/requirements.txt b/skills/transformation/depth-estimation/requirements.txt
index 2717a006..7ee3a71e 100644
--- a/skills/transformation/depth-estimation/requirements.txt
+++ b/skills/transformation/depth-estimation/requirements.txt
@@ -20,3 +20,8 @@ numpy>=1.24.0
opencv-python-headless>=4.8.0
Pillow>=10.0.0
matplotlib>=3.7.0
+
+# ── TensorRT (optional, Windows/Linux NVIDIA) ────────────────────────
+# If available, transform.py auto-selects TRT FP16 for ~7x speedup.
+# Falls back to PyTorch CUDA if not installed.
+tensorrt>=10.0; sys_platform != "darwin"
diff --git a/skills/transformation/depth-estimation/requirements_cpu.txt b/skills/transformation/depth-estimation/requirements_cpu.txt
new file mode 100644
index 00000000..b95bf39d
--- /dev/null
+++ b/skills/transformation/depth-estimation/requirements_cpu.txt
@@ -0,0 +1,13 @@
+# Depth Estimation — ONNX Runtime CPU-only
+# Installed by deploy.bat when no GPU is detected.
+#
+# Smallest install footprint. No GPU acceleration.
+
+onnxruntime>=1.17.0
+
+# ── Common dependencies ─────────────────────────────────────────────
+huggingface_hub>=0.20.0
+numpy>=1.24.0
+opencv-python-headless>=4.8.0
+Pillow>=10.0.0
+matplotlib>=3.7.0
diff --git a/skills/transformation/depth-estimation/requirements_cuda.txt b/skills/transformation/depth-estimation/requirements_cuda.txt
new file mode 100644
index 00000000..b8d305ae
--- /dev/null
+++ b/skills/transformation/depth-estimation/requirements_cuda.txt
@@ -0,0 +1,14 @@
+# Depth Estimation — ONNX Runtime with CUDA Execution Provider (NVIDIA GPUs)
+# Installed by deploy.bat when nvidia-smi is detected.
+#
+# onnxruntime-gpu includes both CUDA and TensorRT execution providers.
+
+onnxruntime-gpu>=1.17.0
+nvidia-cudnn-cu12>=9.0
+
+# ── Common dependencies ─────────────────────────────────────────────
+huggingface_hub>=0.20.0
+numpy>=1.24.0
+opencv-python-headless>=4.8.0
+Pillow>=10.0.0
+matplotlib>=3.7.0
diff --git a/skills/transformation/depth-estimation/requirements_directml.txt b/skills/transformation/depth-estimation/requirements_directml.txt
new file mode 100644
index 00000000..525a5f22
--- /dev/null
+++ b/skills/transformation/depth-estimation/requirements_directml.txt
@@ -0,0 +1,13 @@
+# Depth Estimation — ONNX Runtime with DirectML Execution Provider
+# Installed by deploy.bat when AMD/Intel GPU detected (no NVIDIA).
+#
+# DirectML provides GPU acceleration for AMD, Intel, and Qualcomm GPUs on Windows.
+
+onnxruntime-directml>=1.17.0
+
+# ── Common dependencies ─────────────────────────────────────────────
+huggingface_hub>=0.20.0
+numpy>=1.24.0
+opencv-python-headless>=4.8.0
+Pillow>=10.0.0
+matplotlib>=3.7.0
diff --git a/skills/transformation/depth-estimation/scripts/benchmark.py b/skills/transformation/depth-estimation/scripts/benchmark.py
new file mode 100644
index 00000000..8aeb6a32
--- /dev/null
+++ b/skills/transformation/depth-estimation/scripts/benchmark.py
@@ -0,0 +1,306 @@
+#!/usr/bin/env python3
+"""
+Cross-platform depth estimation benchmark — spawned by Aegis IPC handler.
+
+Supports all backends:
+ macOS → CoreML (Neural Engine)
+ Win/Linux (NVIDIA) → TensorRT FP16 → PyTorch CUDA
+ Any → PyTorch CPU fallback
+
+Usage:
+ python benchmark.py --variant DepthAnythingV2SmallF16 --runs 10 --colormap viridis
+ python benchmark.py --model depth-anything-v2-small --runs 10
+
+Outputs JSONL progress events and a final result event to stdout.
+Progress events: {"event": "progress", "stage": "...", "message": "..."}
+Final result: {"event": "result", ...benchmark data...}
+"""
+
+import sys
+import json
+import time
+import os
+import argparse
+import platform
+import tempfile
+from pathlib import Path
+
+# Import the skill class from the same directory
+_script_dir = Path(__file__).resolve().parent
+sys.path.insert(0, str(_script_dir))
+
+
+MODELS_DIR = Path.home() / ".aegis-ai" / "models" / "feature-extraction"
+
+COLORMAP_MAP = {
+ "inferno": 1, "viridis": 16, "plasma": 13, "magma": 12,
+ "jet": 2, "turbo": 18, "hot": 11, "cool": 8,
+}
+
+COMPUTE_UNIT_MAP = {
+ "all": "ALL",
+ "cpu": "CPU_ONLY",
+ "gpu": "CPU_AND_GPU",
+ "cpu_npu": "CPU_AND_NE",
+ "npu": "ALL",
+}
+
+
+def _log(msg):
+ print(f"[DepthBenchmark] {msg}", file=sys.stderr, flush=True)
+
+
+def _emit(event: dict):
+ """Emit a JSONL event to stdout for the Electron handler to parse."""
+ print(json.dumps(event), flush=True)
+
+
+def download_test_image(url):
+ """Download a test image from URL, return numpy BGR array."""
+ import cv2
+ import numpy as np
+ import urllib.request
+
+ _emit({"event": "progress", "stage": "download", "message": f"Downloading test image..."})
+ _log(f"Downloading test image: {url}")
+ tmp_path = os.path.join(tempfile.gettempdir(), "aegis_depth_bench_test.jpg")
+
+ try:
+ urllib.request.urlretrieve(url, tmp_path)
+ img = cv2.imread(tmp_path)
+ if img is not None:
+ return img
+ except Exception as e:
+ _log(f"Download failed: {e}")
+
+ # Fallback: generate a synthetic test image
+ _log("Using synthetic test image (640x480 gradient)")
+ return np.random.randint(0, 255, (480, 640, 3), dtype=np.uint8)
+
+
+# ── CoreML benchmark (macOS only) ───────────────────────────────────────────
+
+def run_coreml_benchmark(args, test_image):
+ """Run CoreML benchmark (macOS only). Mirrors legacy benchmark_coreml.py."""
+ import cv2
+ import numpy as np
+ import coremltools as ct
+ from PIL import Image
+
+ COREML_INPUT_SIZE = (518, 392) # width, height
+
+ variant_id = args.variant
+ model_path = MODELS_DIR / f"{variant_id}.mlpackage"
+
+ if not model_path.exists():
+ return {"error": f"CoreML model not found: {model_path}"}
+
+ # Load model
+ _emit({"event": "progress", "stage": "model", "message": f"Loading CoreML model: {variant_id}..."})
+ _log(f"Loading CoreML model: {variant_id}")
+ compute_unit_key = COMPUTE_UNIT_MAP.get(args.compute_units, "ALL")
+ compute_unit = getattr(ct.ComputeUnit, compute_unit_key, ct.ComputeUnit.ALL)
+
+ t0 = time.perf_counter()
+ model = ct.models.MLModel(str(model_path), compute_units=compute_unit)
+ load_time_ms = (time.perf_counter() - t0) * 1000
+ _log(f"Model loaded in {load_time_ms:.0f}ms (compute_units={compute_unit_key})")
+
+ original_h, original_w = test_image.shape[:2]
+ input_w, input_h = COREML_INPUT_SIZE
+
+ # Prepare input
+ rgb = cv2.cvtColor(test_image, cv2.COLOR_BGR2RGB)
+ resized = cv2.resize(rgb, (input_w, input_h), interpolation=cv2.INTER_LINEAR)
+ pil_image = Image.fromarray(resized, mode="RGB")
+
+ colormap_id = COLORMAP_MAP.get(args.colormap, 16)
+
+ # Warm-up run
+ _emit({"event": "progress", "stage": "warmup", "message": "Warm-up inference..."})
+ _log("Warm-up inference...")
+ model.predict({"image": pil_image})
+
+ # Benchmark runs
+ _emit({"event": "progress", "stage": "benchmark", "message": f"Running {args.runs} iterations...", "total": args.runs})
+ _log(f"Running {args.runs} benchmark iterations...")
+ times = []
+ last_depth_colored = None
+
+ for i in range(args.runs):
+ t0 = time.perf_counter()
+ prediction = model.predict({"image": pil_image})
+ elapsed_ms = (time.perf_counter() - t0) * 1000
+ times.append(elapsed_ms)
+ _emit({"event": "progress", "stage": "run", "run": i + 1, "total": args.runs,
+ "time_ms": round(elapsed_ms, 1), "message": f"Run {i + 1}/{args.runs} ({elapsed_ms:.1f}ms)"})
+
+ if i == 0:
+ output_key = list(prediction.keys())[0]
+ depth_map = np.array(prediction[output_key])
+ if depth_map.ndim > 2:
+ depth_map = np.squeeze(depth_map)
+ depth_norm = (depth_map - depth_map.min()) / (depth_map.max() - depth_map.min() + 1e-8)
+ depth_uint8 = (depth_norm * 255).astype(np.uint8)
+ last_depth_colored = cv2.applyColorMap(depth_uint8, colormap_id)
+ last_depth_colored = cv2.resize(last_depth_colored, (original_w, original_h))
+
+ return _build_result(
+ times, load_time_ms, args, last_depth_colored,
+ backend="coreml", device="neural_engine",
+ )
+
+
+# ── ONNX / TensorRT / PyTorch benchmark (Windows/Linux) ─────────────────
+
+def run_inference_benchmark(args, test_image):
+ """Run non-macOS benchmark. Uses DepthEstimationSkill (auto: ONNX → TRT → PyTorch)."""
+ import cv2
+ import numpy as np
+ from transform import DepthEstimationSkill
+
+ model_name = args.model or "depth-anything-v2-small"
+ colormap_id = COLORMAP_MAP.get(args.colormap, 16)
+
+ # Create skill and load model (auto-selects TensorRT → PyTorch cascade)
+ skill = DepthEstimationSkill()
+
+ # Hardware detection
+ from transform_base import TransformSkillBase
+ device_pref = args.device or "auto"
+ skill.env = TransformSkillBase._detect_hardware(device_pref)
+ skill.device = skill.env.device
+
+ config = {
+ "model": model_name,
+ "device": device_pref,
+ "colormap": args.colormap,
+ "blend_mode": "depth_only",
+ }
+
+ _emit({"event": "progress", "stage": "model", "message": f"Loading model: {model_name} ({skill.device})..."})
+ _log(f"Loading model: {model_name} (device={skill.device})")
+ t0 = time.perf_counter()
+ ready_info = skill.load_model(config)
+ load_time_ms = (time.perf_counter() - t0) * 1000
+ backend = ready_info.get("backend", "pytorch")
+ device = ready_info.get("device", skill.device)
+ _log(f"Model loaded in {load_time_ms:.0f}ms (backend={backend}, device={device})")
+
+ # Warm-up run
+ _emit({"event": "progress", "stage": "warmup", "message": "Warm-up inference..."})
+ _log("Warm-up inference...")
+ skill.transform_frame(test_image, {"camera_id": "bench", "frame_id": "warmup"})
+
+ # Benchmark runs
+ _emit({"event": "progress", "stage": "benchmark", "message": f"Running {args.runs} iterations...", "total": args.runs})
+ _log(f"Running {args.runs} benchmark iterations...")
+ times = []
+ last_depth_colored = None
+
+ for i in range(args.runs):
+ t0 = time.perf_counter()
+ result = skill.transform_frame(
+ test_image, {"camera_id": "bench", "frame_id": f"run_{i}"}
+ )
+ elapsed_ms = (time.perf_counter() - t0) * 1000
+ times.append(elapsed_ms)
+ _emit({"event": "progress", "stage": "run", "run": i + 1, "total": args.runs,
+ "time_ms": round(elapsed_ms, 1), "message": f"Run {i + 1}/{args.runs} ({elapsed_ms:.1f}ms)"})
+
+ if i == 0:
+ last_depth_colored = result
+
+ return _build_result(
+ times, load_time_ms, args, last_depth_colored,
+ backend=backend, device=device,
+ )
+
+
+# ── Shared result builder ────────────────────────────────────────────────────
+
+def _build_result(times, load_time_ms, args, last_depth_colored,
+ backend="pytorch", device="cpu"):
+ """Build the JSON result dict from benchmark timings."""
+ import statistics
+
+ times_sorted = sorted(times)
+ avg_ms = statistics.mean(times)
+ std_ms = statistics.stdev(times) if len(times) > 1 else 0
+
+ result = {
+ "model_id": args.model or "depth-anything-v2-small",
+ "variant_id": args.variant,
+ "num_runs": args.runs,
+ "successful_runs": len(times),
+ "avg_time_ms": round(avg_ms, 2),
+ "min_time_ms": round(times_sorted[0], 2),
+ "max_time_ms": round(times_sorted[-1], 2),
+ "std_time_ms": round(std_ms, 2),
+ "fps": round(1000.0 / avg_ms, 2) if avg_ms > 0 else 0,
+ "model_load_ms": round(load_time_ms, 2),
+ "backend": backend,
+ "device": device,
+ "compute_units": args.compute_units,
+ "platform": platform.system(),
+ }
+
+ # Encode extraction result as base64 for preview
+ if last_depth_colored is not None:
+ import base64
+ import cv2
+ _, buf = cv2.imencode(".jpg", last_depth_colored, [cv2.IMWRITE_JPEG_QUALITY, 85])
+ result["extraction_result"] = {
+ "success": True,
+ "feature_type": "depth_estimation",
+ "feature_data": base64.b64encode(buf).decode("ascii"),
+ "processing_time": round(times[0], 2),
+ "metadata": {
+ "model": args.variant or args.model,
+ "colormap": args.colormap,
+ "backend": backend,
+ "device": device,
+ },
+ }
+
+ return result
+
+
+# ── Main ─────────────────────────────────────────────────────────────────────
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Cross-platform depth estimation benchmark")
+ parser.add_argument("--variant", default="DepthAnythingV2SmallF16",
+ help="CoreML variant ID (macOS) or model variant name")
+ parser.add_argument("--model", default="depth-anything-v2-small",
+ help="Model name (e.g., depth-anything-v2-small)")
+ parser.add_argument("--runs", type=int, default=10)
+ parser.add_argument("--colormap", default="viridis")
+ parser.add_argument("--compute-units", default="all")
+ parser.add_argument("--device", default="auto",
+ choices=["auto", "cpu", "cuda", "mps"])
+ parser.add_argument("--test-image-url",
+ default="https://ultralytics.com/images/bus.jpg")
+ args = parser.parse_args()
+
+ # Download test image (shared across all backends)
+ test_image = download_test_image(args.test_image_url)
+
+ # Route to appropriate benchmark
+ if platform.system() == "Darwin":
+ try:
+ result = run_coreml_benchmark(args, test_image)
+ except Exception as e:
+ _log(f"CoreML benchmark failed ({e}), falling back to ONNX/PyTorch")
+ result = run_inference_benchmark(args, test_image)
+ else:
+ result = run_inference_benchmark(args, test_image)
+
+ if "error" in result:
+ _log(f"Benchmark failed: {result['error']}")
+ else:
+ _log(f"Benchmark complete: {result['avg_time_ms']:.1f}ms avg ({result['fps']:.1f} FPS)")
+
+ # Emit final result as JSONL (event=result so handler knows to resolve)
+ result["event"] = "result"
+ _emit(result)
diff --git a/skills/transformation/depth-estimation/scripts/transform.py b/skills/transformation/depth-estimation/scripts/transform.py
index c4013c37..33014470 100644
--- a/skills/transformation/depth-estimation/scripts/transform.py
+++ b/skills/transformation/depth-estimation/scripts/transform.py
@@ -4,7 +4,8 @@
Backend selection:
macOS → CoreML (.mlpackage via coremltools) — runs on Neural Engine
- Other → PyTorch (depth_anything_v2 pip package + HF weights) — runs on CUDA/MPS/CPU
+ Other → ONNX Runtime (pre-exported .onnx from HuggingFace) — CUDA/TRT/DirectML/CPU
+ Fallback → PyTorch (depth_anything_v2 pip package + HF weights) — CUDA/MPS/CPU
Implements the TransformSkillBase interface to provide real-time depth map
overlays on camera feeds. When used as a privacy skill, the depth-only mode
@@ -70,6 +71,9 @@
# Where Aegis DepthVisionStudio stores downloaded models
MODELS_DIR = Path.home() / ".aegis-ai" / "models" / "feature-extraction"
+# TensorRT engine cache directory (engines are GPU-specific)
+TRT_CACHE_DIR = MODELS_DIR / "trt_engines"
+
# PyTorch model configs (fallback on non-macOS)
PYTORCH_CONFIGS = {
"depth-anything-v2-small": {
@@ -92,6 +96,15 @@
},
}
+# ONNX model configs — pre-exported models from onnx-community on HuggingFace
+ONNX_CONFIGS = {
+ "depth-anything-v2-small": {
+ "repo": "onnx-community/depth-anything-v2-small",
+ "filename": "onnx/model.onnx",
+ "input_size": (518, 518), # H, W
+ },
+}
+
class DepthEstimationSkill(TransformSkillBase):
"""
@@ -105,11 +118,22 @@ def __init__(self):
super().__init__()
self._tag = "DepthEstimation"
self.model = None
- self.backend = None # "coreml" or "pytorch"
+ self.backend = None # "coreml", "onnx", "tensorrt", or "pytorch"
self.colormap_id = 1
self.opacity = 0.5
self.blend_mode = "depth_only" # Default for privacy: depth_only anonymizes
self._coreml_input_size = COREML_INPUT_SIZE
+ # ONNX Runtime state
+ self._ort_session = None
+ self._ort_input_name = None
+ self._ort_input_size = (518, 518) # H, W default
+ # TensorRT state (populated by _load_tensorrt)
+ self._trt_context = None
+ self._trt_input_name = None
+ self._trt_output_name = None
+ self._trt_input_tensor = None
+ self._trt_output_tensor = None
+ self._trt_stream = None
def parse_extra_args(self, parser: argparse.ArgumentParser):
parser.add_argument("--model", type=str, default="depth-anything-v2-small",
@@ -117,7 +141,7 @@ def parse_extra_args(self, parser: argparse.ArgumentParser):
"depth-anything-v2-large"])
parser.add_argument("--variant", type=str, default=DEFAULT_COREML_VARIANT,
help="CoreML variant ID (macOS only)")
- parser.add_argument("--colormap", type=str, default="inferno",
+ parser.add_argument("--colormap", type=str, default="viridis",
choices=list(COLORMAP_MAP.keys()))
parser.add_argument("--blend-mode", type=str, default="depth_only",
choices=["overlay", "side_by_side", "depth_only"])
@@ -125,7 +149,7 @@ def parse_extra_args(self, parser: argparse.ArgumentParser):
def load_model(self, config: dict) -> dict:
model_name = config.get("model", "depth-anything-v2-small")
- self.colormap_id = COLORMAP_MAP.get(config.get("colormap", "inferno"), 1)
+ self.colormap_id = COLORMAP_MAP.get(config.get("colormap", "viridis"), 16)
self.opacity = config.get("opacity", 0.5)
self.blend_mode = config.get("blend_mode", "depth_only")
@@ -137,6 +161,20 @@ def load_model(self, config: dict) -> dict:
except Exception as e:
_log(f"CoreML load failed ({e}), falling back to PyTorch", self._tag)
+ # Non-macOS: try ONNX Runtime first (lightest, fastest install)
+ try:
+ info = self._load_onnx(model_name, config)
+ return info
+ except Exception as e:
+ _log(f"ONNX Runtime load failed ({e}), trying TensorRT...", self._tag)
+
+ # Try TensorRT (fails fast if not installed)
+ try:
+ info = self._load_tensorrt(model_name, config)
+ return info
+ except Exception as e:
+ _log(f"TensorRT unavailable ({e}), falling back to PyTorch", self._tag)
+
# Fallback: PyTorch
return self._load_pytorch(model_name, config)
@@ -166,7 +204,7 @@ def _load_coreml(self, config: dict) -> dict:
"model": f"coreml-{variant_id}",
"device": "neural_engine",
"blend_mode": self.blend_mode,
- "colormap": config.get("colormap", "inferno"),
+ "colormap": config.get("colormap", "viridis"),
"backend": "coreml",
}
@@ -196,6 +234,229 @@ def _download_coreml_model(self, variant_id: str):
_log(f"CoreML model download failed: {e}", self._tag)
raise
+ # ── ONNX Runtime backend (Windows/Linux — all GPUs) ────────────────
+
+ @staticmethod
+ def _add_nvidia_dll_paths():
+ """Add pip-installed NVIDIA DLL directories to PATH so ORT finds cudnn, cublas, etc."""
+ import site
+ import glob
+
+ for sp in site.getsitepackages():
+ nvidia_dir = os.path.join(sp, "nvidia")
+ if not os.path.isdir(nvidia_dir):
+ continue
+ for bin_dir in glob.glob(os.path.join(nvidia_dir, "*", "bin")):
+ if bin_dir not in os.environ.get("PATH", ""):
+ os.environ["PATH"] = bin_dir + os.pathsep + os.environ.get("PATH", "")
+ # Python 3.8+ on Windows: also register via os.add_dll_directory
+ if hasattr(os, "add_dll_directory"):
+ try:
+ os.add_dll_directory(bin_dir)
+ except OSError:
+ pass
+ _log(f"Added NVIDIA DLL path: {bin_dir}", "DepthEstimation")
+
+
+ def _load_onnx(self, model_name: str, config: dict) -> dict:
+ """Load ONNX model with best available EP: CUDA → TRT → DirectML → CPU."""
+ # Add pip-installed NVIDIA DLL dirs to PATH (cudnn, cublas, etc.)
+ self._add_nvidia_dll_paths()
+
+ import onnxruntime as ort
+ from huggingface_hub import hf_hub_download
+
+ onnx_cfg = ONNX_CONFIGS.get(model_name)
+ if not onnx_cfg:
+ raise ValueError(f"No ONNX config for model: {model_name}")
+
+ # Check local models dir first (placed by deploy.bat or UI download)
+ local_onnx = MODELS_DIR / f"{Path(onnx_cfg['filename']).stem}.onnx"
+ if local_onnx.exists():
+ model_path = str(local_onnx)
+ _log(f"Found local ONNX model: {local_onnx}", self._tag)
+ else:
+ # Fall back to HuggingFace cache download
+ _log(f"Downloading ONNX model: {onnx_cfg['repo']}...", self._tag)
+ model_path = hf_hub_download(onnx_cfg["repo"], onnx_cfg["filename"])
+
+ # Build EP cascade: prefer GPU, fall back to CPU
+ available_eps = ort.get_available_providers()
+ _log(f"Available ONNX EPs: {available_eps}", self._tag)
+
+ ep_priority = [
+ ("CUDAExecutionProvider", "cuda"),
+ ("TensorrtExecutionProvider", "tensorrt"),
+ ("DmlExecutionProvider", "directml"),
+ ("CPUExecutionProvider", "cpu"),
+ ]
+
+ selected_eps = []
+ device_name = "cpu"
+ for ep_name, dev in ep_priority:
+ if ep_name in available_eps:
+ selected_eps.append(ep_name)
+ if device_name == "cpu":
+ device_name = dev # first non-CPU EP
+
+ if not selected_eps:
+ selected_eps = ["CPUExecutionProvider"]
+
+ _log(f"Creating ONNX session with EPs: {selected_eps}", self._tag)
+ sess_opts = ort.SessionOptions()
+ sess_opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+
+ self._ort_session = ort.InferenceSession(
+ model_path, sess_options=sess_opts, providers=selected_eps
+ )
+ self._ort_input_name = self._ort_session.get_inputs()[0].name
+ self._ort_input_size = onnx_cfg["input_size"]
+ self.backend = "onnx"
+
+ active_ep = self._ort_session.get_providers()[0]
+ _log(f"ONNX model loaded: {model_name} (EP={active_ep})", self._tag)
+ return {
+ "model": model_name,
+ "device": device_name,
+ "blend_mode": self.blend_mode,
+ "colormap": config.get("colormap", "viridis"),
+ "backend": "onnx",
+ "execution_provider": active_ep,
+ }
+
+ # ── TensorRT backend (Windows/Linux NVIDIA) ───────────────────────
+
+ def _load_tensorrt(self, model_name: str, config: dict) -> dict:
+ """Load or build a TensorRT FP16 engine for fastest NVIDIA inference."""
+ import torch
+ import tensorrt as trt
+
+ _log(f"Attempting TensorRT FP16 for {model_name}", self._tag)
+
+ cfg = PYTORCH_CONFIGS.get(model_name)
+ if not cfg:
+ raise ValueError(f"Unknown model: {model_name}")
+
+ gpu_tag = torch.cuda.get_device_name(0).replace(" ", "_").lower()
+ engine_path = TRT_CACHE_DIR / f"{cfg['filename'].replace('.pth', '')}_fp16_{gpu_tag}.trt"
+
+ if engine_path.exists():
+ _log(f"Loading cached TRT engine: {engine_path}", self._tag)
+ engine = self._deserialize_engine(engine_path)
+ else:
+ _log("No cached engine — building from ONNX (30-120s)...", self._tag)
+ engine = self._build_trt_engine(cfg, engine_path)
+
+ if engine is None:
+ raise RuntimeError("TensorRT engine build/load failed")
+
+ self._trt_context = engine.create_execution_context()
+ self._trt_input_name = engine.get_tensor_name(0)
+ self._trt_output_name = engine.get_tensor_name(1)
+
+ input_shape = engine.get_tensor_shape(self._trt_input_name)
+ fixed_shape = tuple(1 if d == -1 else d for d in input_shape)
+ self._trt_context.set_input_shape(self._trt_input_name, fixed_shape)
+
+ self._trt_input_tensor = torch.zeros(fixed_shape, dtype=torch.float32, device="cuda")
+ actual_out_shape = self._trt_context.get_tensor_shape(self._trt_output_name)
+ self._trt_output_tensor = torch.empty(list(actual_out_shape), dtype=torch.float32, device="cuda")
+
+ self._trt_context.set_tensor_address(self._trt_input_name, self._trt_input_tensor.data_ptr())
+ self._trt_context.set_tensor_address(self._trt_output_name, self._trt_output_tensor.data_ptr())
+ self._trt_stream = torch.cuda.current_stream().cuda_stream
+
+ self.backend = "tensorrt"
+ _log(f"TensorRT FP16 engine ready: {engine_path.name}", self._tag)
+ return {
+ "model": model_name,
+ "device": "cuda",
+ "blend_mode": self.blend_mode,
+ "colormap": config.get("colormap", "viridis"),
+ "backend": "tensorrt",
+ }
+
+ def _build_trt_engine(self, cfg: dict, engine_path: Path):
+ """Export PyTorch → ONNX → build TRT FP16 engine → serialize to disk."""
+ import torch
+ import tensorrt as trt
+ from depth_anything_v2.dpt import DepthAnythingV2
+ from huggingface_hub import hf_hub_download
+
+ weights_path = hf_hub_download(cfg["repo"], cfg["filename"])
+ pt_model = DepthAnythingV2(
+ encoder=cfg["encoder"], features=cfg["features"],
+ out_channels=cfg["out_channels"],
+ )
+ pt_model.load_state_dict(torch.load(weights_path, map_location="cuda", weights_only=True))
+ pt_model.to("cuda").eval()
+
+ dummy = torch.randn(1, 3, 518, 518, device="cuda")
+ onnx_path = TRT_CACHE_DIR / f"{cfg['filename'].replace('.pth', '')}.onnx"
+ TRT_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+
+ _log(f"Exporting ONNX: {onnx_path.name}", self._tag)
+ torch.onnx.export(
+ pt_model, dummy, str(onnx_path),
+ input_names=["input"], output_names=["depth"],
+ dynamic_axes={"input": {0: "batch"}, "depth": {0: "batch"}},
+ opset_version=17,
+ )
+ del pt_model
+ torch.cuda.empty_cache()
+
+ logger = trt.Logger(trt.Logger.WARNING)
+ builder = trt.Builder(logger)
+ network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))
+ parser = trt.OnnxParser(network, logger)
+
+ _log("Parsing ONNX for TensorRT...", self._tag)
+ with open(str(onnx_path), "rb") as f:
+ if not parser.parse(f.read()):
+ for i in range(parser.num_errors):
+ _log(f" ONNX parse error: {parser.get_error(i)}", self._tag)
+ return None
+
+ config = builder.create_builder_config()
+ config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 1 << 30)
+
+ inp = network.get_input(0)
+ if any(d == -1 for d in inp.shape):
+ profile = builder.create_optimization_profile()
+ fixed = tuple(1 if d == -1 else d for d in inp.shape)
+ profile.set_shape(inp.name, fixed, fixed, fixed)
+ config.add_optimization_profile(profile)
+
+ config.set_flag(trt.BuilderFlag.FP16)
+
+ _log("Building TRT FP16 engine (30-120s)...", self._tag)
+ serialized = builder.build_serialized_network(network, config)
+ if serialized is None:
+ _log("TRT engine build failed!", self._tag)
+ return None
+
+ engine_bytes = bytes(serialized)
+ with open(str(engine_path), "wb") as f:
+ f.write(engine_bytes)
+ _log(f"Engine cached: {engine_path} ({len(engine_bytes) / 1e6:.1f} MB)", self._tag)
+
+ try:
+ onnx_path.unlink()
+ except OSError:
+ pass
+
+ runtime = trt.Runtime(logger)
+ return runtime.deserialize_cuda_engine(engine_bytes)
+
+ @staticmethod
+ def _deserialize_engine(engine_path: Path):
+ """Load a previously serialized TRT engine from disk."""
+ import tensorrt as trt
+ logger = trt.Logger(trt.Logger.WARNING)
+ runtime = trt.Runtime(logger)
+ with open(str(engine_path), "rb") as f:
+ return runtime.deserialize_cuda_engine(f.read())
+
# ── PyTorch backend (fallback) ────────────────────────────────────
def _load_pytorch(self, model_name: str, config: dict) -> dict:
@@ -230,7 +491,7 @@ def _load_pytorch(self, model_name: str, config: dict) -> dict:
"model": model_name,
"device": self.device,
"blend_mode": self.blend_mode,
- "colormap": config.get("colormap", "inferno"),
+ "colormap": config.get("colormap", "viridis"),
"backend": "pytorch",
}
@@ -242,6 +503,10 @@ def transform_frame(self, image, metadata: dict):
if self.backend == "coreml":
depth_colored = self._infer_coreml(image)
+ elif self.backend == "onnx":
+ depth_colored = self._infer_onnx(image)
+ elif self.backend == "tensorrt":
+ depth_colored = self._infer_tensorrt(image)
else:
depth_colored = self._infer_pytorch(image)
@@ -308,6 +573,69 @@ def _infer_pytorch(self, image):
return depth_colored
+ def _infer_onnx(self, image):
+ """Run ONNX Runtime inference and return colorized depth map."""
+ import cv2
+ import numpy as np
+
+ original_h, original_w = image.shape[:2]
+ input_h, input_w = self._ort_input_size
+
+ rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+ resized = cv2.resize(rgb, (input_w, input_h), interpolation=cv2.INTER_LINEAR)
+ img_float = resized.astype(np.float32) / 255.0
+
+ # ImageNet normalization
+ mean = np.array([0.485, 0.456, 0.406], dtype=np.float32)
+ std = np.array([0.229, 0.224, 0.225], dtype=np.float32)
+ img_float = (img_float - mean) / std
+
+ # HWC → NCHW
+ img_nchw = np.transpose(img_float, (2, 0, 1))[np.newaxis].astype(np.float32)
+
+ # Run inference
+ outputs = self._ort_session.run(None, {self._ort_input_name: img_nchw})
+ depth = outputs[0]
+ depth = np.squeeze(depth)
+
+ # Normalize → uint8 → colormap → resize back
+ d_min, d_max = depth.min(), depth.max()
+ depth_norm = ((depth - d_min) / (d_max - d_min + 1e-8) * 255).astype(np.uint8)
+ depth_colored = cv2.applyColorMap(depth_norm, self.colormap_id)
+ depth_colored = cv2.resize(depth_colored, (original_w, original_h))
+
+ return depth_colored
+
+ def _infer_tensorrt(self, image):
+ """Run TensorRT FP16 inference and return colorized depth map."""
+ import torch
+ import cv2
+ import numpy as np
+
+ original_h, original_w = image.shape[:2]
+ rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+
+ resized = cv2.resize(rgb, (518, 518), interpolation=cv2.INTER_LINEAR)
+ img_float = resized.astype(np.float32) / 255.0
+ mean = np.array([0.485, 0.456, 0.406], dtype=np.float32)
+ std = np.array([0.229, 0.224, 0.225], dtype=np.float32)
+ img_float = (img_float - mean) / std
+ img_nchw = np.transpose(img_float, (2, 0, 1))[np.newaxis]
+
+ self._trt_input_tensor.copy_(torch.from_numpy(img_nchw))
+ self._trt_context.execute_async_v3(self._trt_stream)
+ torch.cuda.synchronize()
+
+ depth = self._trt_output_tensor.cpu().numpy()
+ depth = np.squeeze(depth)
+
+ d_min, d_max = depth.min(), depth.max()
+ depth_norm = ((depth - d_min) / (d_max - d_min + 1e-8) * 255).astype(np.uint8)
+ depth_colored = cv2.applyColorMap(depth_norm, self.colormap_id)
+ depth_colored = cv2.resize(depth_colored, (original_w, original_h))
+
+ return depth_colored
+
# ── Config updates ────────────────────────────────────────────────
def on_config_update(self, config: dict):
@@ -322,9 +650,7 @@ def on_config_update(self, config: dict):
self.blend_mode = config["blend_mode"]
_log(f"Blend mode updated: {self.blend_mode}", self._tag)
- def get_output_mode(self) -> str:
- """Use base64 for privacy transforms — avoids temp file cleanup issues."""
- return "base64"
+
if __name__ == "__main__":