diff --git a/client/integration-tests/primitives/tools/annotation_create/basic_create/after/monitoring-state.json b/client/integration-tests/primitives/tools/annotation_create/basic_create/after/monitoring-state.json new file mode 100644 index 00000000..528e5a7b --- /dev/null +++ b/client/integration-tests/primitives/tools/annotation_create/basic_create/after/monitoring-state.json @@ -0,0 +1,12 @@ +{ + "toolName": "annotation_create", + "parameters": { + "category": "note", + "entityKey": "file:test/Example1.ql:L10", + "content": "Potential SQL injection via string concatenation", + "label": "sql-injection-candidate", + "metadata": "{\"severity\":\"high\",\"cwe\":\"CWE-89\"}" + }, + "success": true, + "description": "Successfully created a note annotation with content, label, and structured metadata" +} diff --git a/client/integration-tests/primitives/tools/annotation_create/basic_create/before/monitoring-state.json b/client/integration-tests/primitives/tools/annotation_create/basic_create/before/monitoring-state.json new file mode 100644 index 00000000..23b7558e --- /dev/null +++ b/client/integration-tests/primitives/tools/annotation_create/basic_create/before/monitoring-state.json @@ -0,0 +1,12 @@ +{ + "toolName": "annotation_create", + "parameters": { + "category": "note", + "entityKey": "file:test/Example1.ql:L10", + "content": "Potential SQL injection via string concatenation", + "label": "sql-injection-candidate", + "metadata": "{\"severity\":\"high\",\"cwe\":\"CWE-89\"}" + }, + "expectedSuccess": true, + "description": "Test annotation_create tool creates a note annotation with content, label, and metadata" +} diff --git a/client/integration-tests/primitives/tools/annotation_create/basic_create/test-config.json b/client/integration-tests/primitives/tools/annotation_create/basic_create/test-config.json new file mode 100644 index 00000000..efa61a01 --- /dev/null +++ b/client/integration-tests/primitives/tools/annotation_create/basic_create/test-config.json @@ -0,0 +1,10 @@ +{ + "toolName": "annotation_create", + "arguments": { + "category": "note", + "entityKey": "file:test/Example1.ql:L10", + "content": "Potential SQL injection via string concatenation", + "label": "sql-injection-candidate", + "metadata": "{\"severity\":\"high\",\"cwe\":\"CWE-89\"}" + } +} diff --git a/client/integration-tests/primitives/tools/annotation_delete/delete_by_prefix/after/monitoring-state.json b/client/integration-tests/primitives/tools/annotation_delete/delete_by_prefix/after/monitoring-state.json new file mode 100644 index 00000000..e95ad9fe --- /dev/null +++ b/client/integration-tests/primitives/tools/annotation_delete/delete_by_prefix/after/monitoring-state.json @@ -0,0 +1,8 @@ +{ + "toolName": "annotation_delete", + "parameters": { + "entityKeyPrefix": "file:test/" + }, + "success": true, + "description": "Successfully deleted annotations matching the entity key prefix" +} diff --git a/client/integration-tests/primitives/tools/annotation_delete/delete_by_prefix/before/monitoring-state.json b/client/integration-tests/primitives/tools/annotation_delete/delete_by_prefix/before/monitoring-state.json new file mode 100644 index 00000000..c5e135bb --- /dev/null +++ b/client/integration-tests/primitives/tools/annotation_delete/delete_by_prefix/before/monitoring-state.json @@ -0,0 +1,8 @@ +{ + "toolName": "annotation_delete", + "parameters": { + "entityKeyPrefix": "file:test/" + }, + "expectedSuccess": true, + "description": "Test annotation_delete tool deletes annotations matching an entity key prefix" +} diff --git a/client/integration-tests/primitives/tools/annotation_delete/delete_by_prefix/test-config.json b/client/integration-tests/primitives/tools/annotation_delete/delete_by_prefix/test-config.json new file mode 100644 index 00000000..be62c279 --- /dev/null +++ b/client/integration-tests/primitives/tools/annotation_delete/delete_by_prefix/test-config.json @@ -0,0 +1,6 @@ +{ + "toolName": "annotation_delete", + "arguments": { + "entityKeyPrefix": "file:test/" + } +} diff --git a/client/integration-tests/primitives/tools/annotation_list/filter_by_category_and_prefix/after/monitoring-state.json b/client/integration-tests/primitives/tools/annotation_list/filter_by_category_and_prefix/after/monitoring-state.json new file mode 100644 index 00000000..c0da6f80 --- /dev/null +++ b/client/integration-tests/primitives/tools/annotation_list/filter_by_category_and_prefix/after/monitoring-state.json @@ -0,0 +1,10 @@ +{ + "toolName": "annotation_list", + "parameters": { + "category": "note", + "entityKeyPrefix": "file:test/", + "limit": 10 + }, + "success": true, + "description": "Successfully listed annotations filtered by category and entity key prefix" +} diff --git a/client/integration-tests/primitives/tools/annotation_list/filter_by_category_and_prefix/before/monitoring-state.json b/client/integration-tests/primitives/tools/annotation_list/filter_by_category_and_prefix/before/monitoring-state.json new file mode 100644 index 00000000..9de23e8a --- /dev/null +++ b/client/integration-tests/primitives/tools/annotation_list/filter_by_category_and_prefix/before/monitoring-state.json @@ -0,0 +1,10 @@ +{ + "toolName": "annotation_list", + "parameters": { + "category": "note", + "entityKeyPrefix": "file:test/", + "limit": 10 + }, + "expectedSuccess": true, + "description": "Test annotation_list tool filters annotations by category and entity key prefix" +} diff --git a/client/integration-tests/primitives/tools/annotation_list/filter_by_category_and_prefix/test-config.json b/client/integration-tests/primitives/tools/annotation_list/filter_by_category_and_prefix/test-config.json new file mode 100644 index 00000000..5e0572dd --- /dev/null +++ b/client/integration-tests/primitives/tools/annotation_list/filter_by_category_and_prefix/test-config.json @@ -0,0 +1,8 @@ +{ + "toolName": "annotation_list", + "arguments": { + "category": "note", + "entityKeyPrefix": "file:test/", + "limit": 10 + } +} diff --git a/client/integration-tests/primitives/tools/annotation_search/full_text_search/after/monitoring-state.json b/client/integration-tests/primitives/tools/annotation_search/full_text_search/after/monitoring-state.json new file mode 100644 index 00000000..2224c452 --- /dev/null +++ b/client/integration-tests/primitives/tools/annotation_search/full_text_search/after/monitoring-state.json @@ -0,0 +1,9 @@ +{ + "toolName": "annotation_search", + "parameters": { + "query": "SQL injection", + "limit": 20 + }, + "success": true, + "description": "Successfully searched annotations by content text" +} diff --git a/client/integration-tests/primitives/tools/annotation_search/full_text_search/before/monitoring-state.json b/client/integration-tests/primitives/tools/annotation_search/full_text_search/before/monitoring-state.json new file mode 100644 index 00000000..536032a6 --- /dev/null +++ b/client/integration-tests/primitives/tools/annotation_search/full_text_search/before/monitoring-state.json @@ -0,0 +1,9 @@ +{ + "toolName": "annotation_search", + "parameters": { + "query": "SQL injection", + "limit": 20 + }, + "expectedSuccess": true, + "description": "Test annotation_search tool performs full-text search across annotation content" +} diff --git a/client/integration-tests/primitives/tools/annotation_search/full_text_search/test-config.json b/client/integration-tests/primitives/tools/annotation_search/full_text_search/test-config.json new file mode 100644 index 00000000..d972ecd9 --- /dev/null +++ b/client/integration-tests/primitives/tools/annotation_search/full_text_search/test-config.json @@ -0,0 +1,7 @@ +{ + "toolName": "annotation_search", + "arguments": { + "query": "SQL injection", + "limit": 20 + } +} diff --git a/client/integration-tests/primitives/tools/audit_add_notes/mrva_triage_notes/after/monitoring-state.json b/client/integration-tests/primitives/tools/audit_add_notes/mrva_triage_notes/after/monitoring-state.json new file mode 100644 index 00000000..d84919eb --- /dev/null +++ b/client/integration-tests/primitives/tools/audit_add_notes/mrva_triage_notes/after/monitoring-state.json @@ -0,0 +1,12 @@ +{ + "toolName": "audit_add_notes", + "parameters": { + "owner": "arduino", + "repo": "Arduino", + "sourceLocation": "app/src/main.cpp", + "line": 42, + "notes": "False positive: serial input is sanitized by validate_command() at line 38 before reaching exec()." + }, + "success": true, + "description": "Successfully appended triage notes to MRVA finding at app/src/main.cpp:42" +} diff --git a/client/integration-tests/primitives/tools/audit_add_notes/mrva_triage_notes/before/monitoring-state.json b/client/integration-tests/primitives/tools/audit_add_notes/mrva_triage_notes/before/monitoring-state.json new file mode 100644 index 00000000..efecc9ae --- /dev/null +++ b/client/integration-tests/primitives/tools/audit_add_notes/mrva_triage_notes/before/monitoring-state.json @@ -0,0 +1,12 @@ +{ + "toolName": "audit_add_notes", + "parameters": { + "owner": "arduino", + "repo": "Arduino", + "sourceLocation": "app/src/main.cpp", + "line": 42, + "notes": "False positive: serial input is sanitized by validate_command() at line 38 before reaching exec()." + }, + "expectedSuccess": true, + "description": "Test audit_add_notes appends triage notes to an existing MRVA finding" +} diff --git a/client/integration-tests/primitives/tools/audit_add_notes/mrva_triage_notes/test-config.json b/client/integration-tests/primitives/tools/audit_add_notes/mrva_triage_notes/test-config.json new file mode 100644 index 00000000..14f5013f --- /dev/null +++ b/client/integration-tests/primitives/tools/audit_add_notes/mrva_triage_notes/test-config.json @@ -0,0 +1,10 @@ +{ + "toolName": "audit_add_notes", + "arguments": { + "owner": "arduino", + "repo": "Arduino", + "sourceLocation": "app/src/main.cpp", + "line": 42, + "notes": "False positive: serial input is sanitized by validate_command() at line 38 before reaching exec()." + } +} diff --git a/client/integration-tests/primitives/tools/audit_clear_repo/clear_single_repo/after/monitoring-state.json b/client/integration-tests/primitives/tools/audit_clear_repo/clear_single_repo/after/monitoring-state.json new file mode 100644 index 00000000..da9f31e7 --- /dev/null +++ b/client/integration-tests/primitives/tools/audit_clear_repo/clear_single_repo/after/monitoring-state.json @@ -0,0 +1,9 @@ +{ + "toolName": "audit_clear_repo", + "parameters": { + "owner": "arduino", + "repo": "Arduino" + }, + "success": true, + "description": "Successfully cleared all audit findings for arduino/Arduino repository" +} diff --git a/client/integration-tests/primitives/tools/audit_clear_repo/clear_single_repo/before/monitoring-state.json b/client/integration-tests/primitives/tools/audit_clear_repo/clear_single_repo/before/monitoring-state.json new file mode 100644 index 00000000..9f451ab7 --- /dev/null +++ b/client/integration-tests/primitives/tools/audit_clear_repo/clear_single_repo/before/monitoring-state.json @@ -0,0 +1,9 @@ +{ + "toolName": "audit_clear_repo", + "parameters": { + "owner": "arduino", + "repo": "Arduino" + }, + "expectedSuccess": true, + "description": "Test audit_clear_repo clears all findings for a single repository without affecting others" +} diff --git a/client/integration-tests/primitives/tools/audit_clear_repo/clear_single_repo/test-config.json b/client/integration-tests/primitives/tools/audit_clear_repo/clear_single_repo/test-config.json new file mode 100644 index 00000000..289fccef --- /dev/null +++ b/client/integration-tests/primitives/tools/audit_clear_repo/clear_single_repo/test-config.json @@ -0,0 +1,7 @@ +{ + "toolName": "audit_clear_repo", + "arguments": { + "owner": "arduino", + "repo": "Arduino" + } +} diff --git a/client/integration-tests/primitives/tools/audit_list_findings/mrva_findings_for_repo/after/monitoring-state.json b/client/integration-tests/primitives/tools/audit_list_findings/mrva_findings_for_repo/after/monitoring-state.json new file mode 100644 index 00000000..87d14578 --- /dev/null +++ b/client/integration-tests/primitives/tools/audit_list_findings/mrva_findings_for_repo/after/monitoring-state.json @@ -0,0 +1,9 @@ +{ + "toolName": "audit_list_findings", + "parameters": { + "owner": "arduino", + "repo": "Arduino" + }, + "success": true, + "description": "Successfully listed audit findings for arduino/Arduino repository" +} diff --git a/client/integration-tests/primitives/tools/audit_list_findings/mrva_findings_for_repo/before/monitoring-state.json b/client/integration-tests/primitives/tools/audit_list_findings/mrva_findings_for_repo/before/monitoring-state.json new file mode 100644 index 00000000..98b434ce --- /dev/null +++ b/client/integration-tests/primitives/tools/audit_list_findings/mrva_findings_for_repo/before/monitoring-state.json @@ -0,0 +1,9 @@ +{ + "toolName": "audit_list_findings", + "parameters": { + "owner": "arduino", + "repo": "Arduino" + }, + "expectedSuccess": true, + "description": "Test audit_list_findings lists findings for a single repository from MRVA results" +} diff --git a/client/integration-tests/primitives/tools/audit_list_findings/mrva_findings_for_repo/test-config.json b/client/integration-tests/primitives/tools/audit_list_findings/mrva_findings_for_repo/test-config.json new file mode 100644 index 00000000..a03ad6ec --- /dev/null +++ b/client/integration-tests/primitives/tools/audit_list_findings/mrva_findings_for_repo/test-config.json @@ -0,0 +1,7 @@ +{ + "toolName": "audit_list_findings", + "arguments": { + "owner": "arduino", + "repo": "Arduino" + } +} diff --git a/client/integration-tests/primitives/tools/audit_store_findings/mrva_findings_across_repos/after/monitoring-state.json b/client/integration-tests/primitives/tools/audit_store_findings/mrva_findings_across_repos/after/monitoring-state.json new file mode 100644 index 00000000..f3c04ff1 --- /dev/null +++ b/client/integration-tests/primitives/tools/audit_store_findings/mrva_findings_across_repos/after/monitoring-state.json @@ -0,0 +1,23 @@ +{ + "toolName": "audit_store_findings", + "parameters": { + "owner": "arduino", + "repo": "Arduino", + "findings": [ + { + "sourceLocation": "app/src/main.cpp", + "line": 42, + "sourceType": "RemoteFlowSource", + "description": "User input from serial read flows to command execution" + }, + { + "sourceLocation": "app/src/network.cpp", + "line": 108, + "sourceType": "RemoteFlowSource", + "description": "HTTP request parameter flows to file path" + } + ] + }, + "success": true, + "description": "Successfully stored MRVA findings for arduino/Arduino repository" +} diff --git a/client/integration-tests/primitives/tools/audit_store_findings/mrva_findings_across_repos/before/monitoring-state.json b/client/integration-tests/primitives/tools/audit_store_findings/mrva_findings_across_repos/before/monitoring-state.json new file mode 100644 index 00000000..173cf03a --- /dev/null +++ b/client/integration-tests/primitives/tools/audit_store_findings/mrva_findings_across_repos/before/monitoring-state.json @@ -0,0 +1,23 @@ +{ + "toolName": "audit_store_findings", + "parameters": { + "owner": "arduino", + "repo": "Arduino", + "findings": [ + { + "sourceLocation": "app/src/main.cpp", + "line": 42, + "sourceType": "RemoteFlowSource", + "description": "User input from serial read flows to command execution" + }, + { + "sourceLocation": "app/src/network.cpp", + "line": 108, + "sourceType": "RemoteFlowSource", + "description": "HTTP request parameter flows to file path" + } + ] + }, + "expectedSuccess": true, + "description": "Test audit_store_findings stores MRVA findings for a repository with deduplication" +} diff --git a/client/integration-tests/primitives/tools/audit_store_findings/mrva_findings_across_repos/test-config.json b/client/integration-tests/primitives/tools/audit_store_findings/mrva_findings_across_repos/test-config.json new file mode 100644 index 00000000..f212a9d7 --- /dev/null +++ b/client/integration-tests/primitives/tools/audit_store_findings/mrva_findings_across_repos/test-config.json @@ -0,0 +1,21 @@ +{ + "toolName": "audit_store_findings", + "arguments": { + "owner": "arduino", + "repo": "Arduino", + "findings": [ + { + "sourceLocation": "app/src/main.cpp", + "line": 42, + "sourceType": "RemoteFlowSource", + "description": "User input from serial read flows to command execution" + }, + { + "sourceLocation": "app/src/network.cpp", + "line": 108, + "sourceType": "RemoteFlowSource", + "description": "HTTP request parameter flows to file path" + } + ] + } +} diff --git a/client/integration-tests/primitives/tools/codeql_query_run/cpp_call_graph_from_to_example1/README.md b/client/integration-tests/primitives/tools/codeql_query_run/cpp_call_graph_from_to_example1/README.md new file mode 100644 index 00000000..919ec689 --- /dev/null +++ b/client/integration-tests/primitives/tools/codeql_query_run/cpp_call_graph_from_to_example1/README.md @@ -0,0 +1,5 @@ +# Integration Test: codeql_query_run/cpp_call_graph_from_to_example1 + +## Purpose + +Tests the `codeql_query_run` tool with the CallGraphFromTo query for cpp language, demonstrating external predicates for both source and target function selection to find transitive call graph reachability. diff --git a/client/integration-tests/primitives/tools/codeql_query_run/cpp_call_graph_from_to_example1/after/monitoring-state.json b/client/integration-tests/primitives/tools/codeql_query_run/cpp_call_graph_from_to_example1/after/monitoring-state.json new file mode 100644 index 00000000..711bd3f3 --- /dev/null +++ b/client/integration-tests/primitives/tools/codeql_query_run/cpp_call_graph_from_to_example1/after/monitoring-state.json @@ -0,0 +1,16 @@ +{ + "toolName": "codeql_query_run", + "parameters": { + "queryName": "CallGraphFromTo", + "queryLanguage": "cpp", + "database": "server/ql/cpp/tools/test/CallGraphFromTo/CallGraphFromTo.testproj", + "sourceFunction": "source", + "targetFunction": "target", + "output": "query-results.bqrs", + "format": "sarif-latest", + "interpretedOutput": "query-results.sarif", + "timeout": 300000 + }, + "success": true, + "description": "Successfully executed CallGraphFromTo (@kind problem) query using query name resolution with external predicates for both source and target function selection and SARIF format interpretation" +} diff --git a/client/integration-tests/primitives/tools/codeql_query_run/cpp_call_graph_from_to_example1/before/monitoring-state.json b/client/integration-tests/primitives/tools/codeql_query_run/cpp_call_graph_from_to_example1/before/monitoring-state.json new file mode 100644 index 00000000..f9f33551 --- /dev/null +++ b/client/integration-tests/primitives/tools/codeql_query_run/cpp_call_graph_from_to_example1/before/monitoring-state.json @@ -0,0 +1,16 @@ +{ + "toolName": "codeql_query_run", + "parameters": { + "queryName": "CallGraphFromTo", + "queryLanguage": "cpp", + "database": "server/ql/cpp/tools/test/CallGraphFromTo/CallGraphFromTo.testproj", + "sourceFunction": "source", + "targetFunction": "target", + "output": "query-results.bqrs", + "format": "sarif-latest", + "interpretedOutput": "query-results.sarif", + "timeout": 300000 + }, + "expectedSuccess": true, + "description": "Test codeql_query_run with queryName resolution for CallGraphFromTo (@kind problem) query using external predicates for both source and target function selection and SARIF format interpretation" +} diff --git a/client/integration-tests/primitives/tools/codeql_query_run/javascript_call_graph_from_to_example1/README.md b/client/integration-tests/primitives/tools/codeql_query_run/javascript_call_graph_from_to_example1/README.md new file mode 100644 index 00000000..870052e2 --- /dev/null +++ b/client/integration-tests/primitives/tools/codeql_query_run/javascript_call_graph_from_to_example1/README.md @@ -0,0 +1,5 @@ +# Integration Test: codeql_query_run/javascript_call_graph_from_to_example1 + +## Purpose + +Tests the `codeql_query_run` tool with the CallGraphFromTo query for javascript language, demonstrating external predicates for both source and target function selection to find transitive call graph reachability. diff --git a/client/integration-tests/primitives/tools/codeql_query_run/javascript_call_graph_from_to_example1/after/monitoring-state.json b/client/integration-tests/primitives/tools/codeql_query_run/javascript_call_graph_from_to_example1/after/monitoring-state.json new file mode 100644 index 00000000..c05fcdfa --- /dev/null +++ b/client/integration-tests/primitives/tools/codeql_query_run/javascript_call_graph_from_to_example1/after/monitoring-state.json @@ -0,0 +1,16 @@ +{ + "toolName": "codeql_query_run", + "parameters": { + "queryName": "CallGraphFromTo", + "queryLanguage": "javascript", + "database": "server/ql/javascript/tools/test/CallGraphFromTo/CallGraphFromTo.testproj", + "sourceFunction": "source", + "targetFunction": "target", + "output": "query-results.bqrs", + "format": "sarif-latest", + "interpretedOutput": "query-results.sarif", + "timeout": 300000 + }, + "success": true, + "description": "Successfully executed CallGraphFromTo (@kind problem) query for javascript using query name resolution with external predicates for source and target function selection" +} diff --git a/client/integration-tests/primitives/tools/codeql_query_run/javascript_call_graph_from_to_example1/before/monitoring-state.json b/client/integration-tests/primitives/tools/codeql_query_run/javascript_call_graph_from_to_example1/before/monitoring-state.json new file mode 100644 index 00000000..28bfc22a --- /dev/null +++ b/client/integration-tests/primitives/tools/codeql_query_run/javascript_call_graph_from_to_example1/before/monitoring-state.json @@ -0,0 +1,16 @@ +{ + "toolName": "codeql_query_run", + "parameters": { + "queryName": "CallGraphFromTo", + "queryLanguage": "javascript", + "database": "server/ql/javascript/tools/test/CallGraphFromTo/CallGraphFromTo.testproj", + "sourceFunction": "source", + "targetFunction": "target", + "output": "query-results.bqrs", + "format": "sarif-latest", + "interpretedOutput": "query-results.sarif", + "timeout": 300000 + }, + "expectedSuccess": true, + "description": "Test codeql_query_run with queryName resolution for CallGraphFromTo (@kind problem) query for javascript using external predicates for source and target function selection" +} diff --git a/client/integration-tests/primitives/tools/codeql_query_run/python_call_graph_from_to_example1/README.md b/client/integration-tests/primitives/tools/codeql_query_run/python_call_graph_from_to_example1/README.md new file mode 100644 index 00000000..aa5f4242 --- /dev/null +++ b/client/integration-tests/primitives/tools/codeql_query_run/python_call_graph_from_to_example1/README.md @@ -0,0 +1,5 @@ +# Integration Test: codeql_query_run/python_call_graph_from_to_example1 + +## Purpose + +Tests the `codeql_query_run` tool with the CallGraphFromTo query for python language, demonstrating external predicates for both source and target function selection to find transitive call graph reachability. diff --git a/client/integration-tests/primitives/tools/codeql_query_run/python_call_graph_from_to_example1/after/monitoring-state.json b/client/integration-tests/primitives/tools/codeql_query_run/python_call_graph_from_to_example1/after/monitoring-state.json new file mode 100644 index 00000000..6906ce4e --- /dev/null +++ b/client/integration-tests/primitives/tools/codeql_query_run/python_call_graph_from_to_example1/after/monitoring-state.json @@ -0,0 +1,16 @@ +{ + "toolName": "codeql_query_run", + "parameters": { + "queryName": "CallGraphFromTo", + "queryLanguage": "python", + "database": "server/ql/python/tools/test/CallGraphFromTo/CallGraphFromTo.testproj", + "sourceFunction": "source", + "targetFunction": "target", + "output": "query-results.bqrs", + "format": "sarif-latest", + "interpretedOutput": "query-results.sarif", + "timeout": 300000 + }, + "success": true, + "description": "Successfully executed CallGraphFromTo (@kind problem) query for python using query name resolution with external predicates for source and target function selection" +} diff --git a/client/integration-tests/primitives/tools/codeql_query_run/python_call_graph_from_to_example1/before/monitoring-state.json b/client/integration-tests/primitives/tools/codeql_query_run/python_call_graph_from_to_example1/before/monitoring-state.json new file mode 100644 index 00000000..f31d129f --- /dev/null +++ b/client/integration-tests/primitives/tools/codeql_query_run/python_call_graph_from_to_example1/before/monitoring-state.json @@ -0,0 +1,16 @@ +{ + "toolName": "codeql_query_run", + "parameters": { + "queryName": "CallGraphFromTo", + "queryLanguage": "python", + "database": "server/ql/python/tools/test/CallGraphFromTo/CallGraphFromTo.testproj", + "sourceFunction": "source", + "targetFunction": "target", + "output": "query-results.bqrs", + "format": "sarif-latest", + "interpretedOutput": "query-results.sarif", + "timeout": 300000 + }, + "expectedSuccess": true, + "description": "Test codeql_query_run with queryName resolution for CallGraphFromTo (@kind problem) query for python using external predicates for source and target function selection" +} diff --git a/client/integration-tests/primitives/tools/query_results_cache_clear/clear_all/after/monitoring-state.json b/client/integration-tests/primitives/tools/query_results_cache_clear/clear_all/after/monitoring-state.json new file mode 100644 index 00000000..33af0c5a --- /dev/null +++ b/client/integration-tests/primitives/tools/query_results_cache_clear/clear_all/after/monitoring-state.json @@ -0,0 +1,8 @@ +{ + "toolName": "query_results_cache_clear", + "parameters": { + "all": true + }, + "success": true, + "description": "Successfully cleared all cached query results" +} diff --git a/client/integration-tests/primitives/tools/query_results_cache_clear/clear_all/before/monitoring-state.json b/client/integration-tests/primitives/tools/query_results_cache_clear/clear_all/before/monitoring-state.json new file mode 100644 index 00000000..3285b66b --- /dev/null +++ b/client/integration-tests/primitives/tools/query_results_cache_clear/clear_all/before/monitoring-state.json @@ -0,0 +1,8 @@ +{ + "toolName": "query_results_cache_clear", + "parameters": { + "all": true + }, + "expectedSuccess": true, + "description": "Test query_results_cache_clear clears all cached query results" +} diff --git a/client/integration-tests/primitives/tools/query_results_cache_clear/clear_all/test-config.json b/client/integration-tests/primitives/tools/query_results_cache_clear/clear_all/test-config.json new file mode 100644 index 00000000..9b6dbff8 --- /dev/null +++ b/client/integration-tests/primitives/tools/query_results_cache_clear/clear_all/test-config.json @@ -0,0 +1,6 @@ +{ + "toolName": "query_results_cache_clear", + "arguments": { + "all": true + } +} diff --git a/client/integration-tests/primitives/tools/query_results_cache_compare/compare_across_databases/after/monitoring-state.json b/client/integration-tests/primitives/tools/query_results_cache_compare/compare_across_databases/after/monitoring-state.json new file mode 100644 index 00000000..7339bcd1 --- /dev/null +++ b/client/integration-tests/primitives/tools/query_results_cache_compare/compare_across_databases/after/monitoring-state.json @@ -0,0 +1,8 @@ +{ + "toolName": "query_results_cache_compare", + "parameters": { + "queryName": "CallGraphFrom" + }, + "success": true, + "description": "Successfully compared cached query results across databases" +} diff --git a/client/integration-tests/primitives/tools/query_results_cache_compare/compare_across_databases/before/monitoring-state.json b/client/integration-tests/primitives/tools/query_results_cache_compare/compare_across_databases/before/monitoring-state.json new file mode 100644 index 00000000..63a03526 --- /dev/null +++ b/client/integration-tests/primitives/tools/query_results_cache_compare/compare_across_databases/before/monitoring-state.json @@ -0,0 +1,8 @@ +{ + "toolName": "query_results_cache_compare", + "parameters": { + "queryName": "CallGraphFrom" + }, + "expectedSuccess": true, + "description": "Test query_results_cache_compare compares cached results for a query across databases" +} diff --git a/client/integration-tests/primitives/tools/query_results_cache_compare/compare_across_databases/test-config.json b/client/integration-tests/primitives/tools/query_results_cache_compare/compare_across_databases/test-config.json new file mode 100644 index 00000000..ff7b9e53 --- /dev/null +++ b/client/integration-tests/primitives/tools/query_results_cache_compare/compare_across_databases/test-config.json @@ -0,0 +1,6 @@ +{ + "toolName": "query_results_cache_compare", + "arguments": { + "queryName": "CallGraphFrom" + } +} diff --git a/client/integration-tests/primitives/tools/query_results_cache_lookup/lookup_by_query_name/after/monitoring-state.json b/client/integration-tests/primitives/tools/query_results_cache_lookup/lookup_by_query_name/after/monitoring-state.json new file mode 100644 index 00000000..be5b856f --- /dev/null +++ b/client/integration-tests/primitives/tools/query_results_cache_lookup/lookup_by_query_name/after/monitoring-state.json @@ -0,0 +1,9 @@ +{ + "toolName": "query_results_cache_lookup", + "parameters": { + "queryName": "PrintAST", + "language": "javascript" + }, + "success": true, + "description": "Successfully looked up cached query results by query name and language" +} diff --git a/client/integration-tests/primitives/tools/query_results_cache_lookup/lookup_by_query_name/before/monitoring-state.json b/client/integration-tests/primitives/tools/query_results_cache_lookup/lookup_by_query_name/before/monitoring-state.json new file mode 100644 index 00000000..94ac468a --- /dev/null +++ b/client/integration-tests/primitives/tools/query_results_cache_lookup/lookup_by_query_name/before/monitoring-state.json @@ -0,0 +1,9 @@ +{ + "toolName": "query_results_cache_lookup", + "parameters": { + "queryName": "PrintAST", + "language": "javascript" + }, + "expectedSuccess": true, + "description": "Test query_results_cache_lookup lists cached results by query name and language" +} diff --git a/client/integration-tests/primitives/tools/query_results_cache_lookup/lookup_by_query_name/test-config.json b/client/integration-tests/primitives/tools/query_results_cache_lookup/lookup_by_query_name/test-config.json new file mode 100644 index 00000000..48dfd3b4 --- /dev/null +++ b/client/integration-tests/primitives/tools/query_results_cache_lookup/lookup_by_query_name/test-config.json @@ -0,0 +1,7 @@ +{ + "toolName": "query_results_cache_lookup", + "arguments": { + "queryName": "PrintAST", + "language": "javascript" + } +} diff --git a/client/integration-tests/primitives/tools/query_results_cache_retrieve/retrieve_with_subset/after/monitoring-state.json b/client/integration-tests/primitives/tools/query_results_cache_retrieve/retrieve_with_subset/after/monitoring-state.json new file mode 100644 index 00000000..f617ab58 --- /dev/null +++ b/client/integration-tests/primitives/tools/query_results_cache_retrieve/retrieve_with_subset/after/monitoring-state.json @@ -0,0 +1,9 @@ +{ + "toolName": "query_results_cache_retrieve", + "parameters": { + "cacheKey": "nonexistent-key-for-test", + "maxLines": 50 + }, + "success": true, + "description": "Successfully handled cache retrieve for non-existent key" +} diff --git a/client/integration-tests/primitives/tools/query_results_cache_retrieve/retrieve_with_subset/before/monitoring-state.json b/client/integration-tests/primitives/tools/query_results_cache_retrieve/retrieve_with_subset/before/monitoring-state.json new file mode 100644 index 00000000..4cc57e4b --- /dev/null +++ b/client/integration-tests/primitives/tools/query_results_cache_retrieve/retrieve_with_subset/before/monitoring-state.json @@ -0,0 +1,9 @@ +{ + "toolName": "query_results_cache_retrieve", + "parameters": { + "cacheKey": "nonexistent-key-for-test", + "maxLines": 50 + }, + "expectedSuccess": true, + "description": "Test query_results_cache_retrieve returns appropriate message for missing cache key" +} diff --git a/client/integration-tests/primitives/tools/query_results_cache_retrieve/retrieve_with_subset/test-config.json b/client/integration-tests/primitives/tools/query_results_cache_retrieve/retrieve_with_subset/test-config.json new file mode 100644 index 00000000..b49922bb --- /dev/null +++ b/client/integration-tests/primitives/tools/query_results_cache_retrieve/retrieve_with_subset/test-config.json @@ -0,0 +1,7 @@ +{ + "toolName": "query_results_cache_retrieve", + "arguments": { + "cacheKey": "nonexistent-key-for-test", + "maxLines": 50 + } +} diff --git a/client/integration-tests/workflows/mrva_finding_triage/mrva_triage_workflow/after/monitoring-state.json b/client/integration-tests/workflows/mrva_finding_triage/mrva_triage_workflow/after/monitoring-state.json new file mode 100644 index 00000000..3dfa7bbe --- /dev/null +++ b/client/integration-tests/workflows/mrva_finding_triage/mrva_triage_workflow/after/monitoring-state.json @@ -0,0 +1,90 @@ +{ + "sessions": [ + { + "sessionId": "mrva-triage-workflow", + "queryPath": "/mrva-triage", + "language": "multi", + "queryType": "mrva-workflow", + "description": "MRVA finding triage workflow: discover runs, store findings across repos, annotate with triage decisions, search across annotations", + "status": "active", + "state": { + "compilationStatus": "unknown", + "testStatus": "unknown", + "documentationStatus": "missing", + "filesPresent": [] + }, + "mcpCalls": [ + { + "callId": "step-1-discover", + "timestamp": "2026-01-01T00:00:00.000Z", + "toolName": "list_mrva_run_results", + "parameters": {}, + "success": true, + "duration": 500, + "result": "Discovered MRVA runs" + }, + { + "callId": "step-2-store-repo1", + "timestamp": "2026-01-01T00:01:00.000Z", + "toolName": "audit_store_findings", + "parameters": { + "owner": "arduino", + "repo": "Arduino", + "findings": [{"sourceLocation": "main.cpp", "line": 42, "sourceType": "RemoteFlowSource"}] + }, + "success": true, + "duration": 200, + "result": "Stored 1 finding(s)" + }, + { + "callId": "step-3-store-repo2", + "timestamp": "2026-01-01T00:02:00.000Z", + "toolName": "audit_store_findings", + "parameters": { + "owner": "google", + "repo": "guava", + "findings": [{"sourceLocation": "src/Util.java", "line": 88, "sourceType": "RemoteFlowSource"}] + }, + "success": true, + "duration": 200, + "result": "Stored 1 finding(s)" + }, + { + "callId": "step-4-list-all", + "timestamp": "2026-01-01T00:03:00.000Z", + "toolName": "annotation_list", + "parameters": {"category": "audit-finding"}, + "success": true, + "duration": 100, + "result": "Listed findings across repos" + }, + { + "callId": "step-5-triage", + "timestamp": "2026-01-01T00:04:00.000Z", + "toolName": "audit_add_notes", + "parameters": { + "owner": "arduino", + "repo": "Arduino", + "sourceLocation": "main.cpp", + "line": 42, + "notes": "False positive: input validated before use" + }, + "success": true, + "duration": 100, + "result": "Updated notes" + }, + { + "callId": "step-6-search", + "timestamp": "2026-01-01T00:05:00.000Z", + "toolName": "annotation_search", + "parameters": {"query": "false positive"}, + "success": true, + "duration": 100, + "result": "Found annotated findings" + } + ], + "testExecutions": [], + "qualityScores": [] + } + ] +} diff --git a/client/integration-tests/workflows/mrva_finding_triage/mrva_triage_workflow/before/monitoring-state.json b/client/integration-tests/workflows/mrva_finding_triage/mrva_triage_workflow/before/monitoring-state.json new file mode 100644 index 00000000..3560f5f7 --- /dev/null +++ b/client/integration-tests/workflows/mrva_finding_triage/mrva_triage_workflow/before/monitoring-state.json @@ -0,0 +1,21 @@ +{ + "sessions": [ + { + "sessionId": "mrva-triage-workflow", + "queryPath": "/mrva-triage", + "language": "multi", + "queryType": "mrva-workflow", + "description": "MRVA finding triage workflow: discover runs, store findings across repos, annotate with triage decisions, search across annotations", + "status": "active", + "state": { + "compilationStatus": "unknown", + "testStatus": "unknown", + "documentationStatus": "missing", + "filesPresent": [] + }, + "mcpCalls": [], + "testExecutions": [], + "qualityScores": [] + } + ] +} diff --git a/client/scripts/run-integration-tests.sh b/client/scripts/run-integration-tests.sh index 49aa18dd..6ff77b2f 100755 --- a/client/scripts/run-integration-tests.sh +++ b/client/scripts/run-integration-tests.sh @@ -42,20 +42,30 @@ export URL_SCHEME="${URL_SCHEME:-http}" # Otherwise, run both modes RUN_DEFAULT_MODE=true RUN_MONITORING_MODE=true +RUN_ANNOTATION_MODE=true if [ -n "${ENABLE_MONITORING_TOOLS+x}" ]; then # Variable is explicitly set (even if empty) if [ "$ENABLE_MONITORING_TOOLS" = "true" ]; then RUN_DEFAULT_MODE=false RUN_MONITORING_MODE=true + RUN_ANNOTATION_MODE=false echo "🔧 Mode: Monitoring tools ONLY (ENABLE_MONITORING_TOOLS=true)" else RUN_DEFAULT_MODE=true RUN_MONITORING_MODE=false + RUN_ANNOTATION_MODE=false echo "🔧 Mode: Default mode ONLY (ENABLE_MONITORING_TOOLS=false)" fi +elif [ -n "${ENABLE_ANNOTATION_TOOLS+x}" ]; then + if [ "$ENABLE_ANNOTATION_TOOLS" = "true" ]; then + RUN_DEFAULT_MODE=false + RUN_MONITORING_MODE=false + RUN_ANNOTATION_MODE=true + echo "🔧 Mode: Annotation tools ONLY (ENABLE_ANNOTATION_TOOLS=true)" + fi else - echo "🔧 Mode: Running BOTH default and monitoring modes" + echo "🔧 Mode: Running ALL modes (default + monitoring + annotation)" fi # Check if --no-install-packs was passed @@ -152,6 +162,13 @@ if [ "$RUN_MONITORING_MODE" = true ]; then run_tests_in_mode "MONITORING MODE (session_* tools)" "true" "${EXTRA_ARGS[@]}" fi +if [ "$RUN_ANNOTATION_MODE" = true ]; then + # Annotation mode enables annotation, audit, and cache tools + export ENABLE_ANNOTATION_TOOLS="true" + run_tests_in_mode "ANNOTATION MODE (annotation_*, audit_*, query_results_cache_* tools)" "false" "${EXTRA_ARGS[@]}" + unset ENABLE_ANNOTATION_TOOLS +fi + echo "" echo "═══════════════════════════════════════════════════════════════" echo "✅ All integration tests completed successfully!" diff --git a/extensions/vscode/test/suite/mcp-tool-e2e.integration.test.ts b/extensions/vscode/test/suite/mcp-tool-e2e.integration.test.ts index e97171b1..23c6804d 100644 --- a/extensions/vscode/test/suite/mcp-tool-e2e.integration.test.ts +++ b/extensions/vscode/test/suite/mcp-tool-e2e.integration.test.ts @@ -255,4 +255,209 @@ suite('MCP Server Tool Integration Tests', () => { console.log(`[mcp-tool-e2e] list_mrva_run_results result:\n${text}`); }); + + test('Annotation and audit tools should NOT appear by default', async function () { + this.timeout(15_000); + + const response = await client.listTools(); + const toolNames = response.tools.map(t => t.name); + + assert.ok(!toolNames.includes('annotation_create'), 'annotation_create should NOT be registered by default'); + assert.ok(!toolNames.includes('audit_store_findings'), 'audit_store_findings should NOT be registered by default'); + assert.ok(!toolNames.includes('query_results_cache_lookup'), 'query_results_cache_lookup should NOT be registered by default'); + + console.log('[mcp-tool-e2e] Confirmed annotation/audit/cache tools are not registered by default'); + }); +}); + +/** + * Integration tests for annotation and audit tools. + * These tests spawn a separate server instance with ENABLE_ANNOTATION_TOOLS=true + * to validate opt-in annotation and MRVA audit tool functionality. + */ +suite('MCP Annotation & Audit Tool Integration Tests', () => { + let client: Client; + let transport: StdioClientTransport; + let fixtureStorage: string | undefined; + + suiteSetup(async function () { + this.timeout(30_000); + + const ext = vscode.extensions.getExtension(EXTENSION_ID); + assert.ok(ext, `Extension ${EXTENSION_ID} not found`); + if (!ext.isActive) await ext.activate(); + + fixtureStorage = resolveFixtureStoragePath(); + assert.ok(fixtureStorage, 'Fixture codeql-storage directory not found'); + + const serverPath = resolveServerPath(); + + const env: Record = { + ...process.env as Record, + TRANSPORT_MODE: 'stdio', + ENABLE_ANNOTATION_TOOLS: 'true', + CODEQL_DATABASES_BASE_DIRS: path.join(fixtureStorage, 'databases'), + CODEQL_QUERY_RUN_RESULTS_DIRS: path.join(fixtureStorage, 'queries'), + CODEQL_MRVA_RUN_RESULTS_DIRS: path.join(fixtureStorage, 'variant-analyses'), + }; + + transport = new StdioClientTransport({ + command: 'node', + args: [serverPath], + env, + stderr: 'pipe', + }); + + client = new Client({ name: 'annotation-test', version: '1.0.0' }); + await client.connect(transport); + console.log('[mcp-annotation-e2e] Connected to MCP server with annotation tools enabled'); + }); + + suiteTeardown(async function () { + this.timeout(10_000); + try { if (client) await client.close(); } catch { /* best-effort */ } + try { if (transport) await transport.close(); } catch { /* best-effort */ } + }); + + test('Annotation tools should be available when ENABLE_ANNOTATION_TOOLS=true', async function () { + this.timeout(15_000); + + const response = await client.listTools(); + const toolNames = response.tools.map(t => t.name); + + // Layer 1: annotation tools + assert.ok(toolNames.includes('annotation_create'), 'Should include annotation_create'); + assert.ok(toolNames.includes('annotation_get'), 'Should include annotation_get'); + assert.ok(toolNames.includes('annotation_list'), 'Should include annotation_list'); + assert.ok(toolNames.includes('annotation_update'), 'Should include annotation_update'); + assert.ok(toolNames.includes('annotation_delete'), 'Should include annotation_delete'); + assert.ok(toolNames.includes('annotation_search'), 'Should include annotation_search'); + + // Layer 2: audit tools (gated by same flag) + assert.ok(toolNames.includes('audit_store_findings'), 'Should include audit_store_findings'); + assert.ok(toolNames.includes('audit_list_findings'), 'Should include audit_list_findings'); + assert.ok(toolNames.includes('audit_add_notes'), 'Should include audit_add_notes'); + assert.ok(toolNames.includes('audit_clear_repo'), 'Should include audit_clear_repo'); + + // Layer 3: query results cache tools (gated by same flag) + assert.ok(toolNames.includes('query_results_cache_lookup'), 'Should include query_results_cache_lookup'); + assert.ok(toolNames.includes('query_results_cache_retrieve'), 'Should include query_results_cache_retrieve'); + assert.ok(toolNames.includes('query_results_cache_clear'), 'Should include query_results_cache_clear'); + assert.ok(toolNames.includes('query_results_cache_compare'), 'Should include query_results_cache_compare'); + + console.log(`[mcp-annotation-e2e] All 14 annotation/audit/cache tools registered`); + }); + + test('MRVA + Annotation workflow: store and retrieve findings', async function () { + this.timeout(30_000); + + // Step 1: Discover MRVA runs (from fixture) + const mrvaResult = await client.callTool({ + name: 'list_mrva_run_results', + arguments: {}, + }); + assert.ok(!mrvaResult.isError, 'list_mrva_run_results should succeed'); + const mrvaText = (mrvaResult.content as Array<{ type: string; text: string }>)[0]?.text ?? ''; + assert.ok(mrvaText.includes('10001'), 'Should find fixture MRVA run 10001'); + + // Step 2: Store findings for a repository + const storeResult = await client.callTool({ + name: 'audit_store_findings', + arguments: { + owner: 'arduino', + repo: 'Arduino', + findings: [ + { sourceLocation: 'src/main.cpp', line: 42, sourceType: 'RemoteFlowSource', description: 'Test finding' }, + ], + }, + }); + assert.ok(!storeResult.isError, 'audit_store_findings should succeed'); + const storeText = (storeResult.content as Array<{ type: string; text: string }>)[0]?.text ?? ''; + assert.ok(storeText.includes('Stored 1'), `Should store 1 finding. Got: ${storeText}`); + + // Step 3: List findings for the repo + const listResult = await client.callTool({ + name: 'audit_list_findings', + arguments: { owner: 'arduino', repo: 'Arduino' }, + }); + assert.ok(!listResult.isError, 'audit_list_findings should succeed'); + const listText = (listResult.content as Array<{ type: string; text: string }>)[0]?.text ?? ''; + assert.ok(listText.includes('src/main.cpp'), `Should include finding location. Got: ${listText}`); + + // Step 4: Add triage notes + const notesResult = await client.callTool({ + name: 'audit_add_notes', + arguments: { + owner: 'arduino', + repo: 'Arduino', + sourceLocation: 'src/main.cpp', + line: 42, + notes: 'False positive: validated input', + }, + }); + assert.ok(!notesResult.isError, 'audit_add_notes should succeed'); + + // Step 5: Search for annotated findings + const searchResult = await client.callTool({ + name: 'annotation_search', + arguments: { query: 'false positive' }, + }); + assert.ok(!searchResult.isError, 'annotation_search should succeed'); + const searchText = (searchResult.content as Array<{ type: string; text: string }>)[0]?.text ?? ''; + assert.ok(searchText.includes('validated input'), `Should find triage note. Got: ${searchText}`); + + // Step 6: Clear repo + const clearResult = await client.callTool({ + name: 'audit_clear_repo', + arguments: { owner: 'arduino', repo: 'Arduino' }, + }); + assert.ok(!clearResult.isError, 'audit_clear_repo should succeed'); + const clearText = (clearResult.content as Array<{ type: string; text: string }>)[0]?.text ?? ''; + assert.ok(clearText.includes('Cleared'), `Should confirm clearing. Got: ${clearText}`); + + console.log('[mcp-annotation-e2e] MRVA + Annotation workflow test passed'); + }); + + test('Query results cache: lookup, clear, and compare', async function () { + this.timeout(15_000); + + // Step 1: Lookup should return cached:false for a query not yet run + const lookupResult = await client.callTool({ + name: 'query_results_cache_lookup', + arguments: { queryName: 'PrintAST', language: 'javascript' }, + }); + assert.ok(!lookupResult.isError, 'query_results_cache_lookup should succeed'); + const lookupText = (lookupResult.content as Array<{ type: string; text: string }>)[0]?.text ?? ''; + // May be cached:false or cached:true depending on prior test runs — just verify it doesn't error + assert.ok(lookupText.includes('cached'), `Should contain cached field. Got: ${lookupText}`); + + // Step 2: Compare should work even with empty cache + const compareResult = await client.callTool({ + name: 'query_results_cache_compare', + arguments: { queryName: 'NonExistentQuery' }, + }); + assert.ok(!compareResult.isError, 'query_results_cache_compare should succeed'); + const compareText = (compareResult.content as Array<{ type: string; text: string }>)[0]?.text ?? ''; + assert.ok(compareText.includes('No cached results') || compareText.includes('databases'), `Should handle empty compare. Got: ${compareText}`); + + // Step 3: Clear all — should not error even on empty cache + const clearResult = await client.callTool({ + name: 'query_results_cache_clear', + arguments: { all: true }, + }); + assert.ok(!clearResult.isError, 'query_results_cache_clear should succeed'); + const clearText = (clearResult.content as Array<{ type: string; text: string }>)[0]?.text ?? ''; + assert.ok(clearText.includes('Cleared'), `Should confirm clearing. Got: ${clearText}`); + + // Step 4: Retrieve should handle missing key gracefully + const retrieveResult = await client.callTool({ + name: 'query_results_cache_retrieve', + arguments: { cacheKey: 'nonexistent-test-key', maxLines: 10 }, + }); + assert.ok(!retrieveResult.isError, 'query_results_cache_retrieve should succeed'); + const retrieveText = (retrieveResult.content as Array<{ type: string; text: string }>)[0]?.text ?? ''; + assert.ok(retrieveText.includes('No cached result'), `Should handle missing key. Got: ${retrieveText}`); + + console.log('[mcp-annotation-e2e] Query results cache test passed'); + }); });