mock-generation

.github/workflows/provider-analysis.yml

@@ -1,4 +1,4 @@
-name: Build
+name: Provider Analysis
 
 permissions:
   contents: read
@@ -144,7 +144,7 @@ jobs:
         ref: ${{ env.STACKQL_CORE_REF }}
         path: stackql-core
 
-    - name: Interrogate Providers via CLI and Generate Analysis
+    - name: Pull Providers
       run: |
         stackql exec "registry pull aws ${PROVIDER_VERSION_AWS};"
         stackql exec "registry pull google ${PROVIDER_VERSION_GOOGLE};"
@@ -157,31 +157,87 @@ jobs:
           echo "DATE_PATH=${DATE_PATH}"
           echo "RUN_TS=${RUN_TS}"
         } >> "$GITHUB_ENV"
-        ./build/anysdk aot \
-          ./.stackql \
-          ./.stackql/src/aws/${PROVIDER_VERSION_AWS}/provider.yaml \
-          -v \
-          --schema-dir \
-          cicd/schema-definitions > "cicd/out/${RUN_TS}-aws-${PROVIDER_VERSION_AWS}-summary.json" 2>"cicd/out/${RUN_TS}-aws-${PROVIDER_VERSION_AWS}-analysis.jsonl" || true
-        ./build/anysdk aot \
-          ./.stackql \
-          ./.stackql/src/azure/${PROVIDER_VERSION_AZURE}/provider.yaml \
-          -v \
-          --schema-dir \
-          cicd/schema-definitions > "cicd/out/${RUN_TS}-azure-${PROVIDER_VERSION_AZURE}-summary.json" 2>"cicd/out/${RUN_TS}-azure-${PROVIDER_VERSION_AZURE}-analysis.jsonl" || true
-        ./build/anysdk aot \
-          ./.stackql \
-          ./.stackql/src/googleapis.com/${PROVIDER_VERSION_GOOGLE}/provider.yaml \
-          -v \
-          --schema-dir \
-          cicd/schema-definitions > "cicd/out/${RUN_TS}-google-${PROVIDER_VERSION_GOOGLE}-summary.json" 2>"cicd/out/${RUN_TS}-google-${PROVIDER_VERSION_GOOGLE}-analysis.jsonl" || true
+
+    - name: AOT Analysis and Mock Generation
+      run: |
+        for provider_cfg in \
+          "aws|aws|${PROVIDER_VERSION_AWS}" \
+          "azure|azure|${PROVIDER_VERSION_AZURE}" \
+          "google|googleapis.com|${PROVIDER_VERSION_GOOGLE}"; do
+
+          IFS='|' read -r provider handle version <<< "$provider_cfg"
+
+          ./build/anysdk aot \
+            ./.stackql \
+            ./.stackql/src/${handle}/${version}/provider.yaml \
+            -v \
+            --mock-output-dir "cicd/out/auto-mocks/${provider}" \
+            --mock-expectation-dir "cicd/out/mock-expectations/${provider}" \
+            --mock-query-dir "cicd/out/mock-queries/${provider}" \
+            --schema-dir cicd/schema-definitions \
+            --stdout-file "cicd/out/aot/${RUN_TS}-${provider}-${version}-summary.json" \
+            --stderr-file "cicd/out/aot/${RUN_TS}-${provider}-${version}-analysis.jsonl" || true
+        done
+
+    - name: Generate Per-Resource Closures with Provider Docs
+      run: |
+        MOCK_PORT=5000
+
+        for provider_cfg in \
+          "aws|aws|${PROVIDER_VERSION_AWS}" \
+          "azure|azure|${PROVIDER_VERSION_AZURE}" \
+          "google|googleapis.com|${PROVIDER_VERSION_GOOGLE}"; do
+
+          IFS='|' read -r provider handle version <<< "$provider_cfg"
+
+          for svc_file in .stackql/src/${handle}/${version}/services/*.yaml; do
+            svc_name="$(basename "$svc_file" .yaml)"
+
+            ./build/anysdk closure \
+              ./.stackql \
+              ./.stackql/src/${handle}/${version}/provider.yaml \
+              "$svc_name" \
+              --provider "$provider" \
+              --rewrite-url "http://localhost:${MOCK_PORT}" \
+              --output-dir "cicd/out/closures/${provider}" \
+              2>/dev/null || true
+          done
+        done
 
     - name: Upload Provider Analysis results
       uses: actions/upload-artifact@v4
       if: always()
       with:
         name: provider-analysis-results
-        path: cicd/out/*.json*
+        path: cicd/out/aot/
+
+    - name: Upload Auto Mocks
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: auto-mocks
+        path: cicd/out/auto-mocks/
+
+    - name: Upload Mock Expectations
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: mock-expectations
+        path: cicd/out/mock-expectations/
+
+    - name: Upload Mock Queries
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: mock-queries
+        path: cicd/out/mock-queries/
+
+    - name: Upload Closures
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: closures
+        path: cicd/out/closures/
 
     - name: Authenticate to Google Cloud
       uses: google-github-actions/auth@v2
@@ -191,19 +247,21 @@ jobs:
     - name: Set up Cloud SDK
       uses: google-github-actions/setup-gcloud@v2
 
-    - name: Upload analysis artifacts to GCS
+    - name: Upload all artifacts to GCS
       run: |
         set -e
 
         DEST="gs://${ANALYSIS_RESULTS_BUCKET}/provider-analysis/${DATE_PATH}/${RUN_TS}"
 
         echo "Uploading to ${DEST}"
 
-        # upload everything
-        gcloud storage cp -r cicd/out "${DEST}/"
+        gcloud storage cp -r cicd/out/aot "${DEST}/"
+        gcloud storage cp -r cicd/out/auto-mocks "${DEST}/"
+        gcloud storage cp -r cicd/out/mock-expectations "${DEST}/"
+        gcloud storage cp -r cicd/out/mock-queries "${DEST}/"
+        gcloud storage cp -r cicd/out/closures "${DEST}/"
 
-        # optional: also write a pointer to latest
-        echo "{\"latest\": \"${DEST}/out\"}" > latest.json
+        echo "{\"latest\": \"${DEST}\"}" > latest.json
         gcloud storage cp latest.json "gs://${ANALYSIS_RESULTS_BUCKET}/provider-analysis/latest.json"
 
 

.gitignore

@@ -3,7 +3,7 @@
 opint
 .DS_Store
 dist/
-.venv/
+*.venv/
 __pycache__/
 *.py[co]
 stackql-core/

cicd/.gitignore

@@ -0,0 +1,2 @@
+prior/
+post/

cicd/mock-testing-requirements.txt

@@ -0,0 +1,10 @@
+PyYaml==6.0.2
+requests==2.32.3
+mistune==3.0.2
+psycopg2-binary==2.9.10
+psycopg[binary]>=3.1.16
+PyYaml==6.0.2
+requests==2.32.3
+robotframework==7.0.1
+sqlalchemy==1.4.44
+tabulate==0.9.0

cicd/out/auto-mocks/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

cicd/out/closures/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

cicd/out/generic/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

cicd/out/mock-expectations/aws/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

cicd/out/mock-queries/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

cicd/python/record_parser/README.md

@@ -0,0 +1,74 @@
+# record-parser
+
+Tiny example package.
+
+## Build with uv
+
+```bash
+uv --directory cicd/python/record_parser build
+```
+
+## Install editable
+
+```bash
+uv venv && uv pip install -e ./cicd/python/record_parser
+```
+
+## Use
+
+```python
+from record_parser import parse_record, generate_flask_app, generate_flask_apps_from_file, generate_mocks_from_analysis_run
+
+
+generate_mocks_from_analysis_run("test/assets/analysis-jsonl", "cicd/out/aot")
+
+
+for line in generate_flask_apps_from_file("test/assets/analysis-jsonl/single-entry-observations.jsonl"): print(json.dumps(line))
+
+```
+
+
+
+
+## De facto protocol
+
+
+
+```json
+{
+  "level": "warning",
+  "bin": "empty-response-unsafe",
+  "provider": "aws",
+  "service": "ec2",
+  "resource": "volumes_post_naively_presented",
+  "method": "describeVolumes",
+  "message": "response transform template accesses input directly without nil/empty guards — may fail on empty response bodies",
+  "prior_template": "{{ toJson . }}",
+  "fixed_template": "{{- if . -}}{{ toJson . }}{{- else -}}null{{- end -}}",
+  "empirical_tests": {
+    "results": [
+      {
+        "input": "",
+        "ok": true
+      },
+      {
+        "input": "<root/>",
+        "output": "{\"root\":\"\"}",
+        "ok": true
+      },
+      {
+        "input": "<root></root>",
+        "output": "{\"root\":\"\"}",
+        "ok": true
+      }
+    ]
+  },
+  "sample_response": {
+    "pre_transform": "<Response><DescribeVolumesResponse><NextToken>sample_string</NextToken><Volumes><item><AvailabilityZone>sample_string</AvailabilityZone><Attachments><item></item></Attachments><Size>0</Size><State>sample_string</State><FastRestored>false</FastRestored><CreateTime>sample_string</CreateTime><SnapshotId>sample_string</SnapshotId><Encrypted>false</Encrypted><Iops>0</Iops><Throughput>0</Throughput><KmsKeyId>sample_string</KmsKeyId><OutpostArn>sample_string</OutpostArn><VolumeType>sample_string</VolumeType><Tags><item></item></Tags><VolumeId>sample_string</VolumeId><MultiAttachEnabled>false</MultiAttachEnabled></item></Volumes></DescribeVolumesResponse></Response>",
+    "post_transform": "{\n  \"DescribeVolumesResponse\": {\n    \"NextToken\": \"sample_string\",\n    \"Volumes\": [\n      {\n        \"Attachments\": [\n          {}\n        ],\n        \"AvailabilityZone\": \"sample_string\",\n        \"CreateTime\": \"sample_string\",\n        \"Encrypted\": false,\n        \"FastRestored\": false,\n        \"Iops\": 0,\n        \"KmsKeyId\": \"sample_string\",\n        \"MultiAttachEnabled\": false,\n        \"OutpostArn\": \"sample_string\",\n        \"Size\": 0,\n        \"SnapshotId\": \"sample_string\",\n        \"State\": \"sample_string\",\n        \"Tags\": [\n          {}\n        ],\n        \"Throughput\": 0,\n        \"VolumeId\": \"sample_string\",\n        \"VolumeType\": \"sample_string\"\n      }\n    ]\n  }\n}"
+  },
+  "mock_route": "@app.route('/', methods=['POST'])\ndef aws_ec2_volumes_post_naively_presented_describevolumes():\n    if request.form.get('Action') == 'DescribeVolumes':\n        return Response(MOCK_RESPONSE_AWS_EC2_VOLUMES_POST_NAIVELY_PRESENTED_DESCRIBEVOLUMES, content_type='application/xml')",
+  "stackql_query": "SELECT * FROM aws.ec2.volumes_post_naively_presented WHERE region = 'dummy_region'",
+  "expected_response": "[\n  {\n    \"DescribeVolumesResponse\": {\n      \"NextToken\": \"sample_string\",\n      \"Volumes\": [\n        {\n          \"Attachments\": [\n            {}\n          ],\n          \"AvailabilityZone\": \"sample_string\",\n          \"CreateTime\": \"sample_string\",\n          \"Encrypted\": false,\n          \"FastRestored\": false,\n          \"Iops\": 0,\n          \"KmsKeyId\": \"sample_string\",\n          \"MultiAttachEnabled\": false,\n          \"OutpostArn\": \"sample_string\",\n          \"Size\": 0,\n          \"SnapshotId\": \"sample_string\",\n          \"State\": \"sample_string\",\n          \"Tags\": [\n            {}\n          ],\n          \"Throughput\": 0,\n          \"VolumeId\": \"sample_string\",\n          \"VolumeType\": \"sample_string\"\n        }\n      ]\n    }\n  }\n]"
+}
+```

cicd/python/record_parser/pyproject.toml

@@ -0,0 +1,12 @@
+[build-system]
+requires = ["hatchling==1.27.0"]
+build-backend = "hatchling.build"
+
+[project]
+name = "record-parser"
+version = "0.1.0"
+requires-python = ">=3.11"
+dependencies = []
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/record_parser"]

cicd/python/record_parser/src/record_parser/__init__.py

@@ -0,0 +1,8 @@
+from .core import parse_record, generate_flask_app, generate_flask_apps_from_file, generate_mocks_from_analysis_run
+
+__all__ = [
+    "parse_record", 
+    "generate_flask_app", 
+    "generate_flask_apps_from_file",
+    "generate_mocks_from_analysis_run",
+]

cicd/python/record_parser/src/record_parser/core.py

@@ -0,0 +1,45 @@
+import json
+import typing
+
+def parse_record(record: str) -> typing.Dict:
+    return json.loads(record)
+
+def generate_flask_app(parsed_record: typing.Dict) -> str:
+    app_code = f"""from flask import Flask, request, jsonify
+app = Flask(__name__)
+
+{parsed_record['sample_response']['var_name']} = "{parsed_record['sample_response']['pre_transform']}"
+
+{parsed_record['mock_route']}
+
+"""
+    return app_code
+
+
+def generate_flask_apps_from_file(file_path: str) -> typing.List[typing.Dict]:
+    apps = []
+    with open(file_path, 'r') as f:
+        for line in f:
+            if line.strip():  # Skip empty lines
+                parsed_record = parse_record(line)
+                if not parsed_record.get('mock_route'):
+                    continue  # Skip records without a mock route   
+                app_code = generate_flask_app(parsed_record)
+                apps.append({
+                    "parsed_record": parsed_record,
+                    "app_code": app_code
+                })
+    return apps
+
+def generate_mocks_from_analysis_run(src_dir: str, dest_dir: str):
+    import os
+    for filename in os.listdir(src_dir):
+        if filename.endswith('observations.jsonl'):
+            file_path = os.path.join(src_dir, filename)
+            apps = generate_flask_apps_from_file(file_path)
+            for app in apps:
+                mock_filename = f"{app['parsed_record']['provider']}_{app['parsed_record']['service']}_{app['parsed_record']['method']}_mock.py"
+                mock_file_path = os.path.join(dest_dir, mock_filename)
+                with open(mock_file_path, 'w') as f:
+                    f.write(app['app_code'])
+