diff --git a/.github/workflows/release-production-plugins.yml b/.github/workflows/release-production-plugins.yml index f85490b1e..e17f9a817 100644 --- a/.github/workflows/release-production-plugins.yml +++ b/.github/workflows/release-production-plugins.yml @@ -135,6 +135,14 @@ jobs: package: ./packages/contentstack-branches/package.json tag: latest + # Query Export + - name: Publishing query export (Production) + uses: JS-DevTools/npm-publish@v3 + with: + token: ${{ secrets.NPM_TOKEN }} + package: ./packages/contentstack-query-export/package.json + tag: latest + - name: Create Production Release id: create_release env: diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 73c9b72cc..eecc68d16 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -61,3 +61,7 @@ jobs: - name: Run tests for Contentstack Branches working-directory: ./packages/contentstack-branches run: npm run test:unit + + - name: Run tests for Contentstack Query Export + working-directory: ./packages/contentstack-query-export + run: npm run test:unit diff --git a/.talismanrc b/.talismanrc index e69de29bb..ffec31b23 100644 --- a/.talismanrc +++ b/.talismanrc @@ -0,0 +1,46 @@ +fileignoreconfig: + - filename: packages/contentstack-query-export/.env-example + checksum: 922c7aa9c788ab60b987de2b0a2aee6d90843c463a8bbc29201e4efe31081187 + - filename: packages/contentstack-query-export/.eslintrc + checksum: b34756122b251dc2feedc7c7b98a7772d4d763bc468c8291be483ae2ac3471be + - filename: packages/contentstack-query-export/README.md + checksum: 9be27e9a5f027f2bbbbcc6d4c706b19071cf40f596ce3e778f33ea7579a52626 + - filename: packages/contentstack-query-export/test/unit/common-helper.test.ts + checksum: c1d023d8c23e0400805448eb1466da5cb1fe891b6e838100fb12cbc7e1514a59 + - filename: packages/contentstack-query-export/test/config.json + checksum: 792e177efa078e31aa05a5136807fd1fed4b6ea7a4cd44d69353edd8b96ff33f + - filename: packages/contentstack-query-export/src/utils/logger.ts + checksum: de6dd816bc534aaddf9adbe4e1db935f152d32eedaad5b76445f4affa836fcc9 + - filename: packages/contentstack-query-export/src/utils/common-helper.ts + checksum: 924a9fbc57dd774a7957870d63366ffc16cd4242dbe684321b9b52a888cfa455 + - filename: packages/contentstack-query-export/test/unit/content-type-helper.test.ts + checksum: 1b4b9724a1281032605b61f007f7a7da080731bd9e0e4b2c4bc00b212ff30242 + - filename: packages/contentstack-query-export/src/commands/cm/stacks/export-query.ts + checksum: 874c30c717df26b8caae8f807b25b529f9704de9b93ef53d457a3a9575742cf4 + - filename: packages/contentstack-query-export/skills/framework/SKILL.md + checksum: b45c4bc28025292c168053e95a3c570b9d67500e0ee5241553089bca6914bb3f + - filename: packages/contentstack-query-export/skills/code-review/SKILL.md + checksum: 1c1cb0b1ce20114b9e855278a63c098d87f9302f093b08eb7f05f667840b6166 + - filename: packages/contentstack-query-export/src/types/index.ts + checksum: 686c5ed7fadb6620201dc3f1ed19c5ba94afd73ad165c33379b8b33dec81e519 + - filename: packages/contentstack-query-export/src/utils/config-handler.ts + checksum: 2a17dfe46ff5e77bb585013719065db0b513b21d700eb54e6615e78a6811f885 + - filename: packages/contentstack-query-export/src/utils/dependency-resolver.ts + checksum: 0c85da2a6fa43c8923c3659c45b02d8cf4cf43f6aae5e21e5b47955232dd5d78 + - filename: packages/contentstack-query-export/test/unit/module-exporter.test.ts + checksum: e27fab52e65a8d5430d268f3562a823828e9e3dd9eb9569342f1cdb83eef9ea3 + - filename: packages/contentstack-query-export/test/unit/query-parser-simple.test.ts + checksum: d187ad885a914b70406e343a92ad3ee1ca3c30207b0d8b040f36c6f287da3a6c + - filename: packages/contentstack-query-export/test/unit/dependency-resolver.test.ts + checksum: da6b003331ece39c106b61f0c660da678c7b2ca2f55cfbdd2a8f8a72c183573f + - filename: packages/contentstack-query-export/test/unit/config-handler.test.ts + checksum: a1077cb686431fea29de839762dbc16c951b6d61171f525e311e4a34182b0d08 + - filename: packages/contentstack-query-export/test/unit/referenced-asset-handler.test.ts + checksum: 3d19ad04a0306be741f9acd3a2d164d19e2b3803efc0a50342b156e8686c8b0c + - filename: packages/contentstack-query-export/src/core/query-executor.ts + checksum: 266751e299cda2a15d5e4e551eca72bcd6f0d600a0416f95410560257fc8210f + - filename: packages/contentstack-query-export/test/unit/query-executor.test.ts + checksum: 86713d035ff35d13089e14e8ad84db4191ad8b4f85bb759282b75e3dc96966cb + - filename: pnpm-lock.yaml + checksum: 66cc4bd169899eb1641197c1a95f50910c34601b0a695c23f0c2e6d26f836651 +version: '1.0' diff --git a/packages/contentstack-query-export/.env-example b/packages/contentstack-query-export/.env-example new file mode 100644 index 000000000..a55f16b1d --- /dev/null +++ b/packages/contentstack-query-export/.env-example @@ -0,0 +1 @@ +ENVIRONMENT=NON_PROD \ No newline at end of file diff --git a/packages/contentstack-query-export/.eslintignore b/packages/contentstack-query-export/.eslintignore new file mode 100644 index 000000000..5e42c8d63 --- /dev/null +++ b/packages/contentstack-query-export/.eslintignore @@ -0,0 +1,23 @@ +node_modules +.todo +.env +.dccache +logs +contents +lerna-debug.log +.DS_Store +contentTest +build +_backup* +oclif.manifest.json +.vscode +.nyc_output +contentstack-cli-logs +packages/**/package-lock.json +.dccache +yarn.lock +contents-* +*.http +*.todo +talisman_output.log +snyk_output.log \ No newline at end of file diff --git a/packages/contentstack-query-export/.eslintrc b/packages/contentstack-query-export/.eslintrc new file mode 100644 index 000000000..cb46553b0 --- /dev/null +++ b/packages/contentstack-query-export/.eslintrc @@ -0,0 +1,55 @@ +{ + "env": { + "node": true + }, + "parser": "@typescript-eslint/parser", + "parserOptions": { + "project": "tsconfig.json", + "sourceType": "module" + }, + "extends": [ + // "oclif", + "oclif-typescript", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + "@typescript-eslint/no-unused-vars": [ + "error", + { + "args": "none" + } + ], + "@typescript-eslint/prefer-namespace-keyword": "error", + "@typescript-eslint/quotes": [ + "error", + "single", + { + "avoidEscape": true, + "allowTemplateLiterals": true + } + ], + "semi": "off", + "@typescript-eslint/type-annotation-spacing": "error", + "@typescript-eslint/no-redeclare": "off", + "eqeqeq": [ + "error", + "smart" + ], + "id-match": "error", + "no-eval": "error", + "no-var": "error", + "quotes": "off", + "indent": "off", + "camelcase": "off", + "comma-dangle": "off", + "arrow-parens": "off", + "operator-linebreak": "off", + "object-curly-spacing": "off", + "node/no-missing-import": "off", + "padding-line-between-statements": "off", + "@typescript-eslint/ban-ts-ignore": "off", + "unicorn/no-abusive-eslint-disable": "off", + "unicorn/consistent-function-scoping": "off", + "@typescript-eslint/no-use-before-define": "off" + } +} \ No newline at end of file diff --git a/packages/contentstack-query-export/.gitignore b/packages/contentstack-query-export/.gitignore new file mode 100644 index 000000000..86eaed73e --- /dev/null +++ b/packages/contentstack-query-export/.gitignore @@ -0,0 +1,20 @@ +*-debug.log +*-error.log +/.nyc_output +/dist +/lib +/tmp +/yarn.lock +node_modules +.DS_Store +coverage +./contents +.vscode/ +/lib +.env +_backup_* +contents/ +logs/ +oclif.manifest.json +talisman_output.log +snyk_output.log diff --git a/packages/contentstack-query-export/.mocharc.json b/packages/contentstack-query-export/.mocharc.json new file mode 100644 index 000000000..b90d7f028 --- /dev/null +++ b/packages/contentstack-query-export/.mocharc.json @@ -0,0 +1,12 @@ +{ + "require": [ + "test/helpers/init.js", + "ts-node/register", + "source-map-support/register" + ], + "watch-extensions": [ + "ts" + ], + "recursive": true, + "timeout": 5000 +} \ No newline at end of file diff --git a/packages/contentstack-query-export/.nycrc.json b/packages/contentstack-query-export/.nycrc.json new file mode 100644 index 000000000..2ffb9c510 --- /dev/null +++ b/packages/contentstack-query-export/.nycrc.json @@ -0,0 +1,5 @@ +{ + "include": [ + "lib/**/*.js" + ] +} \ No newline at end of file diff --git a/packages/contentstack-query-export/.prettierignore b/packages/contentstack-query-export/.prettierignore new file mode 100644 index 000000000..70988e213 --- /dev/null +++ b/packages/contentstack-query-export/.prettierignore @@ -0,0 +1 @@ +**/README.md \ No newline at end of file diff --git a/packages/contentstack-query-export/.prettierrc b/packages/contentstack-query-export/.prettierrc new file mode 100644 index 000000000..ba93fc77d --- /dev/null +++ b/packages/contentstack-query-export/.prettierrc @@ -0,0 +1,7 @@ +{ + "semi": true, + "trailingComma": "all", + "singleQuote": true, + "printWidth": 120, + "tabWidth": 2 +} \ No newline at end of file diff --git a/packages/contentstack-query-export/.talismanrc b/packages/contentstack-query-export/.talismanrc new file mode 100644 index 000000000..2ee647a7c --- /dev/null +++ b/packages/contentstack-query-export/.talismanrc @@ -0,0 +1,14 @@ +fileignoreconfig: + - filename: package-lock.json + checksum: b4f6e41c55c3c7fa9d8b7ea903eb9bc85f27464306a428c97b7e96e86ffce5f7 + - filename: skills/testing/SKILL.md + checksum: da9831797a5e6a4d2e6e846c3f6d2583d84008d2dfd454dd7effe2f897c43a7b + - filename: skills/framework/SKILL.md + checksum: 217cf9123bf9b62a4d9b0c891af8ba388173c601c782ed0fea432f6be7ef5e56 + - filename: skills/contentstack-cli/SKILL.md + checksum: b42e3526fb902a31080824b776cc8e233646139ee0915d89c0925744d56d586f + - filename: skills/code-review/SKILL.md + checksum: 687efb830fb4a9fb2b2e6682db052771706ddd9b12dc9cec04943e3cade929b3 + - filename: skills/dev-workflow/SKILL.md + checksum: 03434201e3aaaa9239aff2f97826d64ac4ca31467b77c07330ac4b608ee24939 +version: '1.0' diff --git a/packages/contentstack-query-export/AGENTS.md b/packages/contentstack-query-export/AGENTS.md new file mode 100644 index 000000000..a50993c38 --- /dev/null +++ b/packages/contentstack-query-export/AGENTS.md @@ -0,0 +1,45 @@ +# CLI export-query plugin – Agent guide + +**Universal entry point** for contributors and AI agents. Detailed conventions live in **`skills/*/SKILL.md`**. + +## What this repo is + +| Field | Detail | +| --- | --- | +| **Name:** | `@contentstack/cli-cm-export-query` ([repository](https://github.com/contentstack/cli)) | +| **Purpose:** | OCLIF plugin for **query-based** stack export (`cm:stacks:export-query` / short `EXPRTQRY`); implements `QueryExporter` and related export flow. | +| **Out of scope (if any):** | Other export/import plugins live in sibling packages; this repo is only the query-export plugin. | + +## Tech stack (at a glance) + +| Area | Details | +| --- | --- | +| **Language** | TypeScript **^4.9** (`tsconfig.json`); Node **>= 14** (`engines`) | +| **Build** | `tsc -b` β†’ `lib/`; `prepack` runs compile + `oclif manifest` + `oclif readme`; copies `src/config` β†’ `lib/` | +| **Tests** | Mocha + Chai + Sinon; **nyc** coverage; tests under `test/**/*.test.ts` (see [skills/testing/SKILL.md](skills/testing/SKILL.md)) | +| **Lint / coverage** | ESLint `src/**/*.ts`; nyc in `npm test` | +| **Other** | OCLIF v4, Husky | + +## Commands (quick reference) + +| Command type | Command | +| --- | --- | +| **Build** | `npm run build` | +| **Test** | `npm test` | +| **Lint** | `npm run lint` | + +CI: [.github/workflows/unit-test.yml](.github/workflows/unit-test.yml); also `release.yml`, `sca-scan.yml`, `policy-scan.yml` under [.github/workflows/](.github/workflows/). + +## Where the documentation lives: skills + +| Skill | Path | What it covers | +| --- | --- | --- | +| Development workflow | [skills/dev-workflow/SKILL.md](skills/dev-workflow/SKILL.md) | CI, branches, Husky, PR expectations | +| Contentstack CLI | [skills/contentstack-cli/SKILL.md](skills/contentstack-cli/SKILL.md) | Commands, `QueryExporter`, APIs | +| Framework | [skills/framework/SKILL.md](skills/framework/SKILL.md) | Config, logging, errors, utilities | +| Testing | [skills/testing/SKILL.md](skills/testing/SKILL.md) | Mocha/Chai/Sinon, nyc, TDD | +| Code review | [skills/code-review/SKILL.md](skills/code-review/SKILL.md) | PR checklist | + +## Using Cursor (optional) + +If you use **Cursor**, [.cursor/rules/README.md](.cursor/rules/README.md) only points to **`AGENTS.md`**β€”same docs as everyone else. diff --git a/packages/contentstack-query-export/LICENSE b/packages/contentstack-query-export/LICENSE new file mode 100644 index 000000000..aff1142ee --- /dev/null +++ b/packages/contentstack-query-export/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Contentstack + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/contentstack-query-export/README.md b/packages/contentstack-query-export/README.md new file mode 100644 index 000000000..137aa8012 --- /dev/null +++ b/packages/contentstack-query-export/README.md @@ -0,0 +1,108 @@ +# Contentstack CLI Query Export Plugin + +A powerful CLI plugin for Contentstack that enables query-based content export with intelligent dependency resolution and asset reference detection. + +## Overview + +This plugin extends the Contentstack CLI to export content based on custom queries, automatically resolving dependencies between content types, global fields, extensions, and taxonomies. It intelligently detects and exports referenced assets to ensure complete content portability. + +## Features + +- πŸ” **Query-based Export**: Export content using custom queries instead of entire content types +- πŸ”— **Dependency Resolution**: Automatically resolve and export dependencies (global fields, extensions, taxonomies) +- πŸ–ΌοΈ **Asset Reference Detection**: Intelligent detection of asset references in various formats +- πŸ“ **Organized Output**: Well-structured export with separate folders for each module +- βš™οΈ **Configurable**: Support for external config files and flexible options +- 🌐 **Multi-locale Support**: Export content across different locales +- πŸ“Š **Export Metadata**: Comprehensive metadata tracking for export operations + +## Installation + +```bash +# Install as a Contentstack CLI plugin +npm install -g @contentstack/cli-cm-export-query + +# Or install locally +npm install @contentstack/cli-cm-export-query +``` + +## Usage + +### Basic Export + +```bash +# Export using management token alias +csdx cm:stacks:export-query -a -q "{'title': {'$exists': true}}" + +# Export using API key and management token +csdx cm:stacks:export-query --stack-api-key -A -q "{'title': {'$exists': true}}" +``` + +### Command Options + +| Flag | Description | Required | +|------|-------------|----------| +| `-a, --alias` | Management token alias | Yes (or use -A) | +| `-A, --management-token` | Management token | Yes (or use -a) | +| `--stack-api-key` | Stack API key | Yes | +| `-q, --query` | Query for content export | Yes | +| `-d, --data-dir` | Export directory path | No | +| `--branch` | Branch name | No | +| `--skip-references` | Skip reference resolution | No | +| `--skip-dependencies` | Skip dependency export | No | +| `--secured-assets` | Include secured assets | No | +| `--config` | External config file path | No | + +### Query Examples + +**Basic Content Query:** +```bash +csdx cm:stacks:export-query -a prod -q "{'title': {'$regex': 'blog'}}" +``` + +**Date Range Query:** +```bash +csdx cm:stacks:export-query -a prod -q "{'updated_at': {'$gte': '2024-01-01'}}" +``` + +**Complex Query:** +```bash +csdx cm:stacks:export-query -a prod -q "{'$and': [{'title': {'$exists': true}}, {'tags': {'$in': ['featured']}}]}" +``` + +## Configuration + +### Default Configuration + +The plugin includes a default configuration file at `src/config/export-defaults.json`: + +```json +{ + "skipReferences": false, + "skipDependencies": false, + "securedAssets": false, + "includeGlobalFieldSchema": true, + "includePublishDetails": true, + "includeDimension": false, + "fetchConcurrency": 5, + "writeConcurrency": 5, + "batchSize": 100 +} +``` + +### External Configuration + +Create a custom config file and pass it using the `--config` flag: + +```json +{ + "skipReferences": true, + "batchSize": 50, + "fetchConcurrency": 3, + "securedAssets": true +} +``` + +```bash +csdx cm:stacks:export-query -a prod -q "{'title': {'$exists': true}}" --config ./my-config.json +``` diff --git a/packages/contentstack-query-export/SECURITY.md b/packages/contentstack-query-export/SECURITY.md new file mode 100644 index 000000000..1f44e3424 --- /dev/null +++ b/packages/contentstack-query-export/SECURITY.md @@ -0,0 +1,27 @@ +## Security + +Contentstack takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations. + +If you believe you have found a security vulnerability in any Contentstack-owned repository, please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Send email to [security@contentstack.com](mailto:security@contentstack.com). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + +- Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) +- Full paths of source file(s) related to the manifestation of the issue +- The location of the affected source code (tag/branch/commit or direct URL) +- Any special configuration required to reproduce the issue +- Step-by-step instructions to reproduce the issue +- Proof-of-concept or exploit code (if possible) +- Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +[https://www.contentstack.com/trust/](https://www.contentstack.com/trust/) diff --git a/packages/contentstack-query-export/bin/dev.cmd b/packages/contentstack-query-export/bin/dev.cmd new file mode 100644 index 000000000..077b57ae7 --- /dev/null +++ b/packages/contentstack-query-export/bin/dev.cmd @@ -0,0 +1,3 @@ +@echo off + +node "%~dp0\dev" %* \ No newline at end of file diff --git a/packages/contentstack-query-export/bin/dev.js b/packages/contentstack-query-export/bin/dev.js new file mode 100755 index 000000000..b2e3af809 --- /dev/null +++ b/packages/contentstack-query-export/bin/dev.js @@ -0,0 +1,6 @@ +#!/usr/bin/env node_modules/.bin/ts-node +// eslint-disable-next-line node/shebang, unicorn/prefer-top-level-await +(async () => { + const oclif = await import('@oclif/core'); + await oclif.execute({ development: true, dir: __dirname }); +})(); diff --git a/packages/contentstack-query-export/bin/run.cmd b/packages/contentstack-query-export/bin/run.cmd new file mode 100644 index 000000000..968fc3075 --- /dev/null +++ b/packages/contentstack-query-export/bin/run.cmd @@ -0,0 +1,3 @@ +@echo off + +node "%~dp0\run" %* diff --git a/packages/contentstack-query-export/bin/run.js b/packages/contentstack-query-export/bin/run.js new file mode 100755 index 000000000..8baf30239 --- /dev/null +++ b/packages/contentstack-query-export/bin/run.js @@ -0,0 +1,7 @@ +#!/usr/bin/env node + +// eslint-disable-next-line unicorn/prefer-top-level-await +(async () => { + const oclif = await import('@oclif/core'); + await oclif.execute({ development: false, dir: __dirname }); +})(); diff --git a/packages/contentstack-query-export/messages/index.json b/packages/contentstack-query-export/messages/index.json new file mode 100644 index 000000000..21a72c884 --- /dev/null +++ b/packages/contentstack-query-export/messages/index.json @@ -0,0 +1,75 @@ +{ + "ASSET_EXPORT_COMPLETE": "Asset export process completed successfully", + "ASSET_FOLDERS_EXPORT_COMPLETE": "Asset folder structure exported successfully", + "ASSET_METADATA_EXPORT_COMPLETE": "Asset metadata exported successfully", + "ASSET_VERSIONED_METADATA_EXPORT_COMPLETE": "Versioned asset metadata exported successfully", + "ASSET_DOWNLOAD_COMPLETE": "Asset download completed successfully", + "ASSET_DOWNLOAD_SUCCESS": "Asset '%s' (UID: %s) downloaded successfully", + "ASSET_DOWNLOAD_FAILED": "Failed to download asset '%s' (UID: %s)", + "ASSET_WRITE_FAILED": "Failed to write asset file '%s' (UID: %s)", + "ASSET_QUERY_FAILED": "Failed to query asset data from the API", + "ASSET_VERSIONED_QUERY_FAILED": "Failed to query versioned asset data from the API", + "ASSET_COUNT_QUERY_FAILED": "Failed to retrieve total asset count", + + "CONTENT_TYPE_EXPORT_COMPLETE": "Content types exported successfully", + "CONTENT_TYPE_NO_TYPES": "No content types found", + "CONTENT_TYPE_EXPORT_FAILED": "Failed to export content types", + "CONTENT_TYPE_NO_TYPES_RETURNED": "API returned no content types for the given query", + + "ENVIRONMENT_EXPORT_COMPLETE": "Successfully exported %s environment(s)", + "ENVIRONMENT_EXPORT_SUCCESS": "Environment '%s' exported successfully", + "ENVIRONMENT_NOT_FOUND": "No environments found in the current stack", + + "EXTENSION_EXPORT_COMPLETE": "Successfully exported %s extension(s)", + "EXTENSION_EXPORT_SUCCESS": "Extension '%s' exported successfully", + "EXTENSION_NOT_FOUND": "No extensions found in the current stack", + + "GLOBAL_FIELDS_EXPORT_COMPLETE": "Successfully exported %s global field(s)", + + "LABELS_EXPORT_COMPLETE": "Successfully exported %s label(s)", + "LABEL_EXPORT_SUCCESS": "Label '%s' exported successfully", + "LABELS_NOT_FOUND": "No labels found in the current stack", + + "LOCALES_EXPORT_COMPLETE": "Successfully exported %s locale(s) including %s master locale(s)", + + "TAXONOMY_EXPORT_COMPLETE": "Successfully exported %s taxonomy entries", + "TAXONOMY_EXPORT_SUCCESS": "Taxonomy '%s' exported successfully", + "TAXONOMY_NOT_FOUND": "No taxonomies found in the current stack", + + "WEBHOOK_EXPORT_COMPLETE": "Successfully exported %s webhook(s)", + "WEBHOOK_EXPORT_SUCCESS": "Webhook '%s' exported successfully", + "WEBHOOK_NOT_FOUND": "No webhooks found in the current stack", + + "WORKFLOW_EXPORT_COMPLETE": "Successfully exported %s workflow(s)", + "WORKFLOW_EXPORT_SUCCESS": "Workflow '%s' exported successfully", + "WORKFLOW_NOT_FOUND": "No workflows found in the current stack", + + "PERSONALIZE_URL_NOT_SET": "Cannot export Personalize project: URL not configured", + "PERSONALIZE_SKIPPING_WITH_MANAGEMENT_TOKEN": "Skipping Personalize project export: Management token not supported", + "PERSONALIZE_MODULE_NOT_IMPLEMENTED": "Module '%s' implementation not found", + "PERSONALIZE_NOT_ENABLED": "Personalize feature is not enabled for this organization", + + "MARKETPLACE_APPS_EXPORT_COMPLETE": "Successfully exported %s marketplace app(s)", + "MARKETPLACE_APP_CONFIG_EXPORT": "Exporting configuration for app '%s'", + "MARKETPLACE_APP_CONFIG_SUCCESS": "Successfully exported configuration for app '%s'", + "MARKETPLACE_APP_EXPORT_SUCCESS": "Successfully exported app '%s'", + "MARKETPLACE_APPS_NOT_FOUND": "No marketplace apps found in the current stack", + "MARKETPLACE_APP_CONFIG_EXPORT_FAILED": "Failed to export configuration for app '%s'", + "MARKETPLACE_APP_MANIFEST_EXPORT_FAILED": "Failed to export manifest for app '%s'", + + "COMPOSABLE_STUDIO_EXPORT_START": "Starting Studio project export...", + "COMPOSABLE_STUDIO_NOT_FOUND": "No Studio project found for this stack", + "COMPOSABLE_STUDIO_EXPORT_COMPLETE": "Successfully exported Studio project '%s'", + "COMPOSABLE_STUDIO_EXPORT_FAILED": "Failed to export Studio project: %s", + "COMPOSABLE_STUDIO_AUTH_REQUIRED": "To export Studio projects, you must be logged in", + + "ENTRIES_EXPORT_COMPLETE": "Successfully exported entries (Content Type: %s, Locale: %s)", + "ENTRIES_EXPORT_SUCCESS": "All entries exported successfully", + "ENTRIES_VERSIONED_EXPORT_SUCCESS": "Successfully exported versioned entry (Content Type: %s, UID: %s, Locale: %s)", + "ENTRIES_EXPORT_VERSIONS_FAILED": "Failed to export versions for content type '%s' (UID: %s)", + + "BRANCH_EXPORT_FAILED": "Failed to export contents from branch (UID: %s)", + + "ROLES_NO_CUSTOM_ROLES": "No custom roles found in the current stack", + "ROLES_EXPORTING_ROLE": "Exporting role '%s'" +} diff --git a/packages/contentstack-query-export/package.json b/packages/contentstack-query-export/package.json new file mode 100644 index 000000000..5a4f0aaa8 --- /dev/null +++ b/packages/contentstack-query-export/package.json @@ -0,0 +1,102 @@ +{ + "name": "@contentstack/cli-cm-export-query", + "description": "Contentstack CLI plugin to export content from stack", + "version": "1.0.0", + "author": "Contentstack", + "bugs": "https://github.com/contentstack/cli/issues", + "dependencies": { + "@contentstack/cli-variants": "~1.4.2", + "@contentstack/cli-cm-export": "~1.24.2", + "@contentstack/cli-command": "~1.8.1", + "@contentstack/cli-utilities": "~1.18.2", + "@oclif/core": "^4.10.5", + "async": "^3.2.6", + "big-json": "^3.2.0", + "bluebird": "^3.7.2", + "lodash": "^4.18.1", + "merge": "^2.1.1", + "mkdirp": "^1.0.4", + "progress-stream": "^2.0.0", + "promise-limit": "^2.7.0", + "tslib": "^2.8.1", + "winston": "^3.19.0" + }, + "overrides": { + "brace-expansion": "^5.0.5" + }, + "devDependencies": { + "@contentstack/cli-dev-dependencies": "~1.3.1", + "@oclif/plugin-help": "^6.2.44", + "@oclif/test": "^4.1.18", + "@types/big-json": "^3.2.5", + "@types/chai": "^4.3.20", + "@types/mkdirp": "^1.0.2", + "@types/mocha": "^10.0.10", + "@types/node": "^20.19.39", + "@types/progress-stream": "^2.0.5", + "@types/sinon": "^17.0.4", + "chai": "^4.5.0", + "dotenv": "^16.6.1", + "dotenv-expand": "^9.0.0", + "eslint": "^8.57.1", + "eslint-config-oclif": "^6.0.157", + "husky": "^9.1.7", + "mocha": "10.8.2", + "nyc": "^15.1.0", + "oclif": "^4.17.46", + "sinon": "^17.0.2", + "ts-node": "^10.9.2", + "typescript": "^4.9.5" + }, + "scripts": { + "build": "pnpm compile && pnpm copy-config && oclif manifest && oclif readme", + "clean": "rm -rf ./lib ./node_modules tsconfig.tsbuildinfo", + "compile": "tsc -b tsconfig.json", + "postpack": "rm -f oclif.manifest.json", + "copy-config": "cp -r src/config lib/", + "prepack": "pnpm compile && pnpm copy-config && oclif manifest && oclif readme", + "version": "oclif readme && git add README.md", + "test:report": "tsc -p test && nyc --reporter=lcov --extension .ts mocha --forbid-only \"test/**/*.test.ts\"", + "pretest": "tsc -p test", + "test": "nyc --extension .ts mocha --forbid-only \"test/**/*.test.ts\"", + "posttest": "npm run lint", + "lint": "eslint src/**/*.ts", + "format": "eslint src/**/*.ts --fix", + "test:integration": "INTEGRATION_TEST=true mocha --config ./test/.mocharc.js --forbid-only \"test/run.test.js\"", + "test:integration:report": "INTEGRATION_TEST=true nyc --extension .js mocha --forbid-only \"test/run.test.js\"", + "test:unit": "mocha --forbid-only \"test/unit/**/*.test.ts\"", + "test:unit:report": "nyc --reporter=text --extension .ts mocha --forbid-only \"test/unit/**/*.test.ts\"" + }, + "engines": { + "node": ">=14.0.0" + }, + "files": [ + "/lib", + "/messages", + "/npm-shrinkwrap.json", + "/oclif.manifest.json" + ], + "homepage": "https://github.com/contentstack/cli", + "keywords": [ + "contentstack", + "cli", + "plugin" + ], + "license": "MIT", + "main": "./lib/commands/cm/stacks/export-query.js", + "oclif": { + "commands": "./lib/commands", + "bin": "csdx", + "devPlugins": [ + "@oclif/plugin-help" + ], + "repositoryPrefix": "<%- repo %>/blob/main/packages/contentstack-query-export/<%- commandPath %>" + }, + "csdxConfig": { + "shortCommandName": { + "cm:stacks:export-query": "EXPRTQRY", + "cm:export:query": "EXPRTQRY" + } + }, + "repository": "https://github.com/contentstack/cli" +} diff --git a/packages/contentstack-query-export/skills/README.md b/packages/contentstack-query-export/skills/README.md new file mode 100644 index 000000000..781157144 --- /dev/null +++ b/packages/contentstack-query-export/skills/README.md @@ -0,0 +1,3 @@ +# Skills – CLI export-query + +Source of truth for detailed guidance. Read [AGENTS.md](../AGENTS.md) for the skill index, then open the `SKILL.md` that matches your task. Each folder contains `SKILL.md` with YAML frontmatter (`name`, `description`). diff --git a/packages/contentstack-query-export/skills/code-review/SKILL.md b/packages/contentstack-query-export/skills/code-review/SKILL.md new file mode 100644 index 000000000..ad0a78e21 --- /dev/null +++ b/packages/contentstack-query-export/skills/code-review/SKILL.md @@ -0,0 +1,204 @@ +--- +name: code-review +description: PR review checklist for this repo and similar CLI plugins. Use when reviewing changes to export-query, core, or utils. +--- + +# Code Review Skill + +Use the **Quick checklist template** for a short PR paste. Numbered sections **1**–**7** and **Review checklist summary** below are the full deep review for **`@contentstack/cli-cm-export-query`** and related CLI work. + +## Quick checklist template + +```markdown +## Security Review +- [ ] No hardcoded secrets +- [ ] Input validation present +- [ ] Error handling secure + +## Correctness Review +- [ ] Logic correctly implemented +- [ ] Edge cases handled +- [ ] Error scenarios covered + +## Architecture Review +- [ ] Proper code organization +- [ ] Design patterns followed +- [ ] Good modularity + +## Performance Review +- [ ] Efficient implementation +- [ ] Resource management +- [ ] Appropriate concurrency + +## Testing Review +- [ ] Adequate tests for behavior changed +- [ ] Quality tests +- [ ] Test-first used where practical for new behavior + +## Code Conventions +- [ ] TypeScript standards +- [ ] Code style consistent +- [ ] Documentation adequate +``` + +## 1. Security Review + +### Authentication & Authorization +- [ ] No hardcoded API keys, tokens, or credentials +- [ ] Sensitive data not logged to console or files +- [ ] Proper token validation and expiration handling +- [ ] Environment variables used for secrets + +### Input Validation +- [ ] All user inputs validated and sanitized +- [ ] Command flags properly validated +- [ ] File paths sanitized to prevent directory traversal +- [ ] API responses validated before processing + +### Error Handling +- [ ] Errors don't expose sensitive information +- [ ] Stack traces filtered in production +- [ ] Proper error logging without secrets + +## 2. Correctness Review + +### Logic Validation +- [ ] Business logic correctly implemented +- [ ] Edge cases handled appropriately +- [ ] Null/undefined checks in place +- [ ] Async operations properly awaited + +### Error Scenarios +- [ ] Network failures handled gracefully +- [ ] Rate limiting respected and handled +- [ ] Partial failures in batch operations managed +- [ ] Retry logic implemented correctly + +### Data Integrity +- [ ] Data transformations are reversible where needed +- [ ] Batch operations maintain consistency +- [ ] Rollback mechanisms for critical operations +- [ ] Proper validation before destructive operations + +## 3. Architecture Review + +### Code Organization +- [ ] Proper separation of concerns (Commands β†’ Services β†’ Utils) +- [ ] Single responsibility principle followed +- [ ] Dependencies injected, not hardcoded +- [ ] Interfaces used for abstractions + +### Design Patterns +- [ ] Consistent error handling patterns +- [ ] Proper use of async/await +- [ ] Service layer properly abstracted +- [ ] Configuration management centralized + +### Modularity +- [ ] Functions are focused and testable +- [ ] Classes have clear responsibilities +- [ ] Modules are loosely coupled +- [ ] Common functionality extracted to utilities + +## 4. Performance Review + +### Efficiency +- [ ] Unnecessary API calls eliminated +- [ ] Export/query work batched or paginated where the Management API requires it +- [ ] Proper pagination implemented when listing large result sets +- [ ] Rate limiting respected + +### Resource Management +- [ ] Memory usage optimized for large datasets +- [ ] File handles properly closed +- [ ] Network connections cleaned up +- [ ] No memory leaks in long-running operations + +### Concurrency +- [ ] Appropriate concurrency limits set +- [ ] Race conditions avoided +- [ ] Deadlocks prevented +- [ ] Resource contention minimized + +## 5. Testing Review + +### Test Coverage +- [ ] All new/modified code has tests +- [ ] Both success and failure paths tested +- [ ] Edge cases covered +- [ ] Integration tests for complex workflows + +### Test Quality +- [ ] Tests are focused and readable +- [ ] Proper mocking of external dependencies +- [ ] No real API calls in tests +- [ ] Test data is realistic and maintainable + +### TDD / test discipline +- [ ] New behavior covered by tests where practical (test-first preferred, not mandatory for refactors/docs) +- [ ] Tests fail appropriately when code is broken +- [ ] Tests are independent and can run in any order +- [ ] No test.skip or .only in committed code + +## 6. CLI-Specific Review + +### OCLIF Command Structure +- [ ] Extends appropriate base command class +- [ ] Proper flag definitions with validation +- [ ] Clear command description and examples +- [ ] Appropriate error handling with user-friendly messages + +### User Experience +- [ ] Progress indicators for long operations +- [ ] Clear success/failure messaging +- [ ] Proper use of colors and formatting +- [ ] Confirmation prompts for destructive actions + +### Command patterns +- [ ] Input validation before processing +- [ ] Heavy logic delegated to `src/core/` and `src/utils/` (not the command class) +- [ ] Proper logging for debugging +- [ ] Graceful handling of interruptions + +## 7. Contentstack Integration Review + +### API Usage +- [ ] Proper authentication using CLI utilities +- [ ] Rate limiting respected (10 req/sec for Management API) +- [ ] Appropriate error handling for API-specific errors +- [ ] Retry logic for transient failures + +### Query export behavior +- [ ] Query parsing and flags behave as documented +- [ ] Dependency / reference / asset handling respects `skip-*` flags +- [ ] Failures surface clearly to the user (no silent drops) +- [ ] Logging useful for support without leaking secrets + +### Environment Management +- [ ] Environment validation before operations +- [ ] Cross-environment operations handled safely +- [ ] Proper handling of environment-specific configurations +- [ ] Content type validation + +## Review Checklist Summary + +### Before Approving +- [ ] All critical issues resolved +- [ ] Tests pass; coverage reasonable for the change (~80% repo-wide is aspirational) +- [ ] Security concerns addressed +- [ ] Performance implications considered +- [ ] Documentation updated if needed +- [ ] Breaking changes properly communicated + +### Review Quality +- [ ] Code thoroughly examined, not just skimmed +- [ ] Constructive feedback provided +- [ ] Questions asked for unclear implementations +- [ ] Best practices enforced consistently +- [ ] Knowledge shared through comments + +### Post-Review +- [ ] Appropriate merge strategy selected +- [ ] Deployment considerations discussed +- [ ] Team notified of significant changes +- [ ] Follow-up tasks created if needed \ No newline at end of file diff --git a/packages/contentstack-query-export/skills/contentstack-cli/SKILL.md b/packages/contentstack-query-export/skills/contentstack-cli/SKILL.md new file mode 100644 index 000000000..f78d3ee40 --- /dev/null +++ b/packages/contentstack-query-export/skills/contentstack-cli/SKILL.md @@ -0,0 +1,121 @@ +--- +name: contentstack-cli +description: Contentstack CLI query-export plugin β€” OCLIF command, QueryExporter, cli-utilities, and export behavior. Use for commands, core export logic, utils, and API usage in this repo. +--- + +# Contentstack CLI β€” query export + +Guidance for **`@contentstack/cli-cm-export-query`**: query-driven export with dependency and asset handling. + +## This package + +- **Command:** `ExportQueryCommand` in `src/commands/cm/stacks/export-query.ts` extends **`Command`** from **`@contentstack/cli-command`**. +- **Orchestration:** **`QueryExporter`** and **`ModuleExporter`** in **`src/core/`**. +- **Helpers:** Query parsing, config, dependencies, assets, branches under **`src/utils/`**. +- **Integration:** **`@contentstack/cli-cm-export`**, **`@contentstack/cli-utilities`**, **`@oclif/core`** (transitive / manifest). + +## Practices + +- Authenticate and build the management client via **`@contentstack/cli-utilities`**; never log secrets. +- Keep **`run()`** thin; delegate to **`QueryExporter`** and existing utils. +- Respect rate limits and handle **429** / transient errors when adding API calls. +- Tests: mock SDK and file I/O; no real stack access in unit tests. + +## Repository layout + +| Area | Role | +|------|------| +| `src/commands/cm/stacks/export-query.ts` | CLI entry: flags, config setup, `QueryExporter` | +| `src/core/query-executor.ts` | `QueryExporter` β€” main export pipeline | +| `src/core/module-exporter.ts` | `ModuleExporter` β€” module export details | +| `src/utils/` | Query parser, config, branches, dependencies, assets, files, logger | +| `src/types/index.ts` | Shared types (e.g. `QueryExportConfig`, `Modules`) | +| `src/config/` | Defaults (copied to `lib/` on build) | + +There is **no** `src/services/` directory in this repo. + +## Command pattern + +Use **`@contentstack/cli-command`** and **`@contentstack/cli-utilities`**: + +```typescript +import { Command } from '@contentstack/cli-command'; +import { + flags, + FlagInput, + managementSDKClient, + log, + handleAndLogError, +} from '@contentstack/cli-utilities'; +import { QueryExporter } from '../../../core/query-executor'; + +export default class ExportQueryCommand extends Command { + static description = 'Export content from a stack using query-based filtering'; + + static flags: FlagInput = { + query: flags.string({ + required: true, + description: 'Query as JSON string or file path', + }), + alias: flags.string({ char: 'a', description: 'Management token alias' }), + // ...see export-query.ts for full flags + }; + + async run(): Promise { + const { flags } = await this.parse(ExportQueryCommand); + // setupQueryExportConfig(flags), managementSDKClient(...), then: + // const exporter = new QueryExporter(client, exportQueryConfig); + // await exporter.execute(); + } +} +``` + +### Conventions + +- Validate required inputs early (`query`, stack credentials). +- Use **`log`** with export **context** objects for structured messages. +- Use **`handleAndLogError`** for consistent error reporting where the codebase already does. + +## Export pipeline (conceptual) + +1. **Parse** query via **`QueryParser`** (JSON string or path to JSON file). +2. **Export** general and queried modules (aligned with **`@contentstack/cli-cm-export`**). +3. **Resolve** content types, references, and assets unless disabled (`skip-references`, `skip-dependencies`, `secured-assets`). + +When extending behavior, prefer new methods on **`QueryExporter`** / **`ModuleExporter`** or focused utils under **`src/utils/`**. + +## Authentication and secrets + +- Resolve tokens through CLI utilities and command flags; do not print management tokens or API keys. +- Do not write secrets into export directories. + +## API and rate limits + +- Contentstack APIs are rate-limited; use delays or backoff on **429** when introducing new call patterns. +- In tests, stub **`managementSDKClient`**, stack client methods, and **`fsUtil`** as the existing unit tests do. + +--- + +## Other CLI plugins (context) + +Other Contentstack CLI packages sometimes use **`BaseBulkCommand`**, batch processors, or JSON logs under `bulk-operation/`. **This query-export plugin does not use those patterns.** + +### Rate limit sketch (generic) + +```typescript +class RateLimiter { + private lastRequest = 0; + private readonly minIntervalMs = 100; // order-of-magnitude; tune per API guidance + + async wait(): Promise { + const now = Date.now(); + const elapsed = now - this.lastRequest; + if (elapsed < this.minIntervalMs) { + await new Promise((r) => setTimeout(r, this.minIntervalMs - elapsed)); + } + this.lastRequest = Date.now(); + } +} +``` + +Adapt to whatever **`@contentstack/cli-utilities`** or **`@contentstack/cli-cm-export`** already provides before adding parallel limiters. diff --git a/packages/contentstack-query-export/skills/dev-workflow/SKILL.md b/packages/contentstack-query-export/skills/dev-workflow/SKILL.md new file mode 100644 index 000000000..2845935ee --- /dev/null +++ b/packages/contentstack-query-export/skills/dev-workflow/SKILL.md @@ -0,0 +1,35 @@ +--- +name: dev-workflow +description: CI, Husky hooks, branch and PR expectations for the cli-cm-export-query plugin repo. +--- + +# Development workflow – CLI export-query + +## When to use + +- Running builds/tests before a PR +- Understanding which GitHub Actions run on this package +- Husky / pre-commit expectations + +## Commands + +| Command | Purpose | +| --- | --- | +| `npm run build` | Clean, install, compile, copy `src/config` β†’ `lib/` | +| `npm test` | `pretest` compiles tests; nyc + mocha `test/**/*.test.ts` | +| `npm run test:unit` | Mocha `test/unit/**/*.test.ts` only | +| `npm run lint` | ESLint `src/**/*.ts` | +| `npm run prepack` | Compile + OCLIF manifest/readme + config copy (release path) | + +## CI + +Workflows under [`.github/workflows/`](../../../.github/workflows/): e.g. `unit-test.yml`, `release.yml`, `sca-scan.yml`, `policy-scan.yml`. + +## Git hooks + +- `prepare` runs Husky setup (see `package.json`); hooks live under [`.husky/`](../../../.husky/) when configured. + +## PR expectations + +- Tests and lint pass; no `describe.only` / `it.only` (`--forbid-only` in test scripts). +- Coordinate with [testing](../testing/SKILL.md) and [code-review](../code-review/SKILL.md). diff --git a/packages/contentstack-query-export/skills/framework/SKILL.md b/packages/contentstack-query-export/skills/framework/SKILL.md new file mode 100644 index 000000000..6c75b92c4 --- /dev/null +++ b/packages/contentstack-query-export/skills/framework/SKILL.md @@ -0,0 +1,195 @@ +--- +name: framework +description: Utilities, configuration, logging, and error patterns for @contentstack/cli-cm-export-query. Use when working in src/utils/, config, or shared helpers β€” align with @contentstack/cli-utilities where possible. +--- + +# Framework Patterns + +Core utilities, configuration, logging, and error-handling patterns for **`@contentstack/cli-cm-export-query`** (and similar CLI plugins). Prefer matching patterns already in **`src/utils/`** and **`@contentstack/cli-utilities`** before introducing new abstractions. + +## Configuration Management + +```typescript +export interface AppConfig { + contentstack: { apiKey: string; authToken: string; region: string; }; + batch: { defaultSize: number; maxConcurrency: number; retryAttempts: number; }; + logging: { level: string; format: string; }; +} + +export class ConfigBuilder { + static build(): AppConfig { + return { + contentstack: { apiKey: process.env.CONTENTSTACK_API_KEY!, authToken: process.env.CONTENTSTACK_AUTH_TOKEN!, region: process.env.CONTENTSTACK_REGION || 'us' }, + batch: { defaultSize: parseInt(process.env.BATCH_SIZE || '10'), maxConcurrency: parseInt(process.env.MAX_CONCURRENCY || '3'), retryAttempts: parseInt(process.env.RETRY_ATTEMPTS || '3') }, + logging: { level: process.env.LOG_LEVEL || 'info', format: process.env.LOG_FORMAT || 'json' } + }; + } + static validate(config: AppConfig): void { + if (!config.contentstack.apiKey) throw new Error('CONTENTSTACK_API_KEY is required'); + if (!config.contentstack.authToken) throw new Error('CONTENTSTACK_AUTH_TOKEN is required'); + } +} +``` + +## Logging Framework + +```typescript +export interface Logger { debug(message: string, meta?: object): void; info(message: string, meta?: object): void; warn(message: string, meta?: object): void; error(message: string, meta?: object): void; } + +export class ConsoleLogger implements Logger { + constructor(private level: string = 'info') {} + debug(message: string, meta?: object): void { if (this.shouldLog('debug')) console.debug(this.format('DEBUG', message, meta)); } + info(message: string, meta?: object): void { if (this.shouldLog('info')) console.info(this.format('INFO', message, meta)); } + warn(message: string, meta?: object): void { if (this.shouldLog('warn')) console.warn(this.format('WARN', message, meta)); } + error(message: string, meta?: object): void { if (this.shouldLog('error')) console.error(this.format('ERROR', message, meta)); } + + private shouldLog(level: string): boolean { const levels = ['debug', 'info', 'warn', 'error']; return levels.indexOf(level) >= levels.indexOf(this.level); } + private format(level: string, message: string, meta?: object): string { return JSON.stringify({ timestamp: new Date().toISOString(), level, message, ...meta }); } +} +``` + +## Error Handling Framework + +```typescript +export abstract class BaseError extends Error { + abstract readonly code: string; abstract readonly category: 'validation' | 'api' | 'system' | 'user'; + constructor(message: string, public readonly context?: Record, public readonly cause?: Error) { super(message); this.name = this.constructor.name; } +} + +export class ValidationError extends BaseError { readonly code = 'VALIDATION_ERROR'; readonly category = 'validation' as const; } + +export class ApiError extends BaseError { + readonly code = 'API_ERROR'; readonly category = 'api' as const; + constructor(message: string, public readonly status?: number, context?: Record, cause?: Error) { super(message, { ...context, status }, cause); } +} + +export class ContentstackApiError extends ApiError { + readonly code = 'CONTENTSTACK_API_ERROR'; + static fromResponse(response: any, context?: Record): ContentstackApiError { + return new ContentstackApiError(response.error_message || 'API request failed', response.error_code, { ...context, errorCode: response.error_code, details: response.errors }); + } +} +``` + +## Utility Classes + +### Rate Limiter +```typescript +export class RateLimiter { + private queue: Array<() => void> = []; private running = 0; private lastRequest = 0; + constructor(private maxConcurrent: number = 1, private minInterval: number = 100) {} + + async execute(operation: () => Promise): Promise { + return new Promise((resolve, reject) => { + this.queue.push(async () => { + try { await this.waitForInterval(); this.running++; const result = await operation(); resolve(result); } + catch (error) { reject(error); } finally { this.running--; this.processQueue(); } + }); + this.processQueue(); + }); + } + + private processQueue(): void { if (this.running < this.maxConcurrent && this.queue.length > 0) this.queue.shift()!(); } + private async waitForInterval(): Promise { + const now = Date.now(); const elapsed = now - this.lastRequest; + if (elapsed < this.minInterval) await new Promise(resolve => setTimeout(resolve, this.minInterval - elapsed)); + this.lastRequest = Date.now(); + } +} +``` + +### Retry Strategy +```typescript +export interface RetryOptions { maxAttempts: number; initialDelay: number; maxDelay: number; backoffFactor: number; retryCondition?: (error: any) => boolean; } + +export class RetryStrategy { + constructor(private options: RetryOptions) {} + async execute(operation: () => Promise): Promise { + let lastError: any; let delay = this.options.initialDelay; + for (let attempt = 1; attempt <= this.options.maxAttempts; attempt++) { + try { return await operation(); } + catch (error) { + lastError = error; if (this.options.retryCondition && !this.options.retryCondition(error)) throw error; + if (attempt === this.options.maxAttempts) break; await new Promise(resolve => setTimeout(resolve, delay)); + delay = Math.min(delay * this.options.backoffFactor, this.options.maxDelay); + } + } + throw lastError; + } + + static forContentstack(): RetryStrategy { + return new RetryStrategy({ maxAttempts: 3, initialDelay: 1000, maxDelay: 10000, backoffFactor: 2, retryCondition: (error) => error.status === 429 || (error.status >= 500 && error.status < 600) }); + } +} +``` + +### Batch Processor +```typescript +export interface BatchOptions { batchSize: number; concurrency: number; processor: (item: T) => Promise; onProgress?: (completed: number, total: number) => void; } + +export class BatchProcessor { + static async process(items: T[], options: BatchOptions): Promise { + const batches = this.chunk(items, options.batchSize); const allResults: any[] = []; let completed = 0; + const processBatch = async (batch: T[]): Promise => { + const results = await Promise.allSettled(batch.map(options.processor)); allResults.push(...results); completed += batch.length; options.onProgress?.(completed, items.length); + }; + const semaphore = new Semaphore(options.concurrency); await Promise.all(batches.map(batch => semaphore.acquire(() => processBatch(batch)))); return allResults; + } + private static chunk(array: T[], size: number): T[][] { const chunks: T[][] = []; for (let i = 0; i < array.length; i += size) chunks.push(array.slice(i, i + size)); return chunks; } +} + +class Semaphore { + private permits: number; private waiting: Array<() => void> = []; + constructor(permits: number) { this.permits = permits; } + async acquire(task: () => Promise): Promise { + return new Promise((resolve, reject) => { + const tryAcquire = () => { + if (this.permits > 0) { this.permits--; task().then(resolve).catch(reject).finally(() => { this.permits++; if (this.waiting.length > 0) this.waiting.shift()!(); }); } + else this.waiting.push(tryAcquire); + }; + tryAcquire(); + }); + } +} +``` + +## File System & Validation Utilities + +```typescript +export class FileUtil { + static async writeJson(filePath: string, data: any): Promise { + try { const dir = path.dirname(filePath); await fs.mkdir(dir, { recursive: true }); await fs.writeFile(filePath, JSON.stringify(data, null, 2)); } + catch (error) { throw new Error(`Failed to write file ${filePath}: ${error.message}`); } + } + static async readJson(filePath: string): Promise { + try { const content = await fs.readFile(filePath, 'utf-8'); return JSON.parse(content); } + catch (error) { if (error.code === 'ENOENT') throw new Error(`File not found: ${filePath}`); throw new Error(`Failed to read file ${filePath}: ${error.message}`); } + } + static async exists(filePath: string): Promise { try { await fs.access(filePath); return true; } catch { return false; } } +} + +export class Validator { + static required(value: any, fieldName: string): void { if (value === null || value === undefined || value === '') throw new ValidationError(`${fieldName} is required`); } + static isArray(value: any, fieldName: string): void { if (!Array.isArray(value)) throw new ValidationError(`${fieldName} must be an array`); } + static isString(value: any, fieldName: string): void { if (typeof value !== 'string') throw new ValidationError(`${fieldName} must be a string`); } + static validateEnvironment(env: string): void { this.required(env, 'environment'); this.isString(env, 'environment'); } + static validateBatchSize(size: number): void { this.required(size, 'batchSize'); if (size < 1 || size > 100) throw new ValidationError('batchSize must be between 1 and 100'); } +} +``` + +## Dependency Injection + +```typescript +export class Container { + private services = new Map(); private factories = new Map any>(); + register(name: string, factory: () => T): void { this.factories.set(name, factory); } + get(name: string): T { + if (this.services.has(name)) return this.services.get(name); const factory = this.factories.get(name); if (!factory) throw new Error(`Service not registered: ${name}`); + const instance = factory(); this.services.set(name, instance); return instance; + } + static setup(): Container { + const container = new Container(); container.register('config', () => ConfigBuilder.build()); container.register('logger', () => new ConsoleLogger()); + container.register('rateLimiter', () => new RateLimiter(3, 100)); container.register('retryStrategy', () => RetryStrategy.forContentstack()); return container; + } +} +``` \ No newline at end of file diff --git a/packages/contentstack-query-export/skills/testing/SKILL.md b/packages/contentstack-query-export/skills/testing/SKILL.md new file mode 100644 index 000000000..35b660a82 --- /dev/null +++ b/packages/contentstack-query-export/skills/testing/SKILL.md @@ -0,0 +1,289 @@ +--- +name: testing +description: Mocha/Chai/Sinon testing and TDD for @contentstack/cli-cm-export-query. Use when writing or debugging tests in test/unit/ or adjusting coverage. +--- + +# Testing Patterns + +Testing best practices and TDD workflow for **`@contentstack/cli-cm-export-query`**. + +**RED β†’ GREEN β†’ REFACTOR** for behavior changes; pure refactors / docs-only may skip new tests when behavior is unchanged. + +## Test Structure Standards + +### Basic Test Template +```typescript +describe('[ComponentName]', () => { + beforeEach(() => { + // Setup mocks and test data + sinon.stub(ExternalService.prototype, 'method').resolves(mockData); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('should [expected behavior] when [condition]', () => { + // Arrange + const input = { /* test data */ }; + + // Act + const result = component.method(input); + + // Assert + expect(result).to.equal(expectedOutput); + }); +}); +``` + +### Command testing example +```typescript +describe('ExportQueryCommand', () => { + beforeEach(() => { + sinon.stub(ContentstackClient.prototype, 'stack').returns(mockStack); + }); + + it('should run export when query and auth are valid', async () => { + // Stub parse, setupQueryExportConfig, QueryExporter.prototype.execute, etc. + }); +}); +``` + +## Key Testing Rules + +### Coverage +- **~80%** (lines, branches, functions) is **aspirational**, not a hard gate +- Test both success and failure paths +- Include edge cases and error scenarios + +### Mocking Standards +- **Use sinon** for API responses and external dependencies +- **Never make real API calls** in tests +- **Mock at module boundaries** (SDK, `fsUtil`), not irrelevant internals +- Restore mocks in `afterEach()` to prevent test pollution + +### Test Patterns +- Use descriptive test names: "should [behavior] when [condition]" +- Keep test setup minimal and focused +- Prefer synchronous patterns when possible +- Group related tests in `describe` blocks + +## Common Mock Patterns + +### API Mocking +```typescript +// Mock Contentstack API +sinon.stub(ContentstackClient.prototype, 'fetch').resolves(mockData); + +// Mock with specific responses +sinon.stub(client, 'getEntry') + .withArgs('entry1').resolves(mockEntry1) + .withArgs('entry2').resolves(mockEntry2); +``` + +### Service Mocking +```typescript +// Mock rate limiter +sinon.stub(RateLimiter.prototype, 'wait').resolves(); + +// Mock file operations +sinon.stub(fsUtil, 'writeFile').returns(true); +sinon.stub(fsUtil, 'readFile').resolves(JSON.stringify(mockData)); +``` + +### Error Simulation +```typescript +// Mock API errors +const apiError = new Error('API Error'); +apiError.status = 500; +sinon.stub(client, 'fetch').rejects(apiError); + +// Mock rate limiting +const rateLimitError = new Error('Rate limited'); +rateLimitError.status = 429; +sinon.stub(client, 'fetch').rejects(rateLimitError); +``` + +## Error Testing Patterns + +### Rate Limit Handling +```typescript +it('should handle rate limit errors', () => { + const error = new Error('Rate limited'); + error.status = 429; + + sinon.stub(client, 'fetch').rejects(error); + + expect(service.performOperation()).to.eventually.be.fulfilled; +}); +``` + +### Validation Error Testing +```typescript +it('should throw validation error for invalid input', () => { + const invalidInput = { /* invalid data */ }; + + expect(() => service.validate(invalidInput)) + .to.throw('Validation failed'); +}); +``` + +### Async Error Handling +```typescript +it('should handle async operation failures', async () => { + sinon.stub(service, 'performAsync').rejects(new Error('Operation failed')); + + try { + await service.execute(); + expect.fail('Should have thrown error'); + } catch (error) { + expect(error.message).to.include('Operation failed'); + } +}); +``` + +## Test Organization + +### File Structure +- Mirror modules under `test/unit/`: e.g. `test/unit/query-executor.test.ts`, `test/unit/query-parser-simple.test.ts` +- Use consistent naming: `[module-name].test.ts` +- Group integration tests: `test/integration/` + +### Test Data Management +- Create mock data factories: `test/fixtures/mock-factory.ts` +- Use realistic test data that matches API responses +- Share common mocks across test files + +### Test Configuration +```javascript +// .mocharc.json +{ + "require": ["ts-node/register"], + "extensions": ["ts"], + "spec": "test/**/*.test.ts", + "timeout": 5000, + "forbid-only": true +} +``` + +## Coverage and Quality + +### Coverage Enforcement +```json +// package.json nyc configuration +"nyc": { + "check-coverage": true, + "lines": 80, + "functions": 80, + "branches": 80, + "statements": 80 +} +``` + +### Quality Checklist +- [ ] All public methods tested +- [ ] Error paths covered +- [ ] Edge cases included +- [ ] Mocks properly restored +- [ ] No real API calls +- [ ] Descriptive test names +- [ ] Minimal test setup +- [ ] Fast execution (< 5s per test) + +## Development workflow + +### TDD workflow (recommended) + +For **new behavior or bug fixes**, prefer: + +1. **RED** β†’ Failing test (or extended test) +2. **GREEN** β†’ Minimal code to pass +3. **REFACTOR** β†’ Improve while tests stay green + +**Exceptions:** pure refactors, documentation-only edits, and trivial non-behavior changes may skip new tests. + +## Guidelines + +- Prefer **clear tests** over async-heavy setup when you can +- **NO test.skip or .only** in commits +- **~80% coverage** (lines, branches, functions) is **aspirational**, not a CI gate +- **TypeScript** β€” explicit return types where practical; avoid `any` + +## File structure (this repo) + +- **Commands**: `src/commands/cm/stacks/` +- **Core**: `src/core/` (`QueryExporter`, `ModuleExporter`, …) +- **Utils**: `src/utils/` +- **Tests**: `test/unit/` β€” `*.test.ts` per module (e.g. `query-executor.test.ts`) + +## Naming conventions + +- **Files**: `kebab-case.ts` / `kebab-case.test.ts` +- **Classes**: `PascalCase` +- **Functions/Variables**: `camelCase` +- **Constants**: `SCREAMING_SNAKE_CASE` +- **Test descriptions**: "should [behavior] when [condition]" + +## Code quality standards + +### TypeScript +- Explicit return types for all functions +- No `any` type usage +- Strict null checks enabled +- No unused variables or imports + +### Error handling +- Use custom error classes where the codebase already does +- Include error context and cause +- Never swallow errors silently + +### Import organization +1. Node.js built-ins +2. External libraries +3. Internal modules (relative imports last) + +## Testing + +### Coverage +- Aim high; **~80%** is a guideline +- Test success and failure paths for behavior you touch +- Mock external dependencies (SDK, `fsUtil`, etc.) + +### Test structure +```typescript +describe('[ComponentName]', () => { + beforeEach(() => { + sinon.stub(ExternalService.prototype, 'method').resolves(mockData); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('should [expected behavior] when [condition]', () => { + const input = { /* test data */ }; + const result = component.method(input); + expect(result).to.equal(expectedOutput); + }); +}); +``` + +### Mocking standards +- Use sinon for API response mocking +- Never make real API calls in tests +- Mock at module boundaries (SDK, `fsUtil`, etc.), not irrelevant internals + +## Commit suggestions + +- Conventional commits are optional: `feat(scope): description` +- Include tests when you change behavior +- Run lint and tests before pushing +- No debugging code (`console.log`, `debugger`) left in + +## Development process + +1. **Understand** β†’ Read relevant patterns before coding +2. **Plan** β†’ Break down into testable units +3. **Test first** β†’ When adding behavior, prefer failing test then implementation +4. **Validate** β†’ `npm run lint`, `npm run test`, `npm run test:report` if you need LCOV +5. **Review** β†’ Self-review against the code review checklist diff --git a/packages/contentstack-query-export/src/commands/cm/stacks/export-query.ts b/packages/contentstack-query-export/src/commands/cm/stacks/export-query.ts new file mode 100644 index 000000000..eccc38bca --- /dev/null +++ b/packages/contentstack-query-export/src/commands/cm/stacks/export-query.ts @@ -0,0 +1,121 @@ +import { Command } from '@contentstack/cli-command'; +import { + flags, + FlagInput, + sanitizePath, + managementSDKClient, + ContentstackClient, + log, + handleAndLogError, + messageHandler, +} from '@contentstack/cli-utilities'; +import { QueryExporter } from '../../../core/query-executor'; +import { QueryExportConfig } from '../../../types'; +import { setupQueryExportConfig, setupBranches, createLogContext } from '../../../utils'; + +export default class ExportQueryCommand extends Command { + static description = 'Export content from a stack using query-based filtering'; + private exportDir: string; + + static examples = [ + 'csdx cm:stacks:export-query --query \'{"modules":{"content-types":{"title":{"$in":["Blog","Author"]}}}}\'', + 'csdx cm:stacks:export-query --query ./ct-query.json --skip-references', + 'csdx cm:stacks:export-query --alias --query \'{"modules":{"entries":{"content_type_uid":"blog"}}}\'', + 'csdx cm:stacks:export-query --query \'{"modules":{"assets":{"title":{"$regex":"image"}}}}\'', + ]; + + static usage = 'cm:stacks:export-query --query [options]'; + + static flags: FlagInput = { + config: flags.string({ + char: 'c', + description: 'Path to the configuration file', + }), + 'stack-api-key': flags.string({ + char: 'k', + description: 'Stack API key', + }), + 'data-dir': flags.string({ + char: 'd', + description: 'Path to store exported content', + }), + alias: flags.string({ + char: 'a', + description: 'Management token alias', + }), + branch: flags.string({ + description: 'Branch name to export from', + exclusive: ['branch-alias'], + }), + 'branch-alias': flags.string({ + description: 'Alias of Branch to export from', + exclusive: ['branch'], + }), + query: flags.string({ + required: true, + description: 'Query as JSON string or file path', + }), + 'skip-references': flags.boolean({ + description: 'Skip referenced content types', + }), + 'skip-dependencies': flags.boolean({ + description: 'Skip dependent modules (global-fields, extensions, taxonomies)', + }), + 'secured-assets': flags.boolean({ + description: 'Export secured assets', + }), + yes: flags.boolean({ + char: 'y', + description: 'Skip confirmation prompts', + }), + }; + + async run(): Promise { + try { + const { flags } = await this.parse(ExportQueryCommand); + this.initializeMessageHandler(); + + // Setup export configuration + const exportQueryConfig = await setupQueryExportConfig(flags); + exportQueryConfig.host = this.cmaHost; + exportQueryConfig.region = this.region; + + if (this.developerHubUrl) { + exportQueryConfig.developerHubBaseUrl = this.developerHubUrl; + } + + this.exportDir = sanitizePath(exportQueryConfig.exportDir); + // Create base context without module name - module field is set dynamically during each module export + exportQueryConfig.context = createLogContext(exportQueryConfig); + log.debug('Export configuration setup completed', exportQueryConfig.context); + + // Initialize management API client + const managementAPIClient: ContentstackClient = await managementSDKClient(exportQueryConfig); + + // Setup and validate branch configuration + const stackAPIClient = managementAPIClient.stack({ + api_key: exportQueryConfig.stackApiKey, + management_token: exportQueryConfig.managementToken, + }); + + // Setup branches (validate branch or set default to 'main') + await setupBranches(exportQueryConfig, stackAPIClient); + log.debug('Branch configuration setup completed', exportQueryConfig.context); + + // Initialize and run query export + log.debug('Starting query exporter', exportQueryConfig.context); + const queryExporter = new QueryExporter(managementAPIClient, exportQueryConfig); + await queryExporter.execute(); + log.debug('Query exporter completed successfully', exportQueryConfig.context); + + log.success('Query-based export completed successfully!', exportQueryConfig.context); + log.info(`Export files saved to: ${this.exportDir}`, exportQueryConfig.context); + } catch (error) { + handleAndLogError(error); + } + } + + initializeMessageHandler(): void { + messageHandler.init(this.context); + } +} diff --git a/packages/contentstack-query-export/src/config/export-config.json b/packages/contentstack-query-export/src/config/export-config.json new file mode 100644 index 000000000..e8f2b1d2d --- /dev/null +++ b/packages/contentstack-query-export/src/config/export-config.json @@ -0,0 +1 @@ +{ "skipDependencies": true, "skipStackSettings": false, "personalizationEnabled": true } diff --git a/packages/contentstack-query-export/src/config/index.ts b/packages/contentstack-query-export/src/config/index.ts new file mode 100644 index 000000000..58185f1bd --- /dev/null +++ b/packages/contentstack-query-export/src/config/index.ts @@ -0,0 +1,59 @@ +import { DefaultConfig } from '../types'; + +const config: DefaultConfig = { + contentVersion: 1, + host: 'https://api.contentstack.io/v3', + + // Query-based export module configuration + modules: { + // Always export - general modules + general: ['stack', 'locales', 'environments'], + // Query target modules + queryable: ['content-types'], + dependent: ['global-fields', 'extensions', 'marketplace-apps', 'taxonomies', 'personalize'], + // Content modules + content: ['entries', 'assets'], + // Export order based on dependencies + exportOrder: [ + 'stack', + 'locales', + 'environments', + 'content-types', + 'global-fields', + 'extensions', + 'taxonomies', + 'entries', + 'assets', + ], + }, + // Query-specific settings + queryConfig: { + maxRecursionDepth: 10, + batchSize: 100, + metadataFileName: '_query-meta.json', + validation: { + maxQueryDepth: 5, + maxArraySize: 1000, + allowedDateFormats: ['ISO8601', 'YYYY-MM-DD', 'MM/DD/YYYY'], + }, + }, + // API endpoints + apis: { + stacks: '/stacks/', + locales: '/locales/', + environments: '/environments/', + content_types: '/content_types/', + global_fields: '/global_fields/', + extensions: '/extensions/', + taxonomies: '/taxonomies/', + entries: '/entries/', + assets: '/assets/', + }, + // Performance settings + fetchConcurrency: 5, + writeConcurrency: 5, + maxCTReferenceDepth: 20, + // Optional settings +}; + +export default config; diff --git a/packages/contentstack-query-export/src/core/module-exporter.ts b/packages/contentstack-query-export/src/core/module-exporter.ts new file mode 100644 index 000000000..754c43063 --- /dev/null +++ b/packages/contentstack-query-export/src/core/module-exporter.ts @@ -0,0 +1,105 @@ +import * as path from 'path'; +import { log, handleAndLogError, sanitizePath } from '@contentstack/cli-utilities'; +import ExportCommand from '@contentstack/cli-cm-export'; +import { QueryExportConfig, Modules, ExportOptions } from '../types'; +import { rebuildContentTypesSchemaJson } from '../utils/read-content-type-schemas'; + +export class ModuleExporter { + private exportQueryConfig: QueryExportConfig; + private exportedModules: string[] = []; + + constructor(exportQueryConfig: QueryExportConfig) { + this.exportQueryConfig = exportQueryConfig; + } + + async exportModule(moduleName: Modules, options: ExportOptions = {}): Promise { + try { + const moduleLogContext = { ...this.exportQueryConfig.context, module: moduleName }; + log.info(`Exporting module: ${moduleName}`, moduleLogContext); + log.debug(`Building export command for module: ${moduleName}`, moduleLogContext); + + // Build command arguments + const cmd = this.buildExportCommand(moduleName, options); + + // Configurable delay + const delay = this.exportQueryConfig.exportDelayMs || 2000; + await new Promise((resolve) => setTimeout(resolve, delay)); + + // Create export command instance + await ExportCommand.run(cmd); + log.debug(`Export command completed for module: ${moduleName}`, moduleLogContext); + + if (moduleName === 'content-types') { + const baseDir = options.directory || this.exportQueryConfig.exportDir; + const branch = options.branch || this.exportQueryConfig.branchName || ''; + const ctDir = path.join(sanitizePath(baseDir), sanitizePath(branch), 'content_types'); + rebuildContentTypesSchemaJson(ctDir); + log.debug('Rebuilt content_types/schema.json from all per-UID JSON files', moduleLogContext); + } + + // Read the exported data + // const data = await this.readExportedData(moduleName, options); + + if (!this.exportedModules.includes(moduleName)) { + this.exportedModules.push(moduleName); + } + + // success message + log.success(`Successfully exported ${moduleName}`, moduleLogContext); + } catch (error) { + const moduleLogContext = { ...this.exportQueryConfig.context, module: moduleName }; + handleAndLogError(error, moduleLogContext, `Failed to export ${moduleName}`); + throw error; + } + } + + /** + * Build export command arguments based on module and options + */ + private buildExportCommand(moduleName: Modules, options: ExportOptions): string[] { + const cmd: string[] = []; + + // Stack API key (required) + cmd.push('-k', this.exportQueryConfig.stackApiKey); + + // Directory + const directory = options.directory || this.exportQueryConfig.exportDir; + cmd.push('-d', directory); + + // Module + cmd.push('--module', moduleName); + + // Alias or management token (mutually exclusive for the export CLI) + if (options.alias) { + cmd.push('-a', options.alias); + } else if (this.exportQueryConfig.managementToken) { + cmd.push('-a', this.exportQueryConfig.managementToken); + } + + // Branch + if (options.branch || this.exportQueryConfig.branchName) { + cmd.push('--branch', options.branch || this.exportQueryConfig.branchName); + } + + // Query (if provided) + if (options.query) { + cmd.push('--query', JSON.stringify(options.query)); + } + + // Secured assets + if (options.securedAssets || this.exportQueryConfig.securedAssets) { + cmd.push('--secured-assets'); + } + + // External config file + const externalConfigPath = options.configPath || this.exportQueryConfig.externalConfigPath; + if (externalConfigPath) { + cmd.push('--config', externalConfigPath); + } + + // Auto confirm + cmd.push('-y'); + + return cmd; + } +} diff --git a/packages/contentstack-query-export/src/core/query-executor.ts b/packages/contentstack-query-export/src/core/query-executor.ts new file mode 100644 index 000000000..5bf690479 --- /dev/null +++ b/packages/contentstack-query-export/src/core/query-executor.ts @@ -0,0 +1,389 @@ +import { ContentstackClient, sanitizePath, log, handleAndLogError } from '@contentstack/cli-utilities'; +import * as fs from 'fs'; +import * as path from 'path'; +import { QueryExportConfig, Modules } from '../types'; +import { QueryParser } from '../utils/query-parser'; +import { ModuleExporter } from './module-exporter'; +import { ReferencedContentTypesHandler } from '../utils'; +import { fsUtil } from '../utils'; +import { ContentTypeDependenciesHandler } from '../utils'; +import { AssetReferenceHandler } from '../utils'; +import { readContentTypesFromExportDir, readGlobalFieldSchemasFromDir } from '../utils/read-content-type-schemas'; + +export class QueryExporter { + private stackAPIClient: ReturnType; + private exportQueryConfig: QueryExportConfig; + private queryParser: QueryParser; + private moduleExporter: ModuleExporter; + + constructor(managementAPIClient: ContentstackClient, exportQueryConfig: QueryExportConfig) { + this.exportQueryConfig = exportQueryConfig; + + this.stackAPIClient = managementAPIClient.stack({ + api_key: exportQueryConfig.stackApiKey, + management_token: exportQueryConfig.managementToken, + }); + // Initialize components + this.queryParser = new QueryParser(this.exportQueryConfig); + this.moduleExporter = new ModuleExporter(exportQueryConfig); + } + + async execute(): Promise { + log.info('Starting query-based export...', this.exportQueryConfig.context); + + // Step 1: Parse and validate query + log.debug('Parsing and validating query', this.exportQueryConfig.context); + const parsedQuery = await this.queryParser.parse(this.exportQueryConfig.query); + log.success('Query parsed and validated successfully', this.exportQueryConfig.context); + + // Step 2: Always export general modules + await this.exportGeneralModules(); + + // Step 4: Export queried modules + await this.exportQueriedModule(parsedQuery); + + // Step 5+6: resolve the full transitive closure of referenced content types, + // global fields, extensions, taxonomies, and marketplace apps. + log.debug('Starting schema closure expansion', this.exportQueryConfig.context); + await this.expandSchemaClosure(); + // Step 7: export content modules entries, assets + log.debug('Starting content modules export', this.exportQueryConfig.context); + await this.exportContentModules(); + // Step 9: export all other modules + + log.success('Query-based export completed successfully!', this.exportQueryConfig.context); + } + + // export general modules + private async exportGeneralModules(): Promise { + log.info('Exporting general modules...', this.exportQueryConfig.context); + + for (const module of this.exportQueryConfig.modules.general) { + await this.moduleExporter.exportModule(module); + } + } + + private async exportQueriedModule(parsedQuery: any): Promise { + log.debug('Starting queried module export', this.exportQueryConfig.context); + for (const [moduleName] of Object.entries(parsedQuery.modules)) { + const module = moduleName as Modules; + + if (!this.exportQueryConfig.modules.queryable.includes(module)) { + log.error(`Module "${module}" is not queryable`, this.exportQueryConfig.context); + continue; + } + + log.info(`Exporting ${moduleName} with query...`, this.exportQueryConfig.context); + // Export the queried module + await this.moduleExporter.exportModule(module, { query: parsedQuery }); + } + log.debug('Queried module export completed', this.exportQueryConfig.context); + } + + /** + * Iteratively expand the set of exported content types, global fields, extensions, + * taxonomies, and marketplace apps until no new items are discovered (fixpoint). + * + * Each iteration scans the combined set of CT and GF documents that currently exist on + * disk. Any newly discovered referenced content types or global fields are exported and + * the loop restarts so that their schemas can be scanned in turn. Leaf dependencies + * (extensions, taxonomies, marketplace apps) are collected and exported in the same pass + * without triggering an extra iteration, since they do not themselves produce new schemas. + * + * Personalize is exported exactly once, after the closure stabilises. + */ + private async expandSchemaClosure(): Promise { + log.info('Starting export of referenced content types and dependent modules...', this.exportQueryConfig.context); + + try { + const ctPath = path.join( + sanitizePath(this.exportQueryConfig.exportDir), + sanitizePath(this.exportQueryConfig.branchName || ''), + 'content_types', + ); + const gfPath = path.join( + sanitizePath(this.exportQueryConfig.exportDir), + sanitizePath(this.exportQueryConfig.branchName || ''), + 'global_fields', + ); + + const referencedHandler = new ReferencedContentTypesHandler(this.exportQueryConfig); + const dependenciesHandler = new ContentTypeDependenciesHandler(this.stackAPIClient, this.exportQueryConfig); + + const exportedCTUIDs = new Set(); + const exportedGFUIDs = new Set(); + const exportedExtUIDs = new Set(); + const exportedTaxUIDs = new Set(); + const exportedMarketplaceUIDs = new Set(); + + let iterationCount = 0; + + while (iterationCount < this.exportQueryConfig.maxCTReferenceDepth) { + iterationCount++; + log.debug(`Schema closure iteration ${iterationCount}`, this.exportQueryConfig.context); + + const allCTs = readContentTypesFromExportDir(ctPath); + const allGFs = readGlobalFieldSchemasFromDir(gfPath); + + // Record everything currently on disk so we never re-export it. + allCTs.forEach((ct: any) => exportedCTUIDs.add(ct.uid)); + allGFs.forEach((gf: any) => exportedGFUIDs.add(gf.uid)); + + const allSchemas = [...allCTs, ...allGFs]; + + if (allSchemas.length === 0) { + log.info('No schemas found on disk, stopping closure', this.exportQueryConfig.context); + break; + } + + let foundNewCTs = false; + let foundNewGFs = false; + + // Step A: find and export referenced content types from the combined schema set. + if (!this.exportQueryConfig.skipReferences) { + const referencedUIDs = await referencedHandler.extractReferencedContentTypes(allSchemas); + const newCTUIDs = referencedUIDs.filter((uid: string) => !exportedCTUIDs.has(uid)); + + if (newCTUIDs.length > 0) { + log.info( + `Found ${newCTUIDs.length} new referenced content type(s) to fetch`, + this.exportQueryConfig.context, + ); + await this.moduleExporter.exportModule('content-types', { + query: { modules: { 'content-types': { uid: { $in: newCTUIDs } } } }, + }); + // Track immediately so the dedup filter works even if the disk reader + // hasn't picked up the newly written files yet. + newCTUIDs.forEach((uid: string) => exportedCTUIDs.add(uid)); + foundNewCTs = true; + } + } + + // Step B: find and export dependent modules from the combined schema set. + if (!this.exportQueryConfig.skipDependencies) { + const deps = await dependenciesHandler.extractDependencies(allSchemas); + + const newGFUIDs = [...deps.globalFields].filter((uid: string) => !exportedGFUIDs.has(uid)); + if (newGFUIDs.length > 0) { + log.info(`Found ${newGFUIDs.length} new global field(s)`, this.exportQueryConfig.context); + await this.moduleExporter.exportModule('global-fields', { + query: { modules: { 'global-fields': { uid: { $in: newGFUIDs } } } }, + }); + // Track immediately for the same reason as CTs above. + newGFUIDs.forEach((uid: string) => exportedGFUIDs.add(uid)); + foundNewGFs = true; + } + + // Extensions, taxonomies, and marketplace apps are leaf nodes: they do not + // produce new schemas, so exporting them never requires an extra iteration. + const newExtUIDs = [...deps.extensions].filter((uid: string) => !exportedExtUIDs.has(uid)); + if (newExtUIDs.length > 0) { + log.info(`Found ${newExtUIDs.length} new extension(s)`, this.exportQueryConfig.context); + await this.moduleExporter.exportModule('extensions', { + query: { modules: { extensions: { uid: { $in: newExtUIDs } } } }, + }); + newExtUIDs.forEach((uid: string) => exportedExtUIDs.add(uid)); + } + + const newMarketplaceUIDs = [...deps.marketplaceApps].filter( + (uid: string) => !exportedMarketplaceUIDs.has(uid), + ); + if (newMarketplaceUIDs.length > 0) { + log.info(`Found ${newMarketplaceUIDs.length} new marketplace app(s)`, this.exportQueryConfig.context); + await this.moduleExporter.exportModule('marketplace-apps', { + query: { modules: { 'marketplace-apps': { installation_uid: { $in: newMarketplaceUIDs } } } }, + }); + newMarketplaceUIDs.forEach((uid: string) => exportedMarketplaceUIDs.add(uid)); + } + + const newTaxUIDs = [...deps.taxonomies].filter((uid: string) => !exportedTaxUIDs.has(uid)); + if (newTaxUIDs.length > 0) { + log.info(`Found ${newTaxUIDs.length} new taxonom(ies)`, this.exportQueryConfig.context); + await this.moduleExporter.exportModule('taxonomies', { + query: { modules: { taxonomies: { uid: { $in: newTaxUIDs } } } }, + }); + newTaxUIDs.forEach((uid: string) => exportedTaxUIDs.add(uid)); + } + } + + if (!foundNewCTs && !foundNewGFs) { + log.info('Schema closure complete, no new content types or global fields found', this.exportQueryConfig.context); + break; + } + } + + // Personalize is a single global module exported once after the closure stabilises. + await this.moduleExporter.exportModule('personalize'); + + log.success('Referenced content types and dependent modules exported successfully', this.exportQueryConfig.context); + } catch (error) { + handleAndLogError(error, this.exportQueryConfig.context, 'Error during schema closure expansion'); + throw error; + } + } + + private async exportContentModules(): Promise { + log.info('Starting export of content modules...', this.exportQueryConfig.context); + + try { + // Step 1: Export entries for all exported content types + await this.exportEntries(); + + // Step 2: Export referenced assets from entries + // add a delay of 5 seconds + const delay = (this.exportQueryConfig as any).exportDelayMs || 5000; + await new Promise((resolve) => setTimeout(resolve, delay)); + await this.exportReferencedAssets(); + + log.success('Content modules export completed successfully', this.exportQueryConfig.context); + } catch (error) { + handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting content modules'); + throw error; + } + } + + private async exportEntries(): Promise { + log.info('Exporting entries...', this.exportQueryConfig.context); + + try { + // Export entries - module exporter will automatically read exported content types + // and export entries for all of them + await this.moduleExporter.exportModule('entries'); + + log.success('Entries export completed successfully', this.exportQueryConfig.context); + } catch (error) { + handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting entries'); + throw error; + } + } + + private async exportReferencedAssets(): Promise { + log.info('Starting export of referenced assets...', this.exportQueryConfig.context); + + try { + const assetsDir = path.join( + sanitizePath(this.exportQueryConfig.exportDir), + sanitizePath(this.exportQueryConfig.branchName || ''), + 'assets', + ); + + const metadataFilePath = path.join(assetsDir, 'metadata.json'); + const assetFilePath = path.join(assetsDir, 'assets.json'); + + // Define temp file paths + const tempMetadataFilePath = path.join(assetsDir, 'metadata_temp.json'); + const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json'); + + const assetHandler = new AssetReferenceHandler(this.exportQueryConfig); + + // Extract referenced asset UIDs from all entries + log.debug('Extracting referenced assets from entries', this.exportQueryConfig.context); + const assetUIDs = assetHandler.extractReferencedAssets(); + + if (assetUIDs.length > 0) { + log.info(`Found ${assetUIDs.length} referenced assets to export`, this.exportQueryConfig.context); + + fs.mkdirSync(assetsDir, { recursive: true }); + + // Define batch size - can be configurable through exportQueryConfig + const batchSize = this.exportQueryConfig.assetBatchSize || 100; + + // if asset size is bigger than batch size, then we need to export in batches + // Calculate number of batches + const totalBatches = Math.ceil(assetUIDs.length / batchSize); + log.info(`Processing assets in ${totalBatches} batches of ${batchSize}`, this.exportQueryConfig.context); + + // Process assets in batches + for (let i = 0; i < totalBatches; i++) { + const start = i * batchSize; + const end = Math.min(start + batchSize, assetUIDs.length); + const batchAssetUIDs = assetUIDs.slice(start, end); + + log.info( + `Exporting batch ${i + 1}/${totalBatches} (${batchAssetUIDs.length} assets)...`, + this.exportQueryConfig.context, + ); + + const query = { + modules: { + assets: { + uid: { $in: batchAssetUIDs }, + }, + }, + }; + + await this.moduleExporter.exportModule('assets', { query }); + + // Read the current batch's metadata.json and assets.json files + const currentMetadata: any = fsUtil.readFile(sanitizePath(metadataFilePath)); + const currentAssets: any = fsUtil.readFile(sanitizePath(assetFilePath)); + + // Check if this is the first batch + if (i === 0) { + // For first batch, initialize temp files with current content + fsUtil.writeFile(sanitizePath(tempMetadataFilePath), currentMetadata); + fsUtil.writeFile(sanitizePath(tempAssetFilePath), currentAssets); + log.info(`Initialized temporary files with first batch data`, this.exportQueryConfig.context); + } else { + // For subsequent batches, append to temp files with incremented keys + + // Handle metadata (which contains arrays of asset info) + const tempMetadata: any = fsUtil.readFile(sanitizePath(tempMetadataFilePath)) || {}; + + // Merge metadata by combining arrays + if (currentMetadata) { + Object.keys(currentMetadata).forEach((key: string) => { + if (!tempMetadata[key]) { + tempMetadata[key] = currentMetadata[key]; + } + }); + } + + // Write updated metadata back to temp file + fsUtil.writeFile(sanitizePath(tempMetadataFilePath), tempMetadata); + + // Handle assets (which is an object with numeric keys) + const tempAssets: any = fsUtil.readFile(sanitizePath(tempAssetFilePath)) || {}; + let nextIndex = Object.keys(tempAssets).length + 1; + + // Add current assets with incremented keys + Object.values(currentAssets).forEach((value: any) => { + tempAssets[nextIndex.toString()] = value; + nextIndex++; + }); + + fsUtil.writeFile(sanitizePath(tempAssetFilePath), tempAssets); + + log.info(`Updated temporary files with batch ${i + 1} data`, this.exportQueryConfig.context); + } + + // Optional: Add delay between batches to avoid rate limiting + if (i < totalBatches - 1 && this.exportQueryConfig.batchDelayMs) { + await new Promise((resolve) => setTimeout(resolve, this.exportQueryConfig.batchDelayMs)); + } + } + + // After all batches are processed, copy temp files back to original files + const finalMetadata = fsUtil.readFile(sanitizePath(tempMetadataFilePath)); + const finalAssets = fsUtil.readFile(sanitizePath(tempAssetFilePath)); + + fsUtil.writeFile(sanitizePath(metadataFilePath), finalMetadata); + fsUtil.writeFile(sanitizePath(assetFilePath), finalAssets); + + log.info(`Final data written back to original files`, this.exportQueryConfig.context); + + // Clean up temp files + fsUtil.removeFile(sanitizePath(tempMetadataFilePath)); + fsUtil.removeFile(sanitizePath(tempAssetFilePath)); + + log.info(`Temporary files cleaned up`, this.exportQueryConfig.context); + log.success('Referenced assets exported successfully', this.exportQueryConfig.context); + } else { + log.info('No referenced assets found in entries', this.exportQueryConfig.context); + } + } catch (error) { + handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting referenced assets'); + throw error; + } + } +} diff --git a/packages/contentstack-query-export/src/types/index.ts b/packages/contentstack-query-export/src/types/index.ts new file mode 100644 index 000000000..1a036ea24 --- /dev/null +++ b/packages/contentstack-query-export/src/types/index.ts @@ -0,0 +1,226 @@ +import { ContentstackClient } from '@contentstack/cli-utilities'; + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export interface AuthOptions { + contentstackClient: any; +} + +export interface ContentStackManagementClient { + contentstackClient: object; +} + +export interface PrintOptions { + color?: string; +} + +export interface InquirePayload { + type: string; + name: string; + message: string; + choices?: Array; + transformer?: Function; +} + +export interface User { + email: string; + authtoken: string; +} + +export interface Region { + name: string; + cma: string; + cda: string; + uiHost: string; +} + +export type Modules = + | 'stack' + | 'locales' + | 'environments' + | 'content-types' + | 'global-fields' + | 'extensions' + | 'taxonomies' + | 'entries' + | 'assets' + | 'webhooks' + | 'workflows' + | 'custom-roles' + | 'labels' + | 'marketplace-apps' + | 'personalize'; + +export interface ModuleQueryConfig { + supportedFields: string[]; + supportedOperators: string[]; + defaultLimit: number; + includeGlobalFieldSchema?: boolean; + includePublishDetails?: boolean; + includeDimension?: boolean; +} + +export interface DependencyAnalysisConfig { + enabled: boolean; + fields?: string[]; + extractors?: string[]; +} + +export interface ModuleDefinition { + dirName: string; + fileName: string; + apiEndpoint: string; + queryable: boolean; + dependencies: Modules[]; + queryConfig?: ModuleQueryConfig; + dependencyAnalysis?: DependencyAnalysisConfig; + limit?: number; + batchLimit?: number; +} + +export interface DependencyExtractor { + fieldType: string; + extract: (data: any) => string[]; + targetModule: Modules; +} + +export interface ExportOptions { + query?: any; + alias?: string; + directory?: string; + branch?: string; + skipReferences?: boolean; + skipDependencies?: boolean; + securedAssets?: boolean; + includeGlobalFieldSchema?: boolean; + includePublishDetails?: boolean; + includeDimension?: boolean; + contentTypes?: string[]; + uids?: string[]; + configPath?: string; + fetchConcurrency?: number; + writeConcurrency?: number; + batchSize?: number; + [key: string]: any; +} + +export interface DefaultConfig { + // Basic settings + contentVersion: number; + host: string; + + // Export settings + exportDir?: string; + stackApiKey?: string; + managementToken?: string; + region?: Region; + branchName?: string; + securedAssets?: boolean; + + // Query settings + query?: string; + queryInput?: string; + skipReferences?: boolean; + skipDependencies?: boolean; + isQueryBasedExport?: boolean; + + // Module configuration + modules: { + general: Modules[]; + queryable: Modules[]; + dependent: Modules[]; + content: Modules[]; + // Export order + exportOrder: Modules[]; + // Module definitions + definitions?: Record; + }; + + // Query-specific settings + queryConfig: { + maxRecursionDepth?: number; + batchSize?: number; + metadataFileName?: string; + validation: { + maxQueryDepth?: number; + maxArraySize?: number; + allowedDateFormats?: string[]; + }; + }; + + // Dependency extraction rules + dependencyExtractors?: Record; + + // Performance settings + fetchConcurrency: number; + writeConcurrency: number; + + // Optional settings + developerHubBaseUrl?: string; + branches?: Array<{ uid: string; source: string }>; + branchEnabled?: boolean; + branchDir?: string; + apis: { + stacks: string; + locales: string; + environments: string; + content_types: string; + global_fields: string; + extensions: string; + taxonomies: string; + entries: string; + assets: string; + }; + externalConfigPath?: string; + maxCTReferenceDepth: number; +} + +/** + * Log context interface for centralized logging + */ +export interface LogContext { + command: string; + module: string; + email: string; + sessionId: string; + apiKey: string; + orgId: string; + authenticationMethod: string; + [key: string]: unknown; +} + +export interface QueryExportConfig extends DefaultConfig { + query: string; + skipReferences: boolean; + skipDependencies: boolean; + stackApiKey: string; + managementToken?: string; + branchName: string; + branchAlias?: string; + securedAssets: boolean; + logsPath: string; + dataPath: string; + exportDelayMs?: number; + batchDelayMs?: number; + assetBatchSize?: number; + assetBatchDelayMs?: number; + context?: LogContext; // Log context for centralized logging +} + +export interface QueryMetadata { + query: any; + flags: { + skipReferences: boolean; + skipDependencies: boolean; + }; + timestamp: string; + cliVersion: string; + exportedModules: string[]; + contentTypes: Array<{ + uid: string; + title: string; + }>; + summary: { + totalContentTypes: number; + totalModules: number; + }; +} diff --git a/packages/contentstack-query-export/src/utils/branch-helper.ts b/packages/contentstack-query-export/src/utils/branch-helper.ts new file mode 100644 index 000000000..43b007840 --- /dev/null +++ b/packages/contentstack-query-export/src/utils/branch-helper.ts @@ -0,0 +1,70 @@ + +import { getBranchFromAlias, log, handleAndLogError } from '@contentstack/cli-utilities'; +import { QueryExportConfig } from '../types'; + +/** + * Validates and sets up branch configuration for the stack + * + * @param config The export configuration + * @param stackAPIClient The stack API client + * @returns Promise that resolves when branch setup is complete + */ +export const setupBranches = async (config: QueryExportConfig, stackAPIClient: any): Promise => { + if (typeof config !== 'object') { + throw new Error('The branch configuration is invalid.'); + } + + const context = config.context; + + try { + if (config.branchAlias) { + config.branchName = await getBranchFromAlias(stackAPIClient, config.branchAlias); + return; + } + if (config.branchName) { + // Check if the specified branch exists + log.info(`Validating branch: ${config.branchName}`, context); + + const result = await stackAPIClient + .branch(config.branchName) + .fetch() + .catch((err: unknown): any => { + handleAndLogError(err, context, 'Error fetching branch'); + return null; + }); + + if (result && typeof result === 'object') { + log.success(`Branch '${config.branchName}' found`, context); + } else { + throw new Error(`No branch found named ${config.branchName}.`); + } + } else { + // If no branch name provided, check if the stack has branches + log.info('No branch specified, checking if stack has branches', context); + + const result = await stackAPIClient + .branch() + .query() + .find() + .catch((): any => { + log.info('Stack does not have branches', context); + return null; + }); + + if (result && result.items && Array.isArray(result.items) && result.items.length > 0) { + // Set default branch to 'main' if it exists + config.branchName = 'main'; + } else { + // Stack doesn't have branches + log.info('Stack does not have branches', context); + return; + } + } + config.branchEnabled = true; + } catch (error) { + handleAndLogError(error, context, 'Error setting up branches'); + throw error; + } +}; + +export default setupBranches; diff --git a/packages/contentstack-query-export/src/utils/common-helper.ts b/packages/contentstack-query-export/src/utils/common-helper.ts new file mode 100644 index 000000000..724e2f5b1 --- /dev/null +++ b/packages/contentstack-query-export/src/utils/common-helper.ts @@ -0,0 +1,9 @@ +import { cliux } from '@contentstack/cli-utilities'; + +export const askAPIKey = async (): Promise => { + return await cliux.inquire({ + type: 'input', + message: 'Enter the stack api key', + name: 'apiKey', + }); +}; diff --git a/packages/contentstack-query-export/src/utils/config-handler.ts b/packages/contentstack-query-export/src/utils/config-handler.ts new file mode 100644 index 000000000..94eccfc65 --- /dev/null +++ b/packages/contentstack-query-export/src/utils/config-handler.ts @@ -0,0 +1,56 @@ +import * as path from 'path'; +import { QueryExportConfig } from '../types'; +import { sanitizePath, pathValidator, configHandler, isAuthenticated } from '@contentstack/cli-utilities'; +import config from '../config'; +import { askAPIKey } from './common-helper'; + +export async function setupQueryExportConfig(flags: any): Promise { + const exportDir = sanitizePath(flags['data-dir'] || pathValidator('export')); + + const exportQueryConfig: QueryExportConfig = { + ...config, + exportDir, + stackApiKey: flags['stack-api-key'] || '', + managementToken: flags.alias ? configHandler.get(`tokens.${flags.alias}`)?.token : undefined, + query: flags.query, + skipReferences: flags['skip-references'] || false, + skipDependencies: flags['skip-dependencies'] || false, + branchName: flags.branch, + securedAssets: flags['secured-assets'] || false, + isQueryBasedExport: true, + logsPath: exportDir, + dataPath: exportDir, + // Todo: accept the path of the config file from the user + externalConfigPath: path.join(__dirname, '../config/export-config.json'), + }; + + if (flags['branch-alias']) { + exportQueryConfig.branchAlias = flags['branch-alias']; + } + // override the external config path if the user provides a config file + if (flags.config) { + exportQueryConfig.externalConfigPath = sanitizePath(flags['config']); + } + + // Handle authentication + if (flags.alias) { + const { token, apiKey } = configHandler.get(`tokens.${flags.alias}`) || {}; + exportQueryConfig.managementToken = token; + exportQueryConfig.stackApiKey = apiKey; + if (!exportQueryConfig.managementToken) { + throw new Error(`No management token found for alias ${flags.alias}.`); + } + } + + if (!exportQueryConfig.managementToken) { + if (!isAuthenticated()) { + throw new Error('Log in or provide an alias for the management token.'); + } else { + exportQueryConfig.stackApiKey = flags['stack-api-key'] || (await askAPIKey()); + if (typeof exportQueryConfig.stackApiKey !== 'string') { + throw new Error('The API key is invalid.'); + } + } + } + return exportQueryConfig; +} diff --git a/packages/contentstack-query-export/src/utils/content-type-helper.ts b/packages/contentstack-query-export/src/utils/content-type-helper.ts new file mode 100644 index 000000000..376be7f54 --- /dev/null +++ b/packages/contentstack-query-export/src/utils/content-type-helper.ts @@ -0,0 +1,102 @@ +import * as path from 'path'; +import { log } from '@contentstack/cli-utilities'; +import { QueryExportConfig } from '../types'; + +export class ReferencedContentTypesHandler { + private exportQueryConfig: QueryExportConfig; + + constructor(exportQueryConfig: QueryExportConfig) { + this.exportQueryConfig = exportQueryConfig; + } + + /** + * Extract referenced content types from a batch of content types + * This method only processes the given batch, doesn't orchestrate the entire process + */ + async extractReferencedContentTypes(contentTypeBatch: any[]): Promise { + const allReferencedTypes: Set = new Set(); + + log.info(`Extracting references from ${contentTypeBatch.length} content types`, this.exportQueryConfig.context); + + for (const contentType of contentTypeBatch) { + if (contentType.schema) { + const referencedTypes = this.getReferencedContentTypes(contentType.schema); + referencedTypes.forEach((type) => allReferencedTypes.add(type)); + } + } + + const result = Array.from(allReferencedTypes); + log.info(`Found ${result.length} referenced content types`, this.exportQueryConfig.context); + return result; + } + + /** + * Filter content types to get only newly fetched ones based on UIDs + */ + filterNewlyFetchedContentTypes(allContentTypes: any[], previousUIDs: Set): any[] { + return allContentTypes.filter((ct) => !previousUIDs.has(ct.uid)); + } + + /** + * Extract referenced content types from a content type schema + * Moved from content-type-helper.ts for better encapsulation + */ + private getReferencedContentTypes(schema: any): string[] { + const referencedTypes: Set = new Set(); + + const traverseSchema = (schemaArray: any[]) => { + for (const field of schemaArray) { + if (field.data_type === 'group' || field.data_type === 'global_field') { + // Recursively traverse group and global field schemas. + // field.schema may be absent when a global_field is represented only by + // its reference_to UID (stub form in a content type's inline schema). + if (Array.isArray(field.schema) && field.schema.length > 0) { + traverseSchema(field.schema); + } + } else if (field.data_type === 'blocks') { + // Traverse each block's schema + for (const blockKey in field.blocks) { + if (field.blocks[blockKey]?.schema) { + traverseSchema(field.blocks[blockKey].schema); + } + } + } else if (field.data_type === 'reference' && field.reference_to) { + // Add reference field targets + field.reference_to.forEach((ref: string) => { + if (ref !== 'sys_assets') { + // Exclude system assets + referencedTypes.add(ref); + } + }); + } else if ( + // Handle JSON RTE with embedded entries + field.data_type === 'json' && + field.field_metadata?.rich_text_type && + field.field_metadata?.embed_entry && + field.reference_to + ) { + field.reference_to.forEach((ref: string) => { + if (ref !== 'sys_assets') { + referencedTypes.add(ref); + } + }); + } else if ( + // Handle Text RTE with embedded entries + field.data_type === 'text' && + field.field_metadata?.rich_text_type && + field.field_metadata?.embed_entry && + field.reference_to + ) { + field.reference_to.forEach((ref: string) => { + if (ref !== 'sys_assets') { + referencedTypes.add(ref); + } + }); + } + } + }; + + traverseSchema(schema); + return Array.from(referencedTypes); + } +} diff --git a/packages/contentstack-query-export/src/utils/dependency-resolver.ts b/packages/contentstack-query-export/src/utils/dependency-resolver.ts new file mode 100644 index 000000000..5e99bcec2 --- /dev/null +++ b/packages/contentstack-query-export/src/utils/dependency-resolver.ts @@ -0,0 +1,183 @@ +import * as path from 'path'; +import { QueryExportConfig } from '../types'; +import { ContentstackClient, sanitizePath, log, handleAndLogError } from '@contentstack/cli-utilities'; +import { readContentTypesFromExportDir } from './read-content-type-schemas'; + +export class ContentTypeDependenciesHandler { + private exportQueryConfig: QueryExportConfig; + private stackAPIClient: ReturnType; + + constructor(stackAPIClient: any, exportQueryConfig: QueryExportConfig) { + this.exportQueryConfig = exportQueryConfig; + this.stackAPIClient = stackAPIClient; + } + + /** + * Extract all dependencies (global fields, extensions, taxonomies, marketplace apps) from the + * provided schema documents. When `schemas` is omitted the method falls back to reading content + * type schemas from disk β€” kept for backward compatibility with callers that do not supply + * already-loaded documents. + * + * Pass the combined set of content-type AND global-field documents so that transitive + * dependencies inside global fields are discovered in the same pass. + */ + async extractDependencies(schemas?: any[]): Promise<{ + globalFields: Set; + extensions: Set; + taxonomies: Set; + marketplaceApps: Set; + }> { + let allSchemas: any[]; + + if (schemas !== undefined) { + allSchemas = schemas; + } else { + const contentTypesFilePath = path.join( + sanitizePath(this.exportQueryConfig.exportDir), + sanitizePath(this.exportQueryConfig.branchName || ''), + 'content_types', + ); + allSchemas = readContentTypesFromExportDir(contentTypesFilePath); + } + + if (allSchemas.length === 0) { + log.info('No schemas found, skipping dependency extraction', this.exportQueryConfig.context); + return { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + marketplaceApps: new Set(), + }; + } + + log.info(`Extracting dependencies from ${allSchemas.length} schema(s)`, this.exportQueryConfig.context); + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + marketplaceApps: new Set(), + }; + + for (const doc of allSchemas) { + if (doc.schema) { + this.traverseSchemaForDependencies(doc.schema, dependencies); + } + } + + // Separate extensions from marketplace apps using the extracted extension UIDs + if (dependencies.extensions.size > 0) { + const extensionUIDs = Array.from(dependencies.extensions); + log.info( + `Processing ${extensionUIDs.length} extensions to identify marketplace apps...`, + this.exportQueryConfig.context, + ); + + try { + const { extensions, marketplaceApps } = await this.fetchExtensionsAndMarketplaceApps(extensionUIDs); + dependencies.extensions = new Set(extensions); + dependencies.marketplaceApps = new Set(marketplaceApps); + log.info( + `Dependencies separated - Global Fields: ${dependencies.globalFields.size}, Extensions: ${dependencies.extensions.size}, Taxonomies: ${dependencies.taxonomies.size}, Marketplace Apps: ${dependencies.marketplaceApps.size}`, + this.exportQueryConfig.context, + ); + } catch (error) { + handleAndLogError(error, this.exportQueryConfig.context, 'Failed to separate extensions and Marketplace apps'); + // Keep original extensions if separation fails + } + } else { + log.info( + `Found dependencies - Global Fields: ${dependencies.globalFields.size}, Extensions: ${dependencies.extensions.size}, Taxonomies: ${dependencies.taxonomies.size}, Marketplace Apps: ${dependencies.marketplaceApps.size}`, + this.exportQueryConfig.context, + ); + } + + return dependencies; + } + + // Update the fetchExtensionsAndMarketplaceApps method to only fetch specific extension UIDs + async fetchExtensionsAndMarketplaceApps( + extensionUIDs: string[], + ): Promise<{ extensions: string[]; marketplaceApps: string[] }> { + log.info( + `Fetching details for ${extensionUIDs.length} extensions to identify marketplace apps...`, + this.exportQueryConfig.context, + ); + + try { + // Query parameters to include marketplace extensions + const queryParams = { + include_count: true, + include_marketplace_extensions: true, + query: { + uid: { $in: extensionUIDs }, + }, + }; + + // Fetch all extensions including marketplace apps + const response = await this.stackAPIClient.extension().query(queryParams).find(); + + if (!response || !response.items) { + log.warn(`No extensions found`, this.exportQueryConfig.context); + return { extensions: extensionUIDs, marketplaceApps: [] }; + } + + const marketplaceApps: string[] = []; + const regularExtensions: string[] = []; + + response.items.forEach((item: any) => { + if (item.app_uid && item.app_installation_uid) { + marketplaceApps.push(item.app_installation_uid); + } else { + regularExtensions.push(item.uid); + } + }); + + log.info( + `Identified ${marketplaceApps.length} marketplace apps and ${regularExtensions.length} regular extensions from ${extensionUIDs.length} total extensions`, + this.exportQueryConfig.context, + ); + + return { extensions: regularExtensions, marketplaceApps }; + } catch (error) { + handleAndLogError(error, this.exportQueryConfig.context, 'Failed to fetch extensions and Marketplace apps'); + return { extensions: extensionUIDs, marketplaceApps: [] }; + } + } + + private traverseSchemaForDependencies(schema: any[], dependencies: any): void { + for (const field of schema) { + // Global fields + if (field.data_type === 'global_field' && field.reference_to) { + dependencies.globalFields.add(field.reference_to); + } + + // Extensions + if (field.extension_uid) { + dependencies.extensions.add(field.extension_uid); + } + + // Taxonomies - UPDATED LOGIC + if (field.data_type === 'taxonomy' && field.taxonomies && Array.isArray(field.taxonomies)) { + field.taxonomies.forEach((tax: any) => { + if (tax.taxonomy_uid) { + dependencies.taxonomies.add(tax.taxonomy_uid); + } + }); + } + + // Recursive traversal for nested structures + if ((field.data_type === 'group' || field.data_type === 'global_field') && field.schema) { + this.traverseSchemaForDependencies(field.schema, dependencies); + } + + if (field.data_type === 'blocks' && field.blocks) { + for (const blockKey in field.blocks) { + if (field.blocks[blockKey].schema) { + this.traverseSchemaForDependencies(field.blocks[blockKey].schema, dependencies); + } + } + } + } + } +} diff --git a/packages/contentstack-query-export/src/utils/file-helper.ts b/packages/contentstack-query-export/src/utils/file-helper.ts new file mode 100644 index 000000000..7aa8c6323 --- /dev/null +++ b/packages/contentstack-query-export/src/utils/file-helper.ts @@ -0,0 +1,2 @@ +import { FsUtility, sanitizePath } from '@contentstack/cli-utilities'; +export const fsUtil = new FsUtility(); diff --git a/packages/contentstack-query-export/src/utils/index.ts b/packages/contentstack-query-export/src/utils/index.ts new file mode 100644 index 000000000..0687cbddc --- /dev/null +++ b/packages/contentstack-query-export/src/utils/index.ts @@ -0,0 +1,10 @@ +export * as fileHelper from './file-helper'; +export { fsUtil } from './file-helper'; +export { log, unlinkFileLogger, createLogContext } from './logger'; +export { LogContext } from '../types'; +export * from './common-helper'; +export * from './config-handler'; +export * from './content-type-helper'; +export * from './dependency-resolver'; +export * from './referenced-asset-handler'; +export { setupBranches } from './branch-helper'; diff --git a/packages/contentstack-query-export/src/utils/logger.ts b/packages/contentstack-query-export/src/utils/logger.ts new file mode 100644 index 000000000..9be2a2fe2 --- /dev/null +++ b/packages/contentstack-query-export/src/utils/logger.ts @@ -0,0 +1,184 @@ +/*! + * Contentstack Export + * Copyright (c) 2024 Contentstack LLC + * MIT Licensed + */ + +import * as winston from 'winston'; +import * as path from 'path'; +import mkdirp from 'mkdirp'; +import { QueryExportConfig, LogContext } from '../types'; +import { sanitizePath, redactObject, configHandler } from '@contentstack/cli-utilities'; +const slice = Array.prototype.slice; + +const ansiRegexPattern = [ + '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)', + '(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))', +].join('|'); + +function returnString(args: unknown[]) { + let returnStr = ''; + if (args && args.length) { + returnStr = args + .map(function (item) { + if (item && typeof item === 'object') { + try { + const redactedObject = redactObject(item); + if (redactedObject && typeof redactedObject === 'object') { + return JSON.stringify(redactedObject); + } + } catch (error) {} + return item; + } + return item; + }) + .join(' ') + .trim(); + } + returnStr = returnStr.replace(new RegExp(ansiRegexPattern, 'g'), '').trim(); + return returnStr; +} +const myCustomLevels = { + levels: { + warn: 1, + info: 2, + debug: 3, + }, + colors: { + //colors aren't being used anywhere as of now, we're using chalk to add colors while logging + info: 'blue', + debug: 'green', + warn: 'yellow', + error: 'red', + }, +}; + +let logger: winston.Logger; +let errorLogger: winston.Logger; + +let successTransport; +let errorTransport; + +function init(_logPath: string) { + if (!logger || !errorLogger) { + const logsDir = path.resolve(sanitizePath(_logPath), 'logs', 'export'); + // Create dir if doesn't already exist + mkdirp.sync(logsDir); + + successTransport = { + filename: path.join(sanitizePath(logsDir), 'success.log'), + maxFiles: 20, + maxsize: 1000000, + tailable: true, + level: 'info', + }; + + errorTransport = { + filename: path.join(sanitizePath(logsDir), 'error.log'), + maxFiles: 20, + maxsize: 1000000, + tailable: true, + level: 'error', + }; + + logger = winston.createLogger({ + transports: [ + new winston.transports.File(successTransport), + new winston.transports.Console({ format: winston.format.simple() }), + ], + levels: myCustomLevels.levels, + }); + + errorLogger = winston.createLogger({ + transports: [ + new winston.transports.File(errorTransport), + new winston.transports.Console({ + level: 'error', + format: winston.format.combine( + winston.format.colorize({ all: true, colors: { error: 'red' } }), + winston.format.simple(), + ), + }), + ], + levels: { error: 0 }, + }); + } + + return { + log: function (message: any) { + const args = slice.call(arguments); + const logString = returnString(args); + if (logString) { + logger.log('info', logString); + } + }, + warn: function () { + const args = slice.call(arguments); + const logString = returnString(args); + if (logString) { + logger.log('warn', logString); + } + }, + error: function (message: any) { + const args = slice.call(arguments); + const logString = returnString(args); + if (logString) { + errorLogger.log('error', logString); + } + }, + debug: function () { + const args = slice.call(arguments); + const logString = returnString(args); + if (logString) { + logger.log('debug', logString); + } + }, + }; +} + +export const log = async (config: QueryExportConfig, message: any, type: string) => { + const logsPath = sanitizePath(config.logsPath || config.dataPath); + // ignoring the type argument, as we are not using it to create a logfile anymore + if (type !== 'error') { + // removed type argument from init method + init(logsPath).log(message); + } else { + init(logsPath).error(message); + } +}; + +export const unlinkFileLogger = () => { + if (logger) { + const transports = logger.transports; + transports.forEach((transport: any) => { + if (transport.name === 'file') { + logger.remove(transport); + } + }); + } + + if (errorLogger) { + const transports = errorLogger.transports; + transports.forEach((transport: any) => { + if (transport.name === 'file') { + errorLogger.remove(transport); + } + }); + } +}; + +/** + * Creates a context object for logging from QueryExportConfig + */ +export function createLogContext(config: QueryExportConfig, moduleName?: string): LogContext { + return { + command: 'cm:stacks:export-query', + module: moduleName || '', + email: configHandler.get('email') || '', + sessionId: configHandler.get('sessionId') || '', + apiKey: config.stackApiKey || '', + orgId: configHandler.get('oauthOrgUid') || '', + authenticationMethod: config.managementToken ? 'Management Token' : 'Basic Auth', + }; +} + diff --git a/packages/contentstack-query-export/src/utils/query-parser.ts b/packages/contentstack-query-export/src/utils/query-parser.ts new file mode 100644 index 000000000..752a7fb8a --- /dev/null +++ b/packages/contentstack-query-export/src/utils/query-parser.ts @@ -0,0 +1,66 @@ +import * as fs from 'fs'; +import { CLIError, handleAndLogError } from '@contentstack/cli-utilities'; +import { QueryExportConfig } from '../types'; + +export class QueryParser { + private config: QueryExportConfig; + + constructor(config: QueryExportConfig) { + this.config = config; + } + + async parse(queryInput: string): Promise { + let query: any; + + // Check if it's a file path + if (queryInput.endsWith('.json') && fs.existsSync(queryInput)) { + query = this.parseFromFile(queryInput); + } else { + query = this.parseFromString(queryInput); + } + + this.validate(query); + return query; + } + + private parseFromFile(filePath: string): any { + try { + const content = fs.readFileSync(filePath, 'utf-8'); + return JSON.parse(content); + } catch (error) { + handleAndLogError(error, this.config.context, 'Failed to parse the query file'); + } + } + + private parseFromString(queryString: string): any { + try { + return JSON.parse(queryString); + } catch (error) { + handleAndLogError(error, this.config.context, 'Invalid JSON query'); + throw new CLIError('Invalid JSON query'); + } + } + + private validate(query: any): void { + if (!query || typeof query !== 'object') { + throw new CLIError('The query must be a valid JSON object.'); + } + + if (!query.modules || typeof query.modules !== 'object') { + throw new CLIError('The query must contain a "modules" object.'); + } + + const modules = Object.keys(query.modules); + if (modules.length === 0) { + throw new CLIError('The query must contain at least one module.'); + } + + // Validate supported modules + const queryableModules = this.config.modules.queryable; + for (const module of modules) { + if (!queryableModules.includes(module as any)) { + throw new CLIError(`Module "${module}" is not queryable. Supported modules: ${queryableModules.join(', ')}`); + } + } + } +} diff --git a/packages/contentstack-query-export/src/utils/read-content-type-schemas.ts b/packages/contentstack-query-export/src/utils/read-content-type-schemas.ts new file mode 100644 index 000000000..6450024d3 --- /dev/null +++ b/packages/contentstack-query-export/src/utils/read-content-type-schemas.ts @@ -0,0 +1,138 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { sanitizePath } from '@contentstack/cli-utilities'; +import { fsUtil } from './file-helper'; + +function normalizeToArray(raw: unknown): any[] { + if (raw == null) return []; + if (Array.isArray(raw)) return raw; + if (typeof raw === 'object') return Object.values(raw as Record); + return []; +} + +/** + * CLI v1 aggregate content types under `content_types/schema.json`. + */ +export function readContentTypesFromExportDir(dir: string): any[] { + const schemaPath = path.join(sanitizePath(dir), 'schema.json'); + try { + const raw = fsUtil.readFile(sanitizePath(schemaPath)); + return normalizeToArray(raw); + } catch { + return []; + } +} + +function readGlobalFieldSchemasFromSubdirs(base: string): any[] { + if (!fs.existsSync(base)) return []; + const out: any[] = []; + for (const name of fs.readdirSync(base)) { + if (name === 'globalfields.json' || name === 'schema.json') continue; + const full = path.join(base, name); + let st: fs.Stats; + try { + st = fs.statSync(full); + } catch { + continue; + } + if (!st.isDirectory()) { + if (name.endsWith('.json')) { + try { + const doc = fsUtil.readFile(sanitizePath(full)); + if (doc && typeof doc === 'object') out.push(doc); + } catch { + /* skip invalid */ + } + } + continue; + } + const uid = name; + const candidates = [path.join(full, `${uid}.json`), path.join(full, 'index.json')]; + let loaded = false; + for (const p of candidates) { + if (fs.existsSync(p)) { + try { + const doc = fsUtil.readFile(sanitizePath(p)); + if (doc && typeof doc === 'object') { + out.push(doc); + loaded = true; + break; + } + } catch { + /* try next */ + } + } + } + if (loaded) continue; + try { + for (const f of fs.readdirSync(full)) { + if (!f.endsWith('.json')) continue; + const doc = fsUtil.readFile(sanitizePath(path.join(full, f))); + if (doc && typeof doc === 'object') { + out.push(doc); + break; + } + } + } catch { + /* skip */ + } + } + return out; +} + +/** + * Global fields: prefer `global_fields/globalfields.json` (cm-export v1), then per-file / per-subfolder layouts. + */ +export function readGlobalFieldSchemasFromDir(dir: string): any[] { + const base = sanitizePath(dir); + const aggPath = path.join(base, 'globalfields.json'); + if (fs.existsSync(aggPath)) { + try { + const raw = fsUtil.readFile(sanitizePath(aggPath)); + const list = normalizeToArray(raw); + if (list.length > 0) return list; + } catch { + /* fall through to subdir scan */ + } + } + return readGlobalFieldSchemasFromSubdirs(base); +} + +/** + * Rebuild `content_types/schema.json` from every per–content-type `*.json` in the folder + * (excluding `schema.json` itself). Each `cm:stacks:export --module content-types` run + * overwrites `schema.json` with that run’s batch only; query-export runs that module + * multiple times, so without this merge the aggregate omits earlier types and the + * entries module (which only reads `schema.json`) skips their entries. + */ +export function rebuildContentTypesSchemaJson(ctDir: string): void { + const dir = sanitizePath(ctDir); + if (!fs.existsSync(dir)) return; + + const byUid = new Map(); + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json') || name === 'schema.json') continue; + const fp = path.join(dir, name); + let st: fs.Stats; + try { + st = fs.statSync(fp); + } catch { + continue; + } + if (!st.isFile()) continue; + try { + const doc = fsUtil.readFile(sanitizePath(fp)); + if (doc && typeof doc === 'object' && !Array.isArray(doc) && typeof (doc as { uid?: unknown }).uid === 'string') { + byUid.set((doc as { uid: string }).uid, doc); + } + } catch { + /* skip invalid JSON */ + } + } + + const merged = Array.from(byUid.values()); + if (merged.length === 0) return; + + const schemaPath = path.join(dir, 'schema.json'); + fsUtil.writeFile(sanitizePath(schemaPath), merged); +} diff --git a/packages/contentstack-query-export/src/utils/referenced-asset-handler.ts b/packages/contentstack-query-export/src/utils/referenced-asset-handler.ts new file mode 100644 index 000000000..247b15048 --- /dev/null +++ b/packages/contentstack-query-export/src/utils/referenced-asset-handler.ts @@ -0,0 +1,157 @@ +import * as path from 'path'; +import * as fs from 'fs'; +import { QueryExportConfig } from '../types'; +import { fsUtil } from './index'; +import { sanitizePath, log, formatError, handleAndLogError } from '@contentstack/cli-utilities'; + +export class AssetReferenceHandler { + private exportQueryConfig: QueryExportConfig; + private entriesDir: string; + + constructor(exportQueryConfig: QueryExportConfig) { + this.exportQueryConfig = exportQueryConfig; + this.entriesDir = path.join( + sanitizePath(exportQueryConfig.exportDir), + sanitizePath(exportQueryConfig.branchName || ''), + 'entries', + ); + } + + /** + * Extract all asset UIDs by processing entries file by file (memory efficient) + */ + extractReferencedAssets(): string[] { + log.info('Extracting referenced assets from entries...', this.exportQueryConfig.context); + + try { + if (!fs.existsSync(this.entriesDir)) { + log.warn('Entries directory does not exist', this.exportQueryConfig.context); + return []; + } + + // Global set to maintain unique asset UIDs across all files + const globalAssetUIDs = new Set(); + + // Get all JSON files + const jsonFiles = this.findAllJsonFiles(this.entriesDir); + + // Process files one by one + let totalEntriesProcessed = 0; + for (const jsonFile of jsonFiles) { + const entriesInFile = this.processSingleFile(jsonFile, globalAssetUIDs); + totalEntriesProcessed += entriesInFile; + } + + const result = Array.from(globalAssetUIDs); + log.info( + `Found ${result.length} unique asset UIDs from ${totalEntriesProcessed} entries across ${jsonFiles.length} files`, + this.exportQueryConfig.context, + ); + + return result; + } catch (error) { + handleAndLogError(error, this.exportQueryConfig.context, 'Failed to extract assets'); + return []; + } + } + + /** + * Process a single file and extract asset UIDs from all its entries + */ + private processSingleFile(filePath: string, globalAssetUIDs: Set): number { + // Skip index.json files + if (path.basename(filePath) === 'index.json') { + return 0; + } + + try { + const fileContent = fsUtil.readFile(sanitizePath(filePath)); + + if (!fileContent || typeof fileContent !== 'object') { + return 0; + } + + // Stringify the ENTIRE file content at once + const fileString = JSON.stringify(fileContent); + + // Extract all asset UIDs from the entire file + const assetUIDs = this.extractAssetUIDsFromString(fileString); + + // Add to global set + assetUIDs.forEach((uid) => globalAssetUIDs.add(uid)); + + // Count entries for logging + const entriesCount = Object.keys(fileContent).length; + log.debug(`Processed ${entriesCount} entries from ${path.basename(filePath)}`, this.exportQueryConfig.context); + + return entriesCount; + } catch (error) { + log.warn(`Failed to process file ${filePath}: ${formatError(error)}`, this.exportQueryConfig.context); + return 0; + } + } + + /** + * Extract asset UIDs from stringified content using multiple patterns + */ + private extractAssetUIDsFromString(content: string): string[] { + const assetUIDs = new Set(); + + // Pattern 1: HTML img tags with asset_uid + const htmlAssetRegex = / { + let cliuxInquireStub: SinonStub; + + beforeEach(() => { + restore(); + }); + + afterEach(() => { + restore(); + }); + + describe('askAPIKey', () => { + it('should prompt user for API key and return the response', async () => { + const mockApiKey = 'test-api-key-12345'; + + cliuxInquireStub = stub(cliux, 'inquire').resolves(mockApiKey); + + const result = await askAPIKey(); + + expect(result).to.equal(mockApiKey); + expect(cliuxInquireStub.calledOnce).to.be.true; + + const callArgs = cliuxInquireStub.firstCall.args[0]; + expect(callArgs.type).to.equal('input'); + expect(callArgs.message).to.equal('Enter the stack api key'); + expect(callArgs.name).to.equal('apiKey'); + }); + + it('should handle empty API key input', async () => { + const emptyApiKey = ''; + + cliuxInquireStub = stub(cliux, 'inquire').resolves(emptyApiKey); + + const result = await askAPIKey(); + + expect(result).to.equal(emptyApiKey); + expect(cliuxInquireStub.calledOnce).to.be.true; + }); + + it('should handle inquire errors', async () => { + const error = new Error('Inquire failed'); + + cliuxInquireStub = stub(cliux, 'inquire').rejects(error); + + try { + await askAPIKey(); + expect.fail('Expected an error to be thrown'); + } catch (err) { + expect(err.message).to.equal('Inquire failed'); + } + }); + + it('should validate the inquire call structure', async () => { + const mockApiKey = 'valid-api-key'; + + cliuxInquireStub = stub(cliux, 'inquire').resolves(mockApiKey); + + await askAPIKey(); + + expect(cliuxInquireStub.calledOnce).to.be.true; + + const inquireOptions = cliuxInquireStub.firstCall.args[0]; + expect(inquireOptions).to.have.property('type', 'input'); + expect(inquireOptions).to.have.property('message', 'Enter the stack api key'); + expect(inquireOptions).to.have.property('name', 'apiKey'); + }); + }); +}); diff --git a/packages/contentstack-query-export/test/unit/config-handler.test.ts b/packages/contentstack-query-export/test/unit/config-handler.test.ts new file mode 100644 index 000000000..bb4cb45c5 --- /dev/null +++ b/packages/contentstack-query-export/test/unit/config-handler.test.ts @@ -0,0 +1,349 @@ +import { expect } from 'chai'; +import * as sinon from 'sinon'; +import * as path from 'path'; +import { setupQueryExportConfig } from '../../src/utils/config-handler'; +import * as commonHelper from '../../src/utils/common-helper'; + +// Mock the external utilities module +const mockCliUtilities = { + sanitizePath: sinon.stub(), + pathValidator: sinon.stub(), + configHandler: { + get: sinon.stub(), + }, + isAuthenticated: sinon.stub(), +}; + +describe('Config Handler', () => { + let sandbox: sinon.SinonSandbox; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + + // Set up default mock behavior to avoid interactive prompts + mockCliUtilities.sanitizePath.returns('./mocked-export-dir'); + mockCliUtilities.pathValidator.returns('./mocked-path'); + mockCliUtilities.configHandler.get.returns(null); + mockCliUtilities.isAuthenticated.returns(false); // Default to not authenticated to avoid prompts + + // Stub our own helper to prevent prompts + sandbox.stub(commonHelper, 'askAPIKey').resolves('mocked-api-key'); + }); + + afterEach(() => { + sandbox.restore(); + // Reset mock stubs + mockCliUtilities.sanitizePath.reset(); + mockCliUtilities.pathValidator.reset(); + mockCliUtilities.configHandler.get.reset(); + mockCliUtilities.isAuthenticated.reset(); + }); + + describe('setupQueryExportConfig', () => { + describe('with minimal flags', () => { + it('should create config with default values', async () => { + const flags = { + query: 'content_type_uid:page', + 'stack-api-key': 'test-stack-api-key', // Provide API key to avoid prompts + }; + + try { + const config = await setupQueryExportConfig(flags); + + expect(config).to.be.an('object'); + expect(config.query).to.equal('content_type_uid:page'); + expect(config.skipReferences).to.be.false; + expect(config.skipDependencies).to.be.false; + expect(config.securedAssets).to.be.false; + expect(config.isQueryBasedExport).to.be.true; + expect(config.stackApiKey).to.equal('test-stack-api-key'); + expect(config.exportDir).to.be.a('string'); + expect(config.logsPath).to.be.a('string'); + expect(config.dataPath).to.be.a('string'); + expect(config.externalConfigPath).to.include('export-config.json'); + } catch (error) { + // May fail due to other authentication requirements, but not API key prompts + expect(error).to.be.an('error'); + } + }); + }); + + describe('with custom data directory', () => { + it('should use custom data directory when provided', async () => { + const flags = { + 'data-dir': './custom-export', + query: 'content_type_uid:blog', + 'stack-api-key': 'test-stack-key', + }; + + try { + const config = await setupQueryExportConfig(flags); + expect(config.exportDir).to.be.a('string').and.include('custom-export'); + expect(config.logsPath).to.be.a('string').and.include('custom-export'); + expect(config.dataPath).to.be.a('string').and.include('custom-export'); + } catch (error) { + // May fail due to authentication, but we can test the flag handling + expect(flags['data-dir']).to.equal('./custom-export'); + } + }); + }); + + describe('with skip flags', () => { + it('should set skip flags when provided', async () => { + const flags = { + query: 'content_type_uid:article', + 'skip-references': true, + 'skip-dependencies': true, + 'secured-assets': true, + 'stack-api-key': 'test-stack-api-key', // Provide API key to avoid prompts + }; + + try { + const config = await setupQueryExportConfig(flags); + expect(config.skipReferences).to.be.true; + expect(config.skipDependencies).to.be.true; + expect(config.securedAssets).to.be.true; + } catch (error) { + // Test flag mapping even if authentication fails + expect(flags['skip-references']).to.be.true; + expect(flags['skip-dependencies']).to.be.true; + expect(flags['secured-assets']).to.be.true; + } + }); + }); + + describe('with branch name', () => { + it('should include branch name when provided', async () => { + const flags = { + query: 'content_type_uid:news', + branch: 'development', + 'stack-api-key': 'test-stack-api-key', // Provide API key to avoid prompts + }; + + try { + const config = await setupQueryExportConfig(flags); + expect(config.branchName).to.equal('development'); + } catch (error) { + // Test branch assignment + expect(flags.branch).to.equal('development'); + } + }); + }); + + describe('external config path', () => { + it('should set external config path correctly', async () => { + const flags = { + query: 'content_type_uid:test', + 'stack-api-key': 'test-stack-api-key', // Provide API key to avoid prompts + }; + + try { + const config = await setupQueryExportConfig(flags); + expect(config.externalConfigPath).to.be.a('string').and.include('export-config.json'); + expect(path.isAbsolute(config.externalConfigPath || '')).to.be.true; + } catch (error) { + // Test path construction logic + const expectedPath = path.join(__dirname, '../config/export-config.json'); + expect(expectedPath).to.include('export-config.json'); + } + }); + }); + + describe('stack API key handling', () => { + it('should use provided stack API key', async () => { + const flags = { + query: 'content_type_uid:product', + 'stack-api-key': 'blt123456789', + }; + + try { + const config = await setupQueryExportConfig(flags); + expect(config.stackApiKey).to.equal('blt123456789'); + } catch (error) { + // Verify flag is captured even if auth fails + expect(flags['stack-api-key']).to.equal('blt123456789'); + } + }); + + it('should handle empty stack API key', async () => { + const flags = { + query: 'content_type_uid:empty', + // Intentionally not providing stack-api-key to test this scenario + }; + + try { + const config = await setupQueryExportConfig(flags); + expect(config.stackApiKey).to.be.a('string'); + } catch (error) { + // Expected when not authenticated: explicit error, not an interactive prompt + expect(error.message).to.include('Log in'); + } + }); + }); + + describe('configuration object structure', () => { + it('should include all required configuration properties', async () => { + const flags = { + query: 'content_type_uid:structure_test', + 'stack-api-key': 'test-key', + }; + + try { + const config = await setupQueryExportConfig(flags); + + // Test required properties exist + expect(config).to.have.property('exportDir'); + expect(config).to.have.property('stackApiKey'); + expect(config).to.have.property('query'); + expect(config).to.have.property('skipReferences'); + expect(config).to.have.property('skipDependencies'); + expect(config).to.have.property('securedAssets'); + expect(config).to.have.property('isQueryBasedExport'); + expect(config).to.have.property('logsPath'); + expect(config).to.have.property('dataPath'); + expect(config).to.have.property('externalConfigPath'); + + // Test property types + expect(config.exportDir).to.be.a('string'); + expect(config.stackApiKey).to.be.a('string'); + expect(config.query).to.be.a('string'); + expect(config.skipReferences).to.be.a('boolean'); + expect(config.skipDependencies).to.be.a('boolean'); + expect(config.securedAssets).to.be.a('boolean'); + expect(config.isQueryBasedExport).to.be.a('boolean'); + expect(config.logsPath).to.be.a('string'); + expect(config.dataPath).to.be.a('string'); + expect(config.externalConfigPath).to.be.a('string'); + } catch (error) { + // Test flag structure even if config creation fails + expect(flags).to.have.property('query'); + expect(flags.query).to.be.a('string'); + } + }); + + it('should set isQueryBasedExport to true', async () => { + const flags = { + query: 'content_type_uid:query_based', + 'stack-api-key': 'test-stack-api-key', // Provide API key to avoid prompts + }; + + try { + const config = await setupQueryExportConfig(flags); + expect(config.isQueryBasedExport).to.be.true; + } catch (error) { + // This property should always be true for query-based exports + expect(true).to.be.true; // Placeholder assertion + } + }); + }); + + describe('error scenarios', () => { + it('should handle missing query parameter', async () => { + const flags = { + 'stack-api-key': 'test-stack-api-key', // Provide API key to avoid prompts + }; + + try { + const config = await setupQueryExportConfig(flags); + expect(config.query).to.be.undefined; + } catch (error) { + // Query might be required, test error handling + expect(error).to.be.an('error'); + } + }); + + it('should handle invalid flag types', async () => { + const flags = { + query: 123, // Invalid type + 'skip-references': 'not-boolean', + 'stack-api-key': 'test-stack-api-key', // Provide API key to avoid prompts + }; + + try { + const config = await setupQueryExportConfig(flags); + // Test type coercion + expect(config.query).to.equal(123); + } catch (error) { + expect(error).to.be.an('error'); + } + }); + }); + + describe('path handling', () => { + it('should ensure paths are consistent', async () => { + const flags = { + query: 'content_type_uid:path_test', + 'data-dir': './test-export', + 'stack-api-key': 'test-stack-api-key', // Provide API key to avoid prompts + }; + + try { + const config = await setupQueryExportConfig(flags); + expect(config.exportDir).to.equal(config.logsPath); + expect(config.exportDir).to.equal(config.dataPath); + expect(config.exportDir).to.be.a('string').and.include('test-export'); + } catch (error) { + // Test path consistency logic + expect(flags['data-dir']).to.equal('./test-export'); + } + }); + + it('should handle absolute paths', async () => { + const absolutePath = path.resolve('./absolute-test'); + const flags = { + query: 'content_type_uid:absolute', + 'data-dir': absolutePath, + 'stack-api-key': 'test-stack-api-key', // Provide API key to avoid prompts + }; + + try { + const config = await setupQueryExportConfig(flags); + expect(path.isAbsolute(config.exportDir || '')).to.be.true; + } catch (error) { + // Test absolute path handling + expect(path.isAbsolute(absolutePath)).to.be.true; + } + }); + }); + + describe('askAPIKey integration', () => { + it('should call askAPIKey when no stack API key provided', async () => { + // This test should fail with authentication error, not call askAPIKey in our mock setup + const flags = { + query: 'content_type_uid:prompt_test', + // Intentionally not providing stack-api-key to test this scenario + }; + + try { + await setupQueryExportConfig(flags); + expect.fail('Should have thrown authentication error'); + } catch (error) { + // Expected to fail due to authentication requirements + expect(error.message).to.match(/login|Please login|authentication|token/i); + } + }); + + it('should handle askAPIKey returning non-string value', async () => { + // Override the default stub to return invalid value for this specific test + sandbox.restore(); // Clear existing stubs + sandbox.stub(commonHelper, 'askAPIKey').resolves(undefined as any); + + // Mock isAuthenticated to return true to trigger the askAPIKey path + const mockIsAuthenticated = sandbox.stub().returns(true); + + const flags = { + query: 'content_type_uid:invalid_key', + // Not providing stack-api-key to trigger askAPIKey path + }; + + try { + await setupQueryExportConfig(flags); + expect.fail('Should have thrown error for invalid API key'); + } catch (error) { + // Should fail due to authentication or other issues, test completed + expect(error).to.be.an('error'); + } + }); + }); + }); +}); diff --git a/packages/contentstack-query-export/test/unit/content-type-helper.test.ts b/packages/contentstack-query-export/test/unit/content-type-helper.test.ts new file mode 100644 index 000000000..60606f44e --- /dev/null +++ b/packages/contentstack-query-export/test/unit/content-type-helper.test.ts @@ -0,0 +1,502 @@ +import { expect } from 'chai'; +import { stub, restore, SinonStub } from 'sinon'; +import * as path from 'path'; +import { ReferencedContentTypesHandler } from '../../src/utils/content-type-helper'; +import * as logger from '../../src/utils/logger'; +import { QueryExportConfig } from '../../src/types'; + +describe('Content Type Helper Utilities', () => { + let handler: ReferencedContentTypesHandler; + let mockConfig: QueryExportConfig; + let logStub: SinonStub; + let pathJoinStub: SinonStub; + + beforeEach(() => { + mockConfig = { + maxCTReferenceDepth: 20, + contentVersion: 2, + host: 'https://api.contentstack.io/v3', + exportDir: '/test/export', + stackApiKey: 'test-api-key', + managementToken: 'test-token', + query: '', + skipReferences: false, + skipDependencies: false, + branchName: 'main', + securedAssets: false, + isQueryBasedExport: true, + logsPath: '/test/logs', + dataPath: '/test/data', + modules: { + general: ['stack', 'locales', 'environments'], + queryable: ['content-types'], + dependent: ['global-fields', 'extensions', 'taxonomies'], + content: ['entries', 'assets'], + exportOrder: ['stack', 'content-types'], + }, + queryConfig: { + maxRecursionDepth: 10, + batchSize: 100, + metadataFileName: '_query-meta.json', + validation: { + maxQueryDepth: 5, + maxArraySize: 1000, + allowedDateFormats: ['ISO8601'], + }, + }, + fetchConcurrency: 5, + writeConcurrency: 5, + apis: { + stacks: '/stacks/', + locales: '/locales/', + environments: '/environments/', + content_types: '/content_types/', + global_fields: '/global_fields/', + extensions: '/extensions/', + taxonomies: '/taxonomies/', + entries: '/entries/', + assets: '/assets/', + }, + }; + + handler = new ReferencedContentTypesHandler(mockConfig); + restore(); + }); + + afterEach(() => { + restore(); + }); + + describe('extractReferencedContentTypes', () => { + it('should extract reference field targets', async () => { + const contentTypeBatch = [ + { + uid: 'blog', + schema: [ + { + uid: 'author', + data_type: 'reference', + reference_to: ['author', 'editor'], + }, + { + uid: 'category', + data_type: 'reference', + reference_to: ['category'], + }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + + const result = await handler.extractReferencedContentTypes(contentTypeBatch); + + expect(result).to.deep.equal(['author', 'editor', 'category']); + }); + + it('should exclude sys_assets from references', async () => { + const contentTypeBatch = [ + { + uid: 'blog', + schema: [ + { + uid: 'image', + data_type: 'reference', + reference_to: ['sys_assets', 'custom_asset'], + }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + + const result = await handler.extractReferencedContentTypes(contentTypeBatch); + + expect(result).to.deep.equal(['custom_asset']); + expect(result).to.not.include('sys_assets'); + }); + + it('should handle group fields with nested schemas', async () => { + const contentTypeBatch = [ + { + uid: 'blog', + schema: [ + { + uid: 'metadata', + data_type: 'group', + schema: [ + { + uid: 'author', + data_type: 'reference', + reference_to: ['author'], + }, + ], + }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + + const result = await handler.extractReferencedContentTypes(contentTypeBatch); + + expect(result).to.deep.equal(['author']); + }); + + it('should handle global fields with nested schemas', async () => { + const contentTypeBatch = [ + { + uid: 'blog', + schema: [ + { + uid: 'seo', + data_type: 'global_field', + schema: [ + { + uid: 'related_page', + data_type: 'reference', + reference_to: ['page'], + }, + ], + }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + + const result = await handler.extractReferencedContentTypes(contentTypeBatch); + + expect(result).to.deep.equal(['page']); + }); + + it('should handle blocks with nested schemas', async () => { + const contentTypeBatch = [ + { + uid: 'page', + schema: [ + { + uid: 'content_blocks', + data_type: 'blocks', + blocks: { + hero_block: { + schema: [ + { + uid: 'background_image', + data_type: 'reference', + reference_to: ['image_gallery'], + }, + ], + }, + testimonial_block: { + schema: [ + { + uid: 'testimonial', + data_type: 'reference', + reference_to: ['testimonial'], + }, + ], + }, + }, + }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + + const result = await handler.extractReferencedContentTypes(contentTypeBatch); + + expect(result).to.deep.equal(['image_gallery', 'testimonial']); + }); + + it('should handle JSON RTE with embedded entries', async () => { + const contentTypeBatch = [ + { + uid: 'article', + schema: [ + { + uid: 'content', + data_type: 'json', + field_metadata: { + rich_text_type: true, + embed_entry: true, + }, + reference_to: ['related_article', 'quote'], + }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + + const result = await handler.extractReferencedContentTypes(contentTypeBatch); + + expect(result).to.deep.equal(['related_article', 'quote']); + }); + + it('should handle Text RTE with embedded entries', async () => { + const contentTypeBatch = [ + { + uid: 'article', + schema: [ + { + uid: 'content', + data_type: 'text', + field_metadata: { + rich_text_type: true, + embed_entry: true, + }, + reference_to: ['related_article'], + }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + + const result = await handler.extractReferencedContentTypes(contentTypeBatch); + + expect(result).to.deep.equal(['related_article']); + }); + + it('should handle content types without schemas', async () => { + const contentTypeBatch = [ + { + uid: 'simple', + // No schema property + }, + { + uid: 'with_schema', + schema: [ + { + uid: 'reference_field', + data_type: 'reference', + reference_to: ['author'], + }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + + const result = await handler.extractReferencedContentTypes(contentTypeBatch); + + expect(result).to.deep.equal(['author']); + }); + + it('should return empty array for content types with no references', async () => { + const contentTypeBatch = [ + { + uid: 'simple', + schema: [ + { + uid: 'title', + data_type: 'text', + }, + { + uid: 'description', + data_type: 'text', + }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + + const result = await handler.extractReferencedContentTypes(contentTypeBatch); + + expect(result).to.deep.equal([]); + }); + + it('should handle complex nested structures', async () => { + const contentTypeBatch = [ + { + uid: 'complex_page', + schema: [ + { + uid: 'sections', + data_type: 'group', + schema: [ + { + uid: 'content_blocks', + data_type: 'blocks', + blocks: { + hero: { + schema: [ + { + uid: 'author', + data_type: 'reference', + reference_to: ['author'], + }, + { + uid: 'nested_group', + data_type: 'group', + schema: [ + { + uid: 'category', + data_type: 'reference', + reference_to: ['category'], + }, + ], + }, + ], + }, + }, + }, + ], + }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + + const result = await handler.extractReferencedContentTypes(contentTypeBatch); + + expect(result).to.deep.equal(['author', 'category']); + }); + + it('should remove duplicates from referenced content types', async () => { + const contentTypeBatch = [ + { + uid: 'blog1', + schema: [ + { + uid: 'author1', + data_type: 'reference', + reference_to: ['author', 'category'], + }, + ], + }, + { + uid: 'blog2', + schema: [ + { + uid: 'author2', + data_type: 'reference', + reference_to: ['author', 'tag'], + }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + + const result = await handler.extractReferencedContentTypes(contentTypeBatch); + + expect(result).to.deep.equal(['author', 'category', 'tag']); + expect(result.filter((item) => item === 'author')).to.have.length(1); + }); + }); + + describe('filterNewlyFetchedContentTypes', () => { + it('should filter out content types that were previously fetched', () => { + const allContentTypes = [ + { uid: 'blog', title: 'Blog' }, + { uid: 'author', title: 'Author' }, + { uid: 'category', title: 'Category' }, + { uid: 'tag', title: 'Tag' }, + ]; + + const previousUIDs = new Set(['blog', 'category']); + + const result = handler.filterNewlyFetchedContentTypes(allContentTypes, previousUIDs); + + expect(result).to.deep.equal([ + { uid: 'author', title: 'Author' }, + { uid: 'tag', title: 'Tag' }, + ]); + }); + + it('should return all content types when no previous UIDs', () => { + const allContentTypes = [ + { uid: 'blog', title: 'Blog' }, + { uid: 'author', title: 'Author' }, + ]; + + const previousUIDs = new Set(); + + const result = handler.filterNewlyFetchedContentTypes(allContentTypes, previousUIDs); + + expect(result).to.deep.equal(allContentTypes); + }); + + it('should return empty array when all content types were previously fetched', () => { + const allContentTypes = [ + { uid: 'blog', title: 'Blog' }, + { uid: 'author', title: 'Author' }, + ]; + + const previousUIDs = new Set(['blog', 'author']); + + const result = handler.filterNewlyFetchedContentTypes(allContentTypes, previousUIDs); + + expect(result).to.deep.equal([]); + }); + + it('should handle empty content types array', () => { + const allContentTypes: any[] = []; + const previousUIDs = new Set(['blog']); + + const result = handler.filterNewlyFetchedContentTypes(allContentTypes, previousUIDs); + + expect(result).to.deep.equal([]); + }); + }); + + describe('extractReferencedContentTypes β€” global field and guard behaviour', () => { + it('should not throw when a global_field has no schema property (stub form)', async () => { + const batch = [ + { + uid: 'page', + schema: [ + { + uid: 'seo', + data_type: 'global_field', + reference_to: 'seo_gf', + // no schema array β€” this is the "stub" representation in a CT's inline schema + }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + const result = await handler.extractReferencedContentTypes(batch); + expect(result).to.deep.equal([]); + }); + + it('should find CT references inside a global field document passed directly', async () => { + // The caller passes both the CT doc and the GF doc in the same batch. + const batch = [ + { + uid: 'page', + schema: [{ uid: 'seo', data_type: 'global_field', reference_to: 'seo_gf' }], + }, + { + uid: 'seo_gf', + schema: [{ uid: 'author_ref', data_type: 'reference', reference_to: ['author'] }], + }, + ]; + + logStub = stub(logger, 'log'); + const result = await handler.extractReferencedContentTypes(batch); + expect(result).to.include('author'); + }); + + it('should not recurse into global_field stub when schema is an empty array', async () => { + const batch = [ + { + uid: 'page', + schema: [ + { uid: 'seo', data_type: 'global_field', reference_to: 'seo_gf', schema: [] as any[] }, + ], + }, + ]; + + logStub = stub(logger, 'log'); + const result = await handler.extractReferencedContentTypes(batch); + expect(result).to.deep.equal([]); + }); + }); +}); diff --git a/packages/contentstack-query-export/test/unit/dependency-resolver.test.ts b/packages/contentstack-query-export/test/unit/dependency-resolver.test.ts new file mode 100644 index 000000000..bdf91ffee --- /dev/null +++ b/packages/contentstack-query-export/test/unit/dependency-resolver.test.ts @@ -0,0 +1,651 @@ +import { expect } from 'chai'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import * as sinon from 'sinon'; +import { ContentTypeDependenciesHandler } from '../../src/utils/dependency-resolver'; +import { QueryExportConfig } from '../../src/types'; + +describe('Dependency Resolver Utilities', () => { + let handler: ContentTypeDependenciesHandler; + let mockConfig: QueryExportConfig; + let mockStackAPIClient: any; + + beforeEach(() => { + // Create a mock stack API client + mockStackAPIClient = { + extension: sinon.stub().returns({ + query: sinon.stub().returns({ + find: sinon.stub().resolves({ + items: [], + }), + }), + }), + }; + + mockConfig = { + maxCTReferenceDepth: 20, + contentVersion: 1, + host: 'https://api.contentstack.io/v3', + exportDir: '/test/export', + stackApiKey: 'test-api-key', + managementToken: 'test-token', + query: '', + skipReferences: false, + skipDependencies: false, + branchName: 'main', + securedAssets: false, + isQueryBasedExport: true, + logsPath: '/test/logs', + dataPath: '/test/data', + modules: { + general: ['stack', 'locales', 'environments'], + queryable: ['content-types'], + dependent: ['global-fields', 'extensions', 'taxonomies'], + content: ['entries', 'assets'], + exportOrder: ['stack', 'content-types'], + }, + queryConfig: { + maxRecursionDepth: 10, + batchSize: 100, + metadataFileName: '_query-meta.json', + validation: { + maxQueryDepth: 5, + maxArraySize: 1000, + allowedDateFormats: ['ISO8601'], + }, + }, + fetchConcurrency: 5, + writeConcurrency: 5, + apis: { + stacks: '/stacks/', + locales: '/locales/', + environments: '/environments/', + content_types: '/content_types/', + global_fields: '/global_fields/', + extensions: '/extensions/', + taxonomies: '/taxonomies/', + entries: '/entries/', + assets: '/assets/', + }, + }; + + // Fix: Pass both required arguments to the constructor + handler = new ContentTypeDependenciesHandler(mockStackAPIClient, mockConfig); + }); + + describe('Schema dependency extraction logic', () => { + it('should extract global field dependencies from schema', () => { + const schema = [ + { + uid: 'seo', + data_type: 'global_field', + reference_to: 'seo_fields', + }, + { + uid: 'metadata', + data_type: 'global_field', + reference_to: 'common_metadata', + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + // Access private method for testing + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.globalFields.has('seo_fields')).to.be.true; + expect(dependencies.globalFields.has('common_metadata')).to.be.true; + expect(dependencies.globalFields.size).to.equal(2); + }); + + it('should extract extension dependencies from schema', () => { + const schema = [ + { + uid: 'rich_text', + data_type: 'text', + extension_uid: 'rich_text_editor', + }, + { + uid: 'color_picker', + data_type: 'text', + extension_uid: 'color_picker_ext', + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.extensions.has('rich_text_editor')).to.be.true; + expect(dependencies.extensions.has('color_picker_ext')).to.be.true; + expect(dependencies.extensions.size).to.equal(2); + }); + + it('should extract taxonomy dependencies from schema', () => { + const schema = [ + { + uid: 'categories', + data_type: 'taxonomy', + taxonomies: [{ taxonomy_uid: 'product_categories' }, { taxonomy_uid: 'product_tags' }], + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.taxonomies.has('product_categories')).to.be.true; + expect(dependencies.taxonomies.has('product_tags')).to.be.true; + expect(dependencies.taxonomies.size).to.equal(2); + }); + + it('should handle group fields with nested dependencies', () => { + const schema = [ + { + uid: 'content_section', + data_type: 'group', + schema: [ + { + uid: 'seo', + data_type: 'global_field', + reference_to: 'nested_seo', + }, + { + uid: 'rich_content', + data_type: 'text', + extension_uid: 'nested_editor', + }, + ], + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.globalFields.has('nested_seo')).to.be.true; + expect(dependencies.extensions.has('nested_editor')).to.be.true; + }); + + it('should handle block fields with nested dependencies', () => { + const schema = [ + { + uid: 'content_blocks', + data_type: 'blocks', + blocks: { + hero_block: { + schema: [ + { + uid: 'seo', + data_type: 'global_field', + reference_to: 'hero_seo', + }, + ], + }, + content_block: { + schema: [ + { + uid: 'editor', + data_type: 'text', + extension_uid: 'content_editor', + }, + { + uid: 'tags', + data_type: 'taxonomy', + taxonomies: [{ taxonomy_uid: 'content_tags' }], + }, + ], + }, + }, + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.globalFields.has('hero_seo')).to.be.true; + expect(dependencies.extensions.has('content_editor')).to.be.true; + expect(dependencies.taxonomies.has('content_tags')).to.be.true; + }); + + it('should handle complex nested structures', () => { + const schema = [ + { + uid: 'sections', + data_type: 'group', + schema: [ + { + uid: 'content_blocks', + data_type: 'blocks', + blocks: { + nested_block: { + schema: [ + { + uid: 'nested_group', + data_type: 'group', + schema: [ + { + uid: 'deep_global', + data_type: 'global_field', + reference_to: 'deep_nested_global', + }, + ], + }, + ], + }, + }, + }, + ], + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.globalFields.has('deep_nested_global')).to.be.true; + }); + + it('should ignore fields without dependency information', () => { + const schema = [ + { + uid: 'title', + data_type: 'text', + }, + { + uid: 'description', + data_type: 'text', + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.globalFields.size).to.equal(0); + expect(dependencies.extensions.size).to.equal(0); + expect(dependencies.taxonomies.size).to.equal(0); + }); + + it('should handle taxonomies without taxonomy_uid gracefully', () => { + const schema = [ + { + uid: 'categories', + data_type: 'taxonomy', + taxonomies: [ + { name: 'Category 1' }, // Missing taxonomy_uid + { taxonomy_uid: 'valid_taxonomy' }, + ], + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.taxonomies.has('valid_taxonomy')).to.be.true; + expect(dependencies.taxonomies.size).to.equal(1); + }); + + it('should handle mixed dependency types in single schema', () => { + const schema = [ + { + uid: 'seo', + data_type: 'global_field', + reference_to: 'seo_global', + }, + { + uid: 'rich_text', + data_type: 'text', + extension_uid: 'editor_ext', + }, + { + uid: 'categories', + data_type: 'taxonomy', + taxonomies: [{ taxonomy_uid: 'categories_tax' }], + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.globalFields.has('seo_global')).to.be.true; + expect(dependencies.extensions.has('editor_ext')).to.be.true; + expect(dependencies.taxonomies.has('categories_tax')).to.be.true; + expect(dependencies.globalFields.size).to.equal(1); + expect(dependencies.extensions.size).to.equal(1); + expect(dependencies.taxonomies.size).to.equal(1); + }); + + it('should handle empty schema arrays', () => { + const schema: any[] = []; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.globalFields.size).to.equal(0); + expect(dependencies.extensions.size).to.equal(0); + expect(dependencies.taxonomies.size).to.equal(0); + }); + + it('should collect nested global field inside a global field schema', () => { + const schema = [ + { + uid: 'outer_global', + data_type: 'global_field', + reference_to: 'outer_gf_uid', + schema: [ + { + uid: 'inner_global', + data_type: 'global_field', + reference_to: 'inner_gf_uid', + }, + ], + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.globalFields.has('outer_gf_uid')).to.be.true; + expect(dependencies.globalFields.has('inner_gf_uid')).to.be.true; + expect(dependencies.globalFields.size).to.equal(2); + }); + + it('should collect extension nested inside a global field schema', () => { + const schema = [ + { + uid: 'seo_block', + data_type: 'global_field', + reference_to: 'seo_gf', + schema: [ + { + uid: 'rich_editor', + data_type: 'text', + extension_uid: 'nested_editor_ext', + }, + ], + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.globalFields.has('seo_gf')).to.be.true; + expect(dependencies.extensions.has('nested_editor_ext')).to.be.true; + }); + + it('should collect taxonomy nested inside a global field schema', () => { + const schema = [ + { + uid: 'tags_block', + data_type: 'global_field', + reference_to: 'tags_gf', + schema: [ + { + uid: 'categories', + data_type: 'taxonomy', + taxonomies: [{ taxonomy_uid: 'nested_taxonomy_uid' }], + }, + ], + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.globalFields.has('tags_gf')).to.be.true; + expect(dependencies.taxonomies.has('nested_taxonomy_uid')).to.be.true; + }); + + it('should collect deeply nested global field inside a global field inside a group', () => { + const schema = [ + { + uid: 'content_section', + data_type: 'group', + schema: [ + { + uid: 'outer_gf', + data_type: 'global_field', + reference_to: 'outer_gf_uid', + schema: [ + { + uid: 'inner_gf', + data_type: 'global_field', + reference_to: 'inner_gf_uid', + schema: [ + { + uid: 'deepest_gf', + data_type: 'global_field', + reference_to: 'deepest_gf_uid', + }, + ], + }, + ], + }, + ], + }, + ]; + + const dependencies = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + }; + + (handler as any).traverseSchemaForDependencies(schema, dependencies); + + expect(dependencies.globalFields.has('outer_gf_uid')).to.be.true; + expect(dependencies.globalFields.has('inner_gf_uid')).to.be.true; + expect(dependencies.globalFields.has('deepest_gf_uid')).to.be.true; + expect(dependencies.globalFields.size).to.equal(3); + }); + }); + + describe('extractDependencies β€” explicit schemas parameter', () => { + let extensionQueryStub: sinon.SinonStub; + + beforeEach(() => { + extensionQueryStub = sinon.stub().returns({ find: sinon.stub().resolves({ items: [] }) }); + mockStackAPIClient.extension = sinon.stub().returns({ query: extensionQueryStub }); + handler = new ContentTypeDependenciesHandler(mockStackAPIClient, mockConfig); + }); + + it('should collect global field deps from provided CT schemas', async () => { + const schemas = [ + { uid: 'page', schema: [{ uid: 'seo', data_type: 'global_field', reference_to: 'seo_gf' }] }, + ]; + + const deps = await handler.extractDependencies(schemas); + + expect(deps.globalFields.has('seo_gf')).to.be.true; + }); + + it('should collect global field deps from provided GF schemas (transitive case)', async () => { + // GF A's schema contains a reference to GF B β€” simulates what happens when + // the caller passes the combined [CT doc, GF A doc] list. + const schemas = [ + { uid: 'page', schema: [{ uid: 'gf_a_field', data_type: 'global_field', reference_to: 'gf_a' }] }, + { + uid: 'gf_a', + schema: [{ uid: 'gf_b_field', data_type: 'global_field', reference_to: 'gf_b' }], + }, + ]; + + const deps = await handler.extractDependencies(schemas); + + expect(deps.globalFields.has('gf_a')).to.be.true; + expect(deps.globalFields.has('gf_b')).to.be.true; + }); + + it('should collect extension deps from GF schemas', async () => { + // Return the extension as a regular extension from the API so it ends up in deps.extensions. + extensionQueryStub = sinon.stub().returns({ + find: sinon.stub().resolves({ items: [{ uid: 'color_picker_ext' }] }), + }); + mockStackAPIClient.extension = sinon.stub().returns({ query: extensionQueryStub }); + handler = new ContentTypeDependenciesHandler(mockStackAPIClient, mockConfig); + + const schemas = [ + { uid: 'page', schema: [{ uid: 'gf_a_field', data_type: 'global_field', reference_to: 'gf_a' }] }, + { + uid: 'gf_a', + schema: [{ uid: 'bg_color', data_type: 'text', extension_uid: 'color_picker_ext' }], + }, + ]; + + const deps = await handler.extractDependencies(schemas); + + expect(deps.globalFields.has('gf_a')).to.be.true; + expect(deps.extensions.has('color_picker_ext')).to.be.true; + }); + + it('should collect taxonomy deps from GF schemas', async () => { + const schemas = [ + { uid: 'page', schema: [{ uid: 'gf_a_field', data_type: 'global_field', reference_to: 'gf_a' }] }, + { + uid: 'gf_a', + schema: [ + { + uid: 'tags', + data_type: 'taxonomy', + taxonomies: [{ taxonomy_uid: 'product_taxonomy' }], + }, + ], + }, + ]; + + const deps = await handler.extractDependencies(schemas); + + expect(deps.taxonomies.has('product_taxonomy')).to.be.true; + }); + + it('should return empty sets when schemas array is empty', async () => { + const deps = await handler.extractDependencies([]); + + expect(deps.globalFields.size).to.equal(0); + expect(deps.extensions.size).to.equal(0); + expect(deps.taxonomies.size).to.equal(0); + expect(deps.marketplaceApps.size).to.equal(0); + }); + + it('should skip docs that have no schema array', async () => { + const schemas = [ + { uid: 'page' }, // no schema property + { uid: 'blog', schema: [{ uid: 'seo', data_type: 'global_field', reference_to: 'seo_gf' }] }, + ]; + + const deps = await handler.extractDependencies(schemas); + + expect(deps.globalFields.has('seo_gf')).to.be.true; + expect(deps.globalFields.size).to.equal(1); + }); + }); + + describe('extractDependencies β€” disk fallback (content_types/schema.json)', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'qe-deptest-')); + mockConfig.exportDir = tmpDir; + mockConfig.branchName = 'main'; + (mockConfig as any).context = { command: 'test' }; + handler = new ContentTypeDependenciesHandler(mockStackAPIClient, mockConfig); + }); + + afterEach(() => { + if (tmpDir && fs.existsSync(tmpDir)) { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it('loads content types from schema.json when schemas argument is omitted', async () => { + const ctDir = path.join(tmpDir, 'main', 'content_types'); + fs.mkdirSync(ctDir, { recursive: true }); + fs.writeFileSync( + path.join(ctDir, 'schema.json'), + JSON.stringify([ + { + uid: 'page', + schema: [{ uid: 'seo', data_type: 'global_field', reference_to: 'seo_gf' }], + }, + ]), + ); + + const deps = await handler.extractDependencies(); + + expect(deps.globalFields.has('seo_gf')).to.be.true; + }); + + it('returns empty dependency sets when schema.json is missing', async () => { + const ctDir = path.join(tmpDir, 'main', 'content_types'); + fs.mkdirSync(ctDir, { recursive: true }); + + const deps = await handler.extractDependencies(); + + expect(deps.globalFields.size).to.equal(0); + expect(deps.extensions.size).to.equal(0); + expect(deps.taxonomies.size).to.equal(0); + expect(deps.marketplaceApps.size).to.equal(0); + }); + }); +}); diff --git a/packages/contentstack-query-export/test/unit/module-exporter.test.ts b/packages/contentstack-query-export/test/unit/module-exporter.test.ts new file mode 100644 index 000000000..3b7142c0a --- /dev/null +++ b/packages/contentstack-query-export/test/unit/module-exporter.test.ts @@ -0,0 +1,207 @@ +import { expect } from 'chai'; +import * as sinon from 'sinon'; +import { ModuleExporter } from '../../src/core/module-exporter'; +import * as logger from '../../src/utils/logger'; +import ExportCommand from '@contentstack/cli-cm-export'; + +describe('ModuleExporter', () => { + let sandbox: sinon.SinonSandbox; + let moduleExporter: ModuleExporter; + let mockStackAPIClient: any; + let mockConfig: any; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + + // Mock stack API client + mockStackAPIClient = { + contentType: sandbox.stub(), + entry: sandbox.stub(), + asset: sandbox.stub(), + }; + + // Mock export configuration + mockConfig = { + exportDir: './test-export', + stackApiKey: 'test-stack-api-key', + managementToken: 'test-management-token', + branchName: 'main', + securedAssets: false, + externalConfigPath: './config/export-config.json', + }; + + // Stub logger to prevent console output during tests + sandbox.stub(logger, 'log'); + + moduleExporter = new ModuleExporter(mockConfig); + }); + + afterEach(() => { + sandbox.restore(); + }); + + describe('constructor', () => { + it('should initialize ModuleExporter with correct configuration', () => { + expect(moduleExporter).to.be.an('object'); + expect((moduleExporter as any).exportQueryConfig).to.equal(mockConfig); + expect((moduleExporter as any).exportedModules).to.be.an('array').that.is.empty; + }); + }); + + describe('buildExportCommand', () => { + it('should build basic export command with required parameters', () => { + const cmd = (moduleExporter as any).buildExportCommand('entries', {}); + + expect(cmd).to.include('-k', 'test-stack-api-key'); + expect(cmd).to.include('-d', './test-export'); + expect(cmd).to.include('--module', 'entries'); + expect(cmd).to.include('-a', 'test-management-token'); + expect(cmd).to.include('-y'); + }); + + it('should include branch when specified in config', () => { + const cmd = (moduleExporter as any).buildExportCommand('content-types', {}); + + expect(cmd).to.include('--branch', 'main'); + }); + + it('should include branch from options over config', () => { + const cmd = (moduleExporter as any).buildExportCommand('content-types', { + branch: 'development', + }); + + expect(cmd).to.include('--branch', 'development'); + }); + + it('should include query when provided in options', () => { + const query = { + modules: { + entries: { content_type_uid: 'page' }, + }, + }; + + const cmd = (moduleExporter as any).buildExportCommand('entries', { query }); + + expect(cmd).to.include('--query', JSON.stringify(query)); + }); + + it('should include secured assets flag when enabled in config', () => { + mockConfig.securedAssets = true; + moduleExporter = new ModuleExporter(mockConfig); + + const cmd = (moduleExporter as any).buildExportCommand('assets', {}); + + expect(cmd).to.include('--secured-assets'); + }); + + it('should include secured assets from options over config', () => { + const cmd = (moduleExporter as any).buildExportCommand('assets', { + securedAssets: true, + }); + + expect(cmd).to.include('--secured-assets'); + }); + + it('should use alias over management token when provided', () => { + const cmd = (moduleExporter as any).buildExportCommand('environments', { + alias: 'production-stack', + }); + + expect(cmd).to.include('-a', 'production-stack'); + expect(cmd).to.not.include('test-management-token'); + }); + + it('should include external config path when specified', () => { + const cmd = (moduleExporter as any).buildExportCommand('locales', {}); + + expect(cmd).to.include('--config', './config/export-config.json'); + }); + + it('should use custom config path from options', () => { + const cmd = (moduleExporter as any).buildExportCommand('locales', { + configPath: './custom-config.json', + }); + + expect(cmd).to.include('--config', './custom-config.json'); + }); + + it('should use custom directory from options', () => { + const cmd = (moduleExporter as any).buildExportCommand('entries', { + directory: './custom-export', + }); + + expect(cmd).to.include('-d', './custom-export'); + }); + + it('should handle missing optional parameters', () => { + mockConfig.branchName = undefined; + mockConfig.externalConfigPath = undefined; + mockConfig.managementToken = undefined; + moduleExporter = new ModuleExporter(mockConfig); + + const cmd = (moduleExporter as any).buildExportCommand('entries', {}); + + expect(cmd).to.include('-k', 'test-stack-api-key'); + expect(cmd).to.include('-d', './test-export'); + expect(cmd).to.not.include('--branch'); + expect(cmd).to.not.include('--config'); + expect(cmd).to.not.include('-a'); + }); + }); + + describe('exportModule', () => { + let runStub: sinon.SinonStub; + + beforeEach(() => { + // Stub ExportCommand.run to prevent actual exports + runStub = sandbox.stub(ExportCommand, 'run').resolves(); + }); + + it('should export a module with correct parameters', async () => { + await moduleExporter.exportModule('entries'); + + expect(runStub.calledOnce).to.be.true; + const args = runStub.firstCall.args[0]; + expect(args).to.include('--module', 'entries'); + }); + + it('should handle errors during export', async () => { + const error = new Error('Export failed'); + runStub.rejects(error); + + try { + await moduleExporter.exportModule('entries'); + expect.fail('Should have thrown an error'); + } catch (err) { + expect(err).to.equal(error); + } + }); + + it('should apply delay before exporting', async () => { + const clock = sandbox.useFakeTimers(); + const exportPromise = moduleExporter.exportModule('entries'); + + expect(runStub.called).to.be.false; + clock.tick(2000); // Default delay + await exportPromise; + + expect(runStub.calledOnce).to.be.true; + }); + + it('should use custom delay from config', async () => { + mockConfig.exportDelayMs = 5000; + moduleExporter = new ModuleExporter(mockConfig); + + const clock = sandbox.useFakeTimers(); + const exportPromise = moduleExporter.exportModule('entries'); + + clock.tick(2000); // Not enough time + expect(runStub.called).to.be.false; + + clock.tick(3000); // Complete the delay + await exportPromise; + + expect(runStub.calledOnce).to.be.true; + }); + }); +}); diff --git a/packages/contentstack-query-export/test/unit/query-executor.test.ts b/packages/contentstack-query-export/test/unit/query-executor.test.ts new file mode 100644 index 000000000..48dd66b7d --- /dev/null +++ b/packages/contentstack-query-export/test/unit/query-executor.test.ts @@ -0,0 +1,622 @@ +import { expect } from 'chai'; +import * as sinon from 'sinon'; +import { QueryExporter } from '../../src/core/query-executor'; +import { QueryParser } from '../../src/utils/query-parser'; +import { ModuleExporter } from '../../src/core/module-exporter'; +import * as logger from '../../src/utils/logger'; +import { + ReferencedContentTypesHandler, + ContentTypeDependenciesHandler, + AssetReferenceHandler, + fsUtil, +} from '../../src/utils'; +import * as readCtSchemas from '../../src/utils/read-content-type-schemas'; + +describe('QueryExporter', () => { + let sandbox: sinon.SinonSandbox; + let queryExporter: QueryExporter; + let mockManagementClient: any; + let mockConfig: any; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + + // Mock management client + mockManagementClient = { + stack: sandbox.stub().returns({}), + }; + + // Mock export configuration + mockConfig = { + exportDir: './test-export', + stackApiKey: 'test-stack-api-key', + managementToken: 'test-management-token', + query: '{"modules":{"entries":{"content_type_uid":"test_page"}}}', + modules: { + general: ['environments', 'locales'], + queryable: ['entries', 'assets', 'content-types'], + }, + branchName: 'main', + securedAssets: false, + externalConfigPath: './config/export-config.json', + maxCTReferenceDepth: 20, + }; + + // Stub logger to prevent console output during tests + sandbox.stub(logger, 'log'); + + queryExporter = new QueryExporter(mockManagementClient, mockConfig); + }); + + afterEach(() => { + sandbox.restore(); + }); + + describe('constructor', () => { + it('should initialize QueryExporter with correct configuration', () => { + expect(queryExporter).to.be.an('object'); + expect((queryExporter as any).exportQueryConfig).to.equal(mockConfig); + expect((queryExporter as any).queryParser).to.be.an.instanceof(QueryParser); + expect((queryExporter as any).moduleExporter).to.be.an.instanceof(ModuleExporter); + }); + + it('should create QueryParser instance with correct config', () => { + const queryParser = (queryExporter as any).queryParser; + expect(queryParser).to.be.an.instanceof(QueryParser); + }); + + it('should create ModuleExporter instance', () => { + const moduleExporter = (queryExporter as any).moduleExporter; + expect(moduleExporter).to.be.an.instanceof(ModuleExporter); + }); + }); + + describe('execute', () => { + let queryParserStub: sinon.SinonStub; + let exportGeneralModulesStub: sinon.SinonStub; + let exportQueriedModuleStub: sinon.SinonStub; + let expandSchemaClosureStub: sinon.SinonStub; + let exportContentModulesStub: sinon.SinonStub; + + beforeEach(() => { + queryParserStub = sandbox.stub((queryExporter as any).queryParser, 'parse').resolves({ + modules: { entries: { content_type_uid: 'test_page' } }, + }); + exportGeneralModulesStub = sandbox.stub(queryExporter as any, 'exportGeneralModules').resolves(); + exportQueriedModuleStub = sandbox.stub(queryExporter as any, 'exportQueriedModule').resolves(); + expandSchemaClosureStub = sandbox.stub(queryExporter as any, 'expandSchemaClosure').resolves(); + exportContentModulesStub = sandbox.stub(queryExporter as any, 'exportContentModules').resolves(); + }); + + it('should execute the complete export workflow', async () => { + await queryExporter.execute(); + + expect(queryParserStub.calledOnce).to.be.true; + expect(exportGeneralModulesStub.calledOnce).to.be.true; + expect(exportQueriedModuleStub.calledOnce).to.be.true; + expect(expandSchemaClosureStub.calledOnce).to.be.true; + expect(exportContentModulesStub.calledOnce).to.be.true; + }); + + it('should call methods in correct order', async () => { + await queryExporter.execute(); + + sinon.assert.callOrder( + queryParserStub, + exportGeneralModulesStub, + exportQueriedModuleStub, + expandSchemaClosureStub, + exportContentModulesStub, + ); + }); + + it('should pass parsed query to exportQueriedModule', async () => { + const mockParsedQuery = { modules: { entries: { content_type_uid: 'test_page' } } }; + queryParserStub.resolves(mockParsedQuery); + + await queryExporter.execute(); + + expect(exportQueriedModuleStub.calledWith(mockParsedQuery)).to.be.true; + }); + + it('should handle query parsing errors', async () => { + const queryError = new Error('Invalid query format'); + queryParserStub.rejects(queryError); + + try { + await queryExporter.execute(); + expect.fail('Should have thrown error'); + } catch (error) { + expect(error.message).to.equal('Invalid query format'); + } + + expect(exportGeneralModulesStub.called).to.be.false; + }); + + it('should handle export errors and propagate them', async () => { + const exportError = new Error('Export failed'); + exportGeneralModulesStub.rejects(exportError); + + try { + await queryExporter.execute(); + expect.fail('Should have thrown error'); + } catch (error) { + expect(error.message).to.equal('Export failed'); + } + }); + }); + + describe('exportGeneralModules', () => { + let moduleExporterStub: sinon.SinonStub; + + beforeEach(() => { + moduleExporterStub = sandbox.stub((queryExporter as any).moduleExporter, 'exportModule').resolves(); + }); + + it('should export all general modules', async () => { + await (queryExporter as any).exportGeneralModules(); + + expect(moduleExporterStub.callCount).to.equal(2); + expect(moduleExporterStub.calledWith('environments')).to.be.true; + expect(moduleExporterStub.calledWith('locales')).to.be.true; + }); + + it('should handle empty general modules array', async () => { + mockConfig.modules.general = []; + queryExporter = new QueryExporter(mockManagementClient, mockConfig); + moduleExporterStub = sandbox.stub((queryExporter as any).moduleExporter, 'exportModule').resolves(); + + await (queryExporter as any).exportGeneralModules(); + + expect(moduleExporterStub.called).to.be.false; + }); + + it('should handle module export errors', async () => { + const moduleError = new Error('Module export failed'); + moduleExporterStub.rejects(moduleError); + + try { + await (queryExporter as any).exportGeneralModules(); + expect.fail('Should have thrown error'); + } catch (error) { + expect(error.message).to.equal('Module export failed'); + } + }); + }); + + describe('exportQueriedModule', () => { + let moduleExporterStub: sinon.SinonStub; + + beforeEach(() => { + moduleExporterStub = sandbox.stub((queryExporter as any).moduleExporter, 'exportModule').resolves(); + }); + + it('should export queryable modules with query', async () => { + const parsedQuery = { + modules: { + entries: { content_type_uid: 'test_page' }, + assets: { tags: 'featured' }, + }, + }; + + await (queryExporter as any).exportQueriedModule(parsedQuery); + + expect(moduleExporterStub.callCount).to.equal(2); + expect(moduleExporterStub.calledWith('entries', { query: parsedQuery })).to.be.true; + expect(moduleExporterStub.calledWith('assets', { query: parsedQuery })).to.be.true; + }); + + it('should skip non-queryable modules', async () => { + mockConfig.modules.queryable = ['entries']; // Remove assets from queryable + queryExporter = new QueryExporter(mockManagementClient, mockConfig); + moduleExporterStub = sandbox.stub((queryExporter as any).moduleExporter, 'exportModule').resolves(); + + const parsedQuery = { + modules: { + entries: { content_type_uid: 'test_page' }, + environments: { name: 'production' }, // Not queryable + }, + }; + + await (queryExporter as any).exportQueriedModule(parsedQuery); + + expect(moduleExporterStub.callCount).to.equal(1); + expect(moduleExporterStub.calledWith('entries', { query: parsedQuery })).to.be.true; + }); + + it('should handle empty modules in query', async () => { + const parsedQuery = { modules: {} }; + + await (queryExporter as any).exportQueriedModule(parsedQuery); + + expect(moduleExporterStub.called).to.be.false; + }); + }); + + describe('expandSchemaClosure', () => { + let moduleExporterStub: sinon.SinonStub; + let readContentTypesFromExportDirStub: sinon.SinonStub; + let readGlobalFieldSchemasFromDirStub: sinon.SinonStub; + let referencedHandlerStub: any; + let dependenciesHandlerStub: any; + + const mockCTs = [{ uid: 'page', title: 'Page', schema: [] as any[] }]; + const emptyDeps = { + globalFields: new Set(), + extensions: new Set(), + taxonomies: new Set(), + marketplaceApps: new Set(), + }; + + beforeEach(() => { + moduleExporterStub = sandbox.stub((queryExporter as any).moduleExporter, 'exportModule').resolves(); + + // Default: CTs from schema.json reader, GFs empty. + readContentTypesFromExportDirStub = sandbox.stub(readCtSchemas, 'readContentTypesFromExportDir').returns(mockCTs); + readGlobalFieldSchemasFromDirStub = sandbox.stub(readCtSchemas, 'readGlobalFieldSchemasFromDir').returns([]); + + referencedHandlerStub = { extractReferencedContentTypes: sandbox.stub().resolves([]) }; + sandbox + .stub(ReferencedContentTypesHandler.prototype, 'extractReferencedContentTypes') + .callsFake(referencedHandlerStub.extractReferencedContentTypes); + + dependenciesHandlerStub = { extractDependencies: sandbox.stub().resolves(emptyDeps) }; + sandbox + .stub(ContentTypeDependenciesHandler.prototype, 'extractDependencies') + .callsFake(dependenciesHandlerStub.extractDependencies); + }); + + it('should export personalize exactly once when no new items are found', async () => { + await (queryExporter as any).expandSchemaClosure(); + + const personalizeCalls = moduleExporterStub.getCalls().filter((c) => c.args[0] === 'personalize'); + expect(personalizeCalls).to.have.lengthOf(1); + // No CT or GF export should have happened + expect(moduleExporterStub.getCalls().filter((c) => c.args[0] === 'content-types')).to.have.lengthOf(0); + expect(moduleExporterStub.getCalls().filter((c) => c.args[0] === 'global-fields')).to.have.lengthOf(0); + }); + + it('should pass combined CT and GF schemas to extractReferencedContentTypes', async () => { + const mockGFs = [{ uid: 'seo_gf', schema: [] as any[] }]; + readGlobalFieldSchemasFromDirStub.returns(mockGFs); + + await (queryExporter as any).expandSchemaClosure(); + + const callArgs = referencedHandlerStub.extractReferencedContentTypes.getCall(0).args[0]; + expect(callArgs).to.deep.include({ uid: 'page', title: 'Page', schema: [] as any[] }); + expect(callArgs).to.deep.include({ uid: 'seo_gf', schema: [] as any[] }); + }); + + it('should pass combined CT and GF schemas to extractDependencies', async () => { + const mockGFs = [{ uid: 'seo_gf', schema: [] as any[] }]; + readGlobalFieldSchemasFromDirStub.returns(mockGFs); + + await (queryExporter as any).expandSchemaClosure(); + + const callArgs = dependenciesHandlerStub.extractDependencies.getCall(0).args[0]; + expect(callArgs).to.deep.include({ uid: 'page', title: 'Page', schema: [] as any[] }); + expect(callArgs).to.deep.include({ uid: 'seo_gf', schema: [] as any[] }); + }); + + it('should export new referenced content types found in CT schemas', async () => { + referencedHandlerStub.extractReferencedContentTypes + .onFirstCall() + .resolves(['new_ct']) + .resolves([]); + + await (queryExporter as any).expandSchemaClosure(); + + const ctCall = moduleExporterStub.getCalls().find((c) => c.args[0] === 'content-types'); + expect(ctCall).to.exist; + expect(ctCall!.args[1].query.modules['content-types'].uid.$in).to.deep.equal(['new_ct']); + }); + + it('should export new global fields discovered from CT schemas', async () => { + dependenciesHandlerStub.extractDependencies + .onFirstCall() + .resolves({ globalFields: new Set(['gf_a']), extensions: new Set(), taxonomies: new Set(), marketplaceApps: new Set() }) + .resolves(emptyDeps); + + await (queryExporter as any).expandSchemaClosure(); + + const gfCall = moduleExporterStub.getCalls().find((c) => c.args[0] === 'global-fields'); + expect(gfCall).to.exist; + expect(gfCall!.args[1].query.modules['global-fields'].uid.$in).to.deep.equal(['gf_a']); + }); + + it('should iterate to find CT references inside global field schemas', async () => { + // Iter 1: GF A is newly discovered from CT deps. GF A is not yet on disk. + // Iter 2: GF A is now on disk; its schema exposes a reference to CT B. + const gfADoc = [{ uid: 'gf_a', schema: [] as any[] }]; + + readGlobalFieldSchemasFromDirStub.callsFake(() => + dependenciesHandlerStub.extractDependencies.callCount > 0 ? gfADoc : [], + ); + readContentTypesFromExportDirStub.returns(mockCTs); + + dependenciesHandlerStub.extractDependencies + .onFirstCall() + .resolves({ globalFields: new Set(['gf_a']), extensions: new Set(), taxonomies: new Set(), marketplaceApps: new Set() }) + .resolves(emptyDeps); + + referencedHandlerStub.extractReferencedContentTypes + .onFirstCall().resolves([]) // iter 1: only CTs on disk, no CT refs + .onSecondCall().resolves(['ct_b']) // iter 2: GF A adds a CT ref to ct_b + .resolves([]); + + await (queryExporter as any).expandSchemaClosure(); + + const ctCall = moduleExporterStub.getCalls().find((c) => c.args[0] === 'content-types'); + expect(ctCall).to.exist; + expect(ctCall!.args[1].query.modules['content-types'].uid.$in).to.include('ct_b'); + + const gfCall = moduleExporterStub.getCalls().find((c) => c.args[0] === 'global-fields'); + expect(gfCall).to.exist; + expect(gfCall!.args[1].query.modules['global-fields'].uid.$in).to.include('gf_a'); + }); + + it('should not re-export already exported global fields across iterations', async () => { + // gf_a is returned by extractDependencies on every call, but should only be exported once. + dependenciesHandlerStub.extractDependencies.resolves({ + globalFields: new Set(['gf_a']), + extensions: new Set(), + taxonomies: new Set(), + marketplaceApps: new Set(), + }); + + // Trigger a second iteration via a new CT reference so we can verify gf_a is not re-exported. + referencedHandlerStub.extractReferencedContentTypes + .onFirstCall().resolves(['new_ct']) + .resolves([]); + + await (queryExporter as any).expandSchemaClosure(); + + const gfCalls = moduleExporterStub.getCalls().filter((c) => c.args[0] === 'global-fields'); + expect(gfCalls).to.have.lengthOf(1); + }); + + it('should not re-export already exported content types across iterations', async () => { + // new_ct returned on first AND second call β€” should only be exported once. + referencedHandlerStub.extractReferencedContentTypes + .onFirstCall().resolves(['new_ct']) + .onSecondCall().resolves(['new_ct']) // already exported β€” should be filtered + .resolves([]); + + // Trigger a second iteration via a new GF dep. + dependenciesHandlerStub.extractDependencies + .onFirstCall().resolves({ globalFields: new Set(['gf_a']), extensions: new Set(), taxonomies: new Set(), marketplaceApps: new Set() }) + .resolves(emptyDeps); + + await (queryExporter as any).expandSchemaClosure(); + + const ctCalls = moduleExporterStub.getCalls().filter((c) => c.args[0] === 'content-types'); + expect(ctCalls).to.have.lengthOf(1); + }); + + it('should export extensions, taxonomies, and marketplace apps as leaf deps', async () => { + dependenciesHandlerStub.extractDependencies.resolves({ + globalFields: new Set(), + extensions: new Set(['ext_1']), + taxonomies: new Set(['tax_1']), + marketplaceApps: new Set(['mp_app_1']), + }); + + await (queryExporter as any).expandSchemaClosure(); + + expect(moduleExporterStub.getCalls().some((c) => c.args[0] === 'extensions')).to.be.true; + expect(moduleExporterStub.getCalls().some((c) => c.args[0] === 'taxonomies')).to.be.true; + expect(moduleExporterStub.getCalls().some((c) => c.args[0] === 'marketplace-apps')).to.be.true; + }); + + it('should skip CT reference extraction when skipReferences is true', async () => { + mockConfig.skipReferences = true; + const localExporter = new QueryExporter(mockManagementClient, mockConfig); + const localModuleStub = sandbox.stub((localExporter as any).moduleExporter, 'exportModule').resolves(); + + await (localExporter as any).expandSchemaClosure(); + + expect(referencedHandlerStub.extractReferencedContentTypes.called).to.be.false; + expect(localModuleStub.getCalls().filter((c) => c.args[0] === 'content-types')).to.have.lengthOf(0); + }); + + it('should skip dependency extraction when skipDependencies is true', async () => { + mockConfig.skipDependencies = true; + const localExporter = new QueryExporter(mockManagementClient, mockConfig); + const localModuleStub = sandbox.stub((localExporter as any).moduleExporter, 'exportModule').resolves(); + + await (localExporter as any).expandSchemaClosure(); + + expect(dependenciesHandlerStub.extractDependencies.called).to.be.false; + expect(localModuleStub.getCalls().filter((c) => c.args[0] === 'global-fields')).to.have.lengthOf(0); + }); + + it('should stop after maxCTReferenceDepth iterations', async () => { + mockConfig.maxCTReferenceDepth = 2; + const localExporter = new QueryExporter(mockManagementClient, mockConfig); + sandbox.stub((localExporter as any).moduleExporter, 'exportModule').resolves(); + + // Always report new GFs so the loop never naturally terminates. + let callN = 0; + dependenciesHandlerStub.extractDependencies.callsFake(() => { + callN++; + return Promise.resolve({ + globalFields: new Set([`gf_${callN}`]), + extensions: new Set(), + taxonomies: new Set(), + marketplaceApps: new Set(), + }); + }); + + await (localExporter as any).expandSchemaClosure(); + + expect(dependenciesHandlerStub.extractDependencies.callCount).to.be.at.most(2); + }); + + it('should propagate errors from extractReferencedContentTypes', async () => { + referencedHandlerStub.extractReferencedContentTypes.rejects(new Error('Handler failed')); + + try { + await (queryExporter as any).expandSchemaClosure(); + expect.fail('Should have thrown error'); + } catch (error: any) { + expect(error.message).to.equal('Handler failed'); + } + }); + + it('should propagate errors from extractDependencies', async () => { + dependenciesHandlerStub.extractDependencies.rejects(new Error('Dependencies extraction failed')); + + try { + await (queryExporter as any).expandSchemaClosure(); + expect.fail('Should have thrown error'); + } catch (error: any) { + expect(error.message).to.equal('Dependencies extraction failed'); + } + }); + }); + + describe('exportContentModules', () => { + let exportEntriesStub: sinon.SinonStub; + let exportReferencedAssetsStub: sinon.SinonStub; + let setTimeoutStub: sinon.SinonStub; + + beforeEach(() => { + exportEntriesStub = sandbox.stub(queryExporter as any, 'exportEntries').resolves(); + exportReferencedAssetsStub = sandbox.stub(queryExporter as any, 'exportReferencedAssets').resolves(); + + // Mock setTimeout to avoid actual delays in tests + setTimeoutStub = sandbox.stub(global, 'setTimeout').callsFake((callback) => { + callback(); + return {} as any; + }); + }); + + it('should export entries and then assets', async () => { + await (queryExporter as any).exportContentModules(); + + expect(exportEntriesStub.calledOnce).to.be.true; + expect(exportReferencedAssetsStub.calledOnce).to.be.true; + sinon.assert.callOrder(exportEntriesStub, exportReferencedAssetsStub); + }); + + it('should include delay before asset export', async () => { + await (queryExporter as any).exportContentModules(); + + expect(setTimeoutStub.calledOnce).to.be.true; + expect(setTimeoutStub.calledWith(sinon.match.func, 5000)).to.be.true; + }); + + it('should handle entries export errors', async () => { + const entriesError = new Error('Entries export failed'); + exportEntriesStub.rejects(entriesError); + + try { + await (queryExporter as any).exportContentModules(); + expect.fail('Should have thrown error'); + } catch (error) { + expect(error.message).to.equal('Entries export failed'); + } + + expect(exportReferencedAssetsStub.called).to.be.false; + }); + + it('should handle assets export errors', async () => { + const assetsError = new Error('Assets export failed'); + exportReferencedAssetsStub.rejects(assetsError); + + try { + await (queryExporter as any).exportContentModules(); + expect.fail('Should have thrown error'); + } catch (error) { + expect(error.message).to.equal('Assets export failed'); + } + }); + }); + + describe('exportEntries', () => { + let moduleExporterStub: sinon.SinonStub; + + beforeEach(() => { + moduleExporterStub = sandbox.stub((queryExporter as any).moduleExporter, 'exportModule').resolves(); + }); + + it('should export entries module', async () => { + await (queryExporter as any).exportEntries(); + + expect(moduleExporterStub.calledOnce).to.be.true; + expect(moduleExporterStub.calledWith('entries')).to.be.true; + }); + + it('should handle entries export errors', async () => { + const entriesError = new Error('Entries export failed'); + moduleExporterStub.rejects(entriesError); + + try { + await (queryExporter as any).exportEntries(); + expect.fail('Should have thrown error'); + } catch (error) { + expect(error.message).to.equal('Entries export failed'); + } + }); + }); + + describe('exportReferencedAssets', () => { + let moduleExporterStub: sinon.SinonStub; + let assetHandlerStub: any; + + beforeEach(() => { + moduleExporterStub = sandbox.stub((queryExporter as any).moduleExporter, 'exportModule').resolves(); + + // Mock AssetReferenceHandler + assetHandlerStub = { + extractReferencedAssets: sandbox.stub().returns(['asset_1', 'asset_2', 'asset_3']), + }; + sandbox + .stub(AssetReferenceHandler.prototype, 'extractReferencedAssets') + .callsFake(assetHandlerStub.extractReferencedAssets); + }); + + it('should export referenced assets when found', async () => { + await (queryExporter as any).exportReferencedAssets(); + + expect(moduleExporterStub.calledOnce).to.be.true; + const exportCall = moduleExporterStub.getCall(0); + expect(exportCall.args[0]).to.equal('assets'); + expect(exportCall.args[1].query.modules.assets.uid.$in).to.deep.equal(['asset_1', 'asset_2', 'asset_3']); + }); + + it('should skip export when no assets found', async () => { + assetHandlerStub.extractReferencedAssets.returns([]); + + await (queryExporter as any).exportReferencedAssets(); + + expect(moduleExporterStub.called).to.be.false; + }); + + it('should handle asset extraction errors', async () => { + const assetError = new Error('Asset extraction failed'); + assetHandlerStub.extractReferencedAssets.throws(assetError); + + try { + await (queryExporter as any).exportReferencedAssets(); + expect.fail('Should have thrown error'); + } catch (error) { + expect(error.message).to.equal('Asset extraction failed'); + } + + expect(moduleExporterStub.called).to.be.false; + }); + + it('should handle asset export errors', async () => { + const exportError = new Error('Asset export failed'); + moduleExporterStub.rejects(exportError); + + try { + await (queryExporter as any).exportReferencedAssets(); + expect.fail('Should have thrown error'); + } catch (error) { + expect(error.message).to.equal('Asset export failed'); + } + }); + }); +}); diff --git a/packages/contentstack-query-export/test/unit/query-parser-simple.test.ts b/packages/contentstack-query-export/test/unit/query-parser-simple.test.ts new file mode 100644 index 000000000..297259bb5 --- /dev/null +++ b/packages/contentstack-query-export/test/unit/query-parser-simple.test.ts @@ -0,0 +1,156 @@ +import { expect } from 'chai'; +import { CLIError } from '@contentstack/cli-utilities'; +import { QueryParser } from '../../src/utils/query-parser'; +import { QueryExportConfig } from '../../src/types'; + +describe('Query Parser Simple Tests', () => { + let queryParser: QueryParser; + let mockConfig: QueryExportConfig; + + beforeEach(() => { + mockConfig = { + maxCTReferenceDepth: 20, + contentVersion: 2, + host: 'https://api.contentstack.io/v3', + exportDir: '/test/export', + stackApiKey: 'test-api-key', + managementToken: 'test-token', + query: '', + skipReferences: false, + skipDependencies: false, + branchName: 'main', + securedAssets: false, + isQueryBasedExport: true, + logsPath: '/test/logs', + dataPath: '/test/data', + modules: { + general: ['stack', 'locales', 'environments'], + queryable: ['content-types'], + dependent: ['global-fields', 'extensions', 'taxonomies'], + content: ['entries', 'assets'], + exportOrder: ['stack', 'content-types'], + }, + queryConfig: { + maxRecursionDepth: 10, + batchSize: 100, + metadataFileName: '_query-meta.json', + validation: { + maxQueryDepth: 5, + maxArraySize: 1000, + allowedDateFormats: ['ISO8601'], + }, + }, + fetchConcurrency: 5, + writeConcurrency: 5, + apis: { + stacks: '/stacks/', + locales: '/locales/', + environments: '/environments/', + content_types: '/content_types/', + global_fields: '/global_fields/', + extensions: '/extensions/', + taxonomies: '/taxonomies/', + entries: '/entries/', + assets: '/assets/', + }, + }; + + queryParser = new QueryParser(mockConfig); + }); + + describe('JSON string parsing and validation', () => { + it('should parse and validate a simple valid query', async () => { + const queryString = '{"modules": {"content-types": {"title": {"$exists": true}}}}'; + + const result = await queryParser.parse(queryString); + + expect(result).to.be.an('object'); + expect(result.modules).to.have.property('content-types'); + expect(result.modules['content-types']).to.deep.equal({ + title: { $exists: true }, + }); + }); + + it('should validate and reject queries without modules', async () => { + const queryString = '{"title": {"$exists": true}}'; + + try { + await queryParser.parse(queryString); + expect.fail('Expected validation error'); + } catch (error) { + expect(error).to.be.instanceOf(CLIError); + expect(error.message).to.equal('The query must contain a "modules" object.'); + } + }); + + it('should validate and reject queries with empty modules', async () => { + const queryString = '{"modules": {}}'; + + try { + await queryParser.parse(queryString); + expect.fail('Expected validation error'); + } catch (error) { + expect(error).to.be.instanceOf(CLIError); + expect(error.message).to.equal('The query must contain at least one module.'); + } + }); + + it('should validate and reject queries with non-queryable modules', async () => { + const queryString = '{"modules": {"invalid-module": {"title": {"$exists": true}}}}'; + + try { + await queryParser.parse(queryString); + expect.fail('Expected validation error'); + } catch (error) { + expect(error).to.be.instanceOf(CLIError); + expect(error.message).to.include('Module "invalid-module" is not queryable'); + } + }); + + it('should handle invalid JSON gracefully', async () => { + const invalidQuery = '{"modules": invalid json}'; + + try { + await queryParser.parse(invalidQuery); + expect.fail('Expected JSON parse error'); + } catch (error) { + expect(error).to.be.instanceOf(CLIError); + expect(error.message).to.include('Invalid JSON query'); + } + }); + + it('should handle complex valid queries', async () => { + const complexQuery = { + modules: { + 'content-types': { + $and: [{ title: { $exists: true } }, { updated_at: { $gte: '2024-01-01' } }], + }, + }, + }; + + const result = await queryParser.parse(JSON.stringify(complexQuery)); + + expect(result).to.deep.equal(complexQuery); + }); + + it('should reject null queries', async () => { + try { + await queryParser.parse('null'); + expect.fail('Expected validation error'); + } catch (error) { + expect(error).to.be.instanceOf(CLIError); + expect(error.message).to.equal('The query must be a valid JSON object.'); + } + }); + + it('should reject string queries', async () => { + try { + await queryParser.parse('"string query"'); + expect.fail('Expected validation error'); + } catch (error) { + expect(error).to.be.instanceOf(CLIError); + expect(error.message).to.equal('The query must be a valid JSON object.'); + } + }); + }); +}); diff --git a/packages/contentstack-query-export/test/unit/read-content-type-schemas.test.ts b/packages/contentstack-query-export/test/unit/read-content-type-schemas.test.ts new file mode 100644 index 000000000..e11219f99 --- /dev/null +++ b/packages/contentstack-query-export/test/unit/read-content-type-schemas.test.ts @@ -0,0 +1,52 @@ +import { expect } from 'chai'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { rebuildContentTypesSchemaJson, readContentTypesFromExportDir } from '../../src/utils/read-content-type-schemas'; + +describe('read-content-type-schemas', () => { + describe('rebuildContentTypesSchemaJson', () => { + let tmp: string; + + beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'qe-ctschema-')); + }); + + afterEach(() => { + if (tmp && fs.existsSync(tmp)) fs.rmSync(tmp, { recursive: true, force: true }); + }); + + it('merges all per-UID JSON files into schema.json', () => { + fs.writeFileSync(path.join(tmp, 'a.json'), JSON.stringify({ uid: 'a', title: 'A', schema: [] })); + fs.writeFileSync(path.join(tmp, 'b.json'), JSON.stringify({ uid: 'b', title: 'B', schema: [] })); + fs.writeFileSync(path.join(tmp, 'schema.json'), JSON.stringify([{ uid: 'stale', schema: [] }])); + + rebuildContentTypesSchemaJson(tmp); + + const merged = readContentTypesFromExportDir(tmp); + const uids = merged.map((x: { uid: string }) => x.uid).sort(); + expect(uids).to.deep.equal(['a', 'b']); + }); + + it('dedupes by uid when the same uid appears twice', () => { + fs.writeFileSync(path.join(tmp, 'x.json'), JSON.stringify({ uid: 'same', title: 'first', schema: [] })); + fs.writeFileSync(path.join(tmp, 'y.json'), JSON.stringify({ uid: 'same', title: 'second', schema: [] })); + + rebuildContentTypesSchemaJson(tmp); + + const merged = readContentTypesFromExportDir(tmp); + expect(merged).to.have.lengthOf(1); + expect((merged[0] as { title: string }).title).to.equal('second'); + }); + + it('ignores non-object or missing uid files', () => { + fs.writeFileSync(path.join(tmp, 'bad.json'), JSON.stringify([1, 2])); + fs.writeFileSync(path.join(tmp, 'good.json'), JSON.stringify({ uid: 'good', schema: [] })); + + rebuildContentTypesSchemaJson(tmp); + + const merged = readContentTypesFromExportDir(tmp); + expect(merged.map((x: { uid: string }) => x.uid)).to.deep.equal(['good']); + }); + }); +}); diff --git a/packages/contentstack-query-export/test/unit/referenced-asset-handler.test.ts b/packages/contentstack-query-export/test/unit/referenced-asset-handler.test.ts new file mode 100644 index 000000000..9c735ac0c --- /dev/null +++ b/packages/contentstack-query-export/test/unit/referenced-asset-handler.test.ts @@ -0,0 +1,280 @@ +import { expect } from 'chai'; +import { AssetReferenceHandler } from '../../src/utils/referenced-asset-handler'; +import { QueryExportConfig } from '../../src/types'; + +describe('Referenced Asset Handler Utilities', () => { + let handler: AssetReferenceHandler; + let mockConfig: QueryExportConfig; + + beforeEach(() => { + mockConfig = { + maxCTReferenceDepth: 20, + contentVersion: 2, + host: 'https://api.contentstack.io/v3', + exportDir: '/test/export', + stackApiKey: 'test-api-key', + managementToken: 'test-token', + query: '', + skipReferences: false, + skipDependencies: false, + branchName: 'main', + securedAssets: false, + isQueryBasedExport: true, + logsPath: '/test/logs', + dataPath: '/test/data', + modules: { + general: ['stack', 'locales', 'environments'], + queryable: ['content-types'], + dependent: ['global-fields', 'extensions', 'taxonomies'], + content: ['entries', 'assets'], + exportOrder: ['stack', 'content-types'], + }, + queryConfig: { + maxRecursionDepth: 10, + batchSize: 100, + metadataFileName: '_query-meta.json', + validation: { + maxQueryDepth: 5, + maxArraySize: 1000, + allowedDateFormats: ['ISO8601'], + }, + }, + fetchConcurrency: 5, + writeConcurrency: 5, + apis: { + stacks: '/stacks/', + locales: '/locales/', + environments: '/environments/', + content_types: '/content_types/', + global_fields: '/global_fields/', + extensions: '/extensions/', + taxonomies: '/taxonomies/', + entries: '/entries/', + assets: '/assets/', + }, + }; + + handler = new AssetReferenceHandler(mockConfig); + }); + + describe('Asset UID extraction from content strings', () => { + it('should extract asset UIDs from HTML img tags', () => { + // Simulate JSON.stringify() content as it would appear in real usage + // Note: The regex expects asset_uid to be the first attribute after Some content with images:

+ Image 1 + Image 2 +

More content

+ + `; + const content = JSON.stringify({ field: htmlContent }); + + const result = (handler as any).extractAssetUIDsFromString(content); + + expect(result).to.include('asset123'); + expect(result).to.include('asset456'); + expect(result).to.include('asset789'); + expect(result.length).to.equal(3); + }); + + it('should extract asset UIDs from Contentstack asset URLs', () => { + const content = ` + Check out this asset: "https://images.contentstack.io/v3/assets/stack123/asset456/version789/filename.jpg" + And this one: "https://eu-images.contentstack.io/v3/assets/stack456/asset123/version456/image.png" + Also: "https://assets.contentstack.com/v3/assets/stackabc/assetdef/versionghi/file.pdf" + `; + + const result = (handler as any).extractAssetUIDsFromString(content); + + expect(result).to.include('asset456'); + expect(result).to.include('asset123'); + expect(result).to.include('assetdef'); + expect(result.length).to.equal(3); + }); + + it('should handle mixed asset references in content', () => { + const htmlContent = ` +
+ +

Link to: "https://images.contentstack.io/v3/assets/mystack/url_asset_456/v1/document.pdf"

+ +
+ `; + const content = JSON.stringify({ field: htmlContent }); + + const result = (handler as any).extractAssetUIDsFromString(content); + + expect(result).to.include('img_asset_123'); + expect(result).to.include('url_asset_456'); + expect(result).to.include('img_asset_789'); + expect(result.length).to.equal(3); + }); + + it('should handle Azure region URLs', () => { + const content = ` + "https://azure-na-images.contentstack.io/v3/assets/stack123/azure_asset_123/v1/file.jpg" + "https://azure-eu-images.contentstack.io/v3/assets/stack456/azure_asset_456/v2/document.pdf" + `; + + const result = (handler as any).extractAssetUIDsFromString(content); + + expect(result).to.include('azure_asset_123'); + expect(result).to.include('azure_asset_456'); + expect(result.length).to.equal(2); + }); + + it('should handle GCP region URLs', () => { + const content = ` + "https://gcp-na-images.contentstack.io/v3/assets/stack123/gcp_asset_123/v1/file.jpg" + "https://gcp-eu-images.contentstack.io/v3/assets/stack456/gcp_asset_456/v2/document.pdf" + `; + + const result = (handler as any).extractAssetUIDsFromString(content); + + expect(result).to.include('gcp_asset_123'); + expect(result).to.include('gcp_asset_456'); + expect(result.length).to.equal(2); + }); + + it('should return empty array for content without assets', () => { + const content = ` +
+

Title

+

Just some text content without any asset references.

+ External link +
+ `; + + const result = (handler as any).extractAssetUIDsFromString(content); + + expect(result).to.be.an('array'); + expect(result.length).to.equal(0); + }); + + it('should handle malformed asset references gracefully', () => { + const content = ` + + + "https://images.contentstack.io/v3/assets/" + "https://images.contentstack.io/v3/assets/stack123/" + `; + + const result = (handler as any).extractAssetUIDsFromString(content); + + // Should not include empty or malformed UIDs + expect(result).to.be.an('array'); + expect(result.length).to.equal(0); + }); + + it('should deduplicate asset UIDs from same content', () => { + const htmlContent = ` + + + "https://images.contentstack.io/v3/assets/stack123/duplicate_asset/v1/file.jpg" + + `; + const content = JSON.stringify({ field: htmlContent }); + + const result = (handler as any).extractAssetUIDsFromString(content); + + expect(result).to.include('duplicate_asset'); + expect(result).to.include('unique_asset'); + expect(result.length).to.equal(2); + + // Check that duplicate_asset appears only once + const duplicateCount = result.filter((uid: any) => uid === 'duplicate_asset').length; + expect(duplicateCount).to.equal(1); + }); + + it('should handle escaped quotes in HTML', () => { + const content = ``; + + const result = (handler as any).extractAssetUIDsFromString(content); + + expect(result).to.include('escaped_asset_123'); + expect(result.length).to.equal(1); + }); + + it('should handle JSON-stringified content with asset references', () => { + const jsonContent = JSON.stringify({ + content: '', + url: 'https://images.contentstack.io/v3/assets/stack123/json_asset_456/v1/file.jpg', + }); + + const result = (handler as any).extractAssetUIDsFromString(jsonContent); + + expect(result).to.include('json_asset_123'); + expect(result).to.include('json_asset_456'); + expect(result.length).to.equal(2); + }); + + it('should handle content with special characters in asset UIDs', () => { + const htmlContent = ` + + + + `; + const content = JSON.stringify({ field: htmlContent }); + + const result = (handler as any).extractAssetUIDsFromString(content); + + expect(result).to.include('asset-with-dashes-123'); + expect(result).to.include('asset_with_underscores_456'); + expect(result).to.include('asset123ABC'); + expect(result.length).to.equal(3); + }); + + it('should handle large content strings efficiently', () => { + // Create a large content string with asset references + const assetReferences: string[] = []; + let htmlContent = '
'; + + for (let i = 0; i < 100; i++) { + const assetUID = `asset_${i}`; + assetReferences.push(assetUID); + htmlContent += ``; + } + htmlContent += '
'; + + const content = JSON.stringify({ field: htmlContent }); + + const result = (handler as any).extractAssetUIDsFromString(content); + + expect(result.length).to.equal(100); + assetReferences.forEach((uid) => { + expect(result).to.include(uid); + }); + }); + + it('should handle contentstack.com domain URLs', () => { + const content = ` + "https://assets.contentstack.com/v3/assets/stack123/com_asset_123/v1/file.jpg" + "https://images.contentstack.com/v3/assets/stack456/com_asset_456/v2/image.png" + `; + + const result = (handler as any).extractAssetUIDsFromString(content); + + expect(result).to.include('com_asset_123'); + expect(result).to.include('com_asset_456'); + expect(result.length).to.equal(2); + }); + }); + + describe('Constructor and initialization', () => { + it('should initialize with correct export directory path', () => { + expect(handler).to.be.instanceOf(AssetReferenceHandler); + + // Check that entriesDir is set correctly + const entriesDir = (handler as any).entriesDir; + expect(entriesDir).to.include('/test/export'); + expect(entriesDir).to.include('entries'); + }); + + it('should store export configuration', () => { + const config = (handler as any).exportQueryConfig; + expect(config).to.equal(mockConfig); + expect(config.exportDir).to.equal('/test/export'); + }); + }); +}); diff --git a/packages/contentstack-query-export/test/unit/test-cases-summary.txt b/packages/contentstack-query-export/test/unit/test-cases-summary.txt new file mode 100644 index 000000000..a6b13def1 --- /dev/null +++ b/packages/contentstack-query-export/test/unit/test-cases-summary.txt @@ -0,0 +1,282 @@ +CLI QUERY EXPORT - UNIT TEST CASES SUMMARY +=========================================== +Total: 131 Test Cases Across 8 Test Files +Coverage: 69.33% Overall | Core: 89.75% | Utils: 55.85% + +═══════════════════════════════════════════════════════════════════ + +πŸ“ MODULE-EXPORTER.TEST.TS (32 Test Cases) +═══════════════════════════════════════════════════════════════════ + +πŸ”§ Constructor (2 tests) +- βœ… should initialize ModuleExporter with correct configuration +- βœ… should initialize empty exported modules array + +πŸ› οΈ buildExportCommand (14 tests) +- βœ… should build basic export command with required parameters +- βœ… should include branch when specified in config +- βœ… should include branch from options over config +- βœ… should include query when provided in options +- βœ… should include secured assets flag when enabled in config +- βœ… should include secured assets from options over config +- βœ… should use alias over management token when provided +- βœ… should include external config path when specified +- βœ… should use custom config path from options +- βœ… should use custom directory from options +- βœ… should handle missing optional parameters +- βœ… should build different commands for different modules +- βœ… should handle complex query structures + +πŸ“€ exportModule (7 tests) +- βœ… should export module successfully +- βœ… should pass correct command to ExportCommand.run +- βœ… should track exported modules without duplicates +- βœ… should handle export command errors +- βœ… should export with query options +- βœ… should export with all options +- βœ… should handle different module types + +πŸ“– readExportedData (2 tests) +- βœ… should handle file reading logic (private method testing) +- βœ… should handle JSON parsing scenarios + +πŸ“‹ getExportedModules (3 tests) +- βœ… should return empty array initially +- βœ… should return copy of exported modules array +- βœ… should reflect modules added through exportModule + +❌ error handling (3 tests) +- βœ… should handle export command initialization errors +- βœ… should handle malformed configuration gracefully +- βœ… should handle missing stack API client gracefully + +πŸ”„ integration scenarios (3 tests) +- βœ… should handle sequential module exports +- βœ… should handle concurrent module exports +- βœ… should handle mixed success and failure scenarios + +═══════════════════════════════════════════════════════════════════ + +πŸ“ QUERY-EXECUTOR.TEST.TS (35 Test Cases) +═══════════════════════════════════════════════════════════════════ + +πŸ”§ constructor (3 tests) +- βœ… should initialize QueryExporter with correct configuration +- βœ… should create QueryParser instance with correct config +- βœ… should create ModuleExporter instance + +▢️ execute (5 tests) +- βœ… should execute the complete export workflow +- βœ… should call methods in correct order +- βœ… should pass parsed query to exportQueriedModule +- βœ… should handle query parsing errors +- βœ… should handle export errors and propagate them + +🌐 exportGeneralModules (3 tests) +- βœ… should export all general modules +- βœ… should handle empty general modules array +- βœ… should handle module export errors + +πŸ” exportQueriedModule (3 tests) +- βœ… should export queryable modules with query +- βœ… should skip non-queryable modules +- βœ… should handle empty modules in query + +πŸ”— exportReferencedContentTypes (3 tests) +- βœ… should handle no referenced content types found +- βœ… should export new referenced content types +- βœ… should handle file system errors gracefully + +🧩 exportDependentModules (4 tests) +- βœ… should export all dependency types when found +- βœ… should skip empty dependency sets +- βœ… should handle partial dependencies +- βœ… should handle dependencies extraction errors + +πŸ“ exportContentModules (4 tests) +- βœ… should export entries and then assets +- βœ… should include delay before asset export +- βœ… should handle entries export errors +- βœ… should handle assets export errors + +πŸ“‹ exportEntries (2 tests) +- βœ… should export entries module +- βœ… should handle entries export errors + +πŸ–ΌοΈ exportReferencedAssets (4 tests) +- βœ… should export referenced assets when found +- βœ… should skip export when no assets found +- βœ… should handle asset extraction errors +- βœ… should handle asset export errors + +═══════════════════════════════════════════════════════════════════ + +πŸ“ COMMON-HELPER.TEST.TS (4 Test Cases) +═══════════════════════════════════════════════════════════════════ + +πŸ”‘ askAPIKey (4 tests) +- βœ… should prompt user for API key and return the response +- βœ… should handle empty API key input +- βœ… should handle inquire errors +- βœ… should validate the inquire call structure + +═══════════════════════════════════════════════════════════════════ + +πŸ“ CONFIG-HANDLER.TEST.TS (16 Test Cases) +═══════════════════════════════════════════════════════════════════ + +βš™οΈ setupQueryExportConfig + πŸ“ with minimal flags (1 test) + - βœ… should create config with default values + + πŸ“‚ with custom data directory (1 test) + - βœ… should use custom data directory when provided + + ⏭️ with skip flags (1 test) + - βœ… should set skip flags when provided + + 🌿 with branch name (1 test) + - βœ… should include branch name when provided + + πŸ“„ external config path (1 test) + - βœ… should set external config path correctly + + πŸ” stack API key handling (2 tests) + - βœ… should use provided stack API key + - βœ… should handle empty stack API key + + πŸ—οΈ configuration object structure (2 tests) + - βœ… should include all required configuration properties + - βœ… should set isQueryBasedExport to true + + ❌ error scenarios (2 tests) + - βœ… should handle missing query parameter + - βœ… should handle invalid flag types + + πŸ“ path handling (2 tests) + - βœ… should ensure paths are consistent + - βœ… should handle absolute paths + + πŸ”‘ askAPIKey integration (2 tests) + - βœ… should call askAPIKey when no stack API key provided + - βœ… should handle askAPIKey returning non-string value + +═══════════════════════════════════════════════════════════════════ + +πŸ“ CONTENT-TYPE-HELPER.TEST.TS (15 Test Cases) +═══════════════════════════════════════════════════════════════════ + +πŸ”— extractReferencedContentTypes (11 tests) +- βœ… should extract reference field targets +- βœ… should exclude sys_assets from references +- βœ… should handle group fields with nested schemas +- βœ… should handle global fields with nested schemas +- βœ… should handle blocks with nested schemas +- βœ… should handle JSON RTE with embedded entries +- βœ… should handle Text RTE with embedded entries +- βœ… should handle content types without schemas +- βœ… should return empty array for content types with no references +- βœ… should handle complex nested structures +- βœ… should remove duplicates from referenced content types + +πŸ†• filterNewlyFetchedContentTypes (4 tests) +- βœ… should filter out content types that were previously fetched +- βœ… should return all content types when no previous UIDs +- βœ… should return empty array when all content types were previously fetched +- βœ… should handle empty content types array + +═══════════════════════════════════════════════════════════════════ + +πŸ“ DEPENDENCY-RESOLVER.TEST.TS (10 Test Cases) +═══════════════════════════════════════════════════════════════════ + +πŸ”— Schema dependency extraction logic (10 tests) +- βœ… should extract global field dependencies from schema +- βœ… should extract extension dependencies from schema +- βœ… should extract taxonomy dependencies from schema +- βœ… should handle group fields with nested dependencies +- βœ… should handle block fields with nested dependencies +- βœ… should handle complex nested structures +- βœ… should ignore fields without dependency information +- βœ… should handle taxonomies without taxonomy_uid gracefully +- βœ… should handle mixed dependency types in single schema +- βœ… should handle empty schema arrays + +═══════════════════════════════════════════════════════════════════ + +πŸ“ QUERY-PARSER-SIMPLE.TEST.TS (8 Test Cases) +═══════════════════════════════════════════════════════════════════ + +πŸ” JSON string parsing and validation (8 tests) +- βœ… should parse and validate a simple valid query +- βœ… should validate and reject queries without modules +- βœ… should validate and reject queries with empty modules +- βœ… should validate and reject queries with non-queryable modules +- βœ… should handle invalid JSON gracefully +- βœ… should handle complex valid queries +- βœ… should reject null queries +- βœ… should reject string queries + +═══════════════════════════════════════════════════════════════════ + +πŸ“ REFERENCED-ASSET-HANDLER.TEST.TS (15 Test Cases) +═══════════════════════════════════════════════════════════════════ + +πŸ–ΌοΈ Asset UID extraction from content strings (13 tests) +- βœ… should extract asset UIDs from HTML img tags +- βœ… should extract asset UIDs from Contentstack asset URLs +- βœ… should handle mixed asset references in content +- βœ… should handle Azure region URLs +- βœ… should handle GCP region URLs +- βœ… should return empty array for content without assets +- βœ… should handle malformed asset references gracefully +- βœ… should deduplicate asset UIDs from same content +- βœ… should handle escaped quotes in HTML +- βœ… should handle JSON-stringified content with asset references +- βœ… should handle content with special characters in asset UIDs +- βœ… should handle large content strings efficiently +- βœ… should handle contentstack.com domain URLs + +πŸ—οΈ Constructor and initialization (2 tests) +- βœ… should initialize with correct export directory path +- βœ… should store export configuration + +═══════════════════════════════════════════════════════════════════ + +πŸ“Š SUMMARY STATISTICS +═══════════════════════════════════════════════════════════════════ + +πŸ“ˆ Test Distribution: +- Core Modules: 67 tests (51.1%) - QueryExporter + ModuleExporter +- Utils Modules: 64 tests (48.9%) - All utility functions + +🎯 Coverage Breakdown: +- query-executor.ts: 100% coverage (PERFECT!) +- module-exporter.ts: 70.17% coverage +- content-type-helper.ts: 100% coverage (PERFECT!) +- common-helper.ts: 100% coverage (PERFECT!) +- file-helper.ts: 100% coverage (PERFECT!) +- query-parser.ts: 81.48% coverage +- config-handler.ts: 70% coverage +- dependency-resolver.ts: 70% coverage +- referenced-asset-handler.ts: 30.64% coverage +- logger.ts: 18.03% coverage + +⚑ Performance: +- Execution Time: ~160ms +- Exit Code: 0 (Perfect CI/CD compatibility) +- No interactive prompts (Non-blocking) + +πŸ† Key Testing Achievements: +- Complete export workflow orchestration +- Comprehensive command building logic +- Error handling and recovery patterns +- Business logic validation +- Edge case coverage +- Integration scenario testing +- TypeScript type safety validation + +═══════════════════════════════════════════════════════════════════ +Generated: $(date) +CLI Query Export Plugin - Contentstack +═══════════════════════════════════════════════════════════════════ \ No newline at end of file diff --git a/packages/contentstack-query-export/tsconfig.json b/packages/contentstack-query-export/tsconfig.json new file mode 100644 index 000000000..513662339 --- /dev/null +++ b/packages/contentstack-query-export/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "declaration": true, + "importHelpers": true, + "module": "commonjs", + "outDir": "lib", + "rootDir": "src", + "strict": false, + "target": "es2017", + "allowJs": true, + "skipLibCheck": true, + "sourceMap": false, + "esModuleInterop": true, + "noImplicitAny": true, + "lib": [ + "ES2019", + "es2020.promise" + ], + "strictPropertyInitialization": false, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "src/**/*", + "types/*" + ], + "exclude": [ + "node_modules", + "lib" + ] +} \ No newline at end of file diff --git a/packages/contentstack-query-export/types/index.d.ts b/packages/contentstack-query-export/types/index.d.ts new file mode 100644 index 000000000..68a9e1353 --- /dev/null +++ b/packages/contentstack-query-export/types/index.d.ts @@ -0,0 +1 @@ +declare module 'big-json'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 76dc531b7..b5cd0d09b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -792,6 +792,121 @@ importers: specifier: ^4.9.5 version: 4.9.5 + packages/contentstack-query-export: + dependencies: + '@contentstack/cli-cm-export': + specifier: ~1.24.2 + version: link:../contentstack-export + '@contentstack/cli-command': + specifier: ~1.8.1 + version: 1.8.1(@types/node@20.19.39) + '@contentstack/cli-utilities': + specifier: ~1.18.2 + version: 1.18.2(@types/node@20.19.39) + '@contentstack/cli-variants': + specifier: ~1.4.2 + version: link:../contentstack-variants + '@oclif/core': + specifier: ^4.10.5 + version: 4.10.6 + async: + specifier: ^3.2.6 + version: 3.2.6 + big-json: + specifier: ^3.2.0 + version: 3.2.0 + bluebird: + specifier: ^3.7.2 + version: 3.7.2 + lodash: + specifier: ^4.18.1 + version: 4.18.1 + merge: + specifier: ^2.1.1 + version: 2.1.1 + mkdirp: + specifier: ^1.0.4 + version: 1.0.4 + progress-stream: + specifier: ^2.0.0 + version: 2.0.0 + promise-limit: + specifier: ^2.7.0 + version: 2.7.0 + tslib: + specifier: ^2.8.1 + version: 2.8.1 + winston: + specifier: ^3.19.0 + version: 3.19.0 + devDependencies: + '@contentstack/cli-dev-dependencies': + specifier: ~1.3.1 + version: 1.3.1 + '@oclif/plugin-help': + specifier: ^6.2.44 + version: 6.2.45 + '@oclif/test': + specifier: ^4.1.18 + version: 4.1.18(@oclif/core@4.10.6) + '@types/big-json': + specifier: ^3.2.5 + version: 3.2.5 + '@types/chai': + specifier: ^4.3.20 + version: 4.3.20 + '@types/mkdirp': + specifier: ^1.0.2 + version: 1.0.2 + '@types/mocha': + specifier: ^10.0.10 + version: 10.0.10 + '@types/node': + specifier: ^20.19.39 + version: 20.19.39 + '@types/progress-stream': + specifier: ^2.0.5 + version: 2.0.5 + '@types/sinon': + specifier: ^17.0.4 + version: 17.0.4 + chai: + specifier: ^4.5.0 + version: 4.5.0 + dotenv: + specifier: ^16.6.1 + version: 16.6.1 + dotenv-expand: + specifier: ^9.0.0 + version: 9.0.0 + eslint: + specifier: ^8.57.1 + version: 8.57.1 + eslint-config-oclif: + specifier: ^6.0.157 + version: 6.0.160(eslint@8.57.1)(typescript@4.9.5) + husky: + specifier: ^9.1.7 + version: 9.1.7 + mocha: + specifier: 10.8.2 + version: 10.8.2 + nyc: + specifier: ^15.1.0 + version: 15.1.0 + oclif: + specifier: ^4.17.46 + version: 4.23.0(@types/node@20.19.39) + sinon: + specifier: ^17.0.2 + version: 17.0.2 + ts-node: + specifier: ^10.9.2 + version: 10.9.2(@types/node@20.19.39)(typescript@4.9.5) + typescript: + specifier: ^4.9.5 + version: 4.9.5 + packages/contentstack-seed: dependencies: '@contentstack/cli-cm-import': @@ -1332,10 +1447,18 @@ packages: resolution: {integrity: sha512-pHoYRWS08oeU0qVez1pZCcbqHzoJnM5VMtrxH2nWDJ0ukq9DkwWV1BTY+PWK+eWBbndN9W0O9WjJTyAHsDoPOg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@eslint/eslintrc@2.1.4': + resolution: {integrity: sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + '@eslint/eslintrc@3.3.5': resolution: {integrity: sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@eslint/js@8.57.1': + resolution: {integrity: sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + '@eslint/js@9.39.4': resolution: {integrity: sha512-nE7DEIchvtiFTwBw4Lfbu59PG+kCofhjsKaCWzxTpt4lfRjRMqG6uMBzKXuEcyXhOHoUp9riAm7/aWYGhXZ9cw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -1374,6 +1497,11 @@ packages: resolution: {integrity: sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==} engines: {node: '>=18.18.0'} + '@humanwhocodes/config-array@0.13.0': + resolution: {integrity: sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==} + engines: {node: '>=10.10.0'} + deprecated: Use @eslint/config-array instead + '@humanwhocodes/module-importer@1.0.1': resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} engines: {node: '>=12.22'} @@ -1382,6 +1510,10 @@ packages: resolution: {integrity: sha512-KWiFQpSAqEIyrTXko3hFNLeQvSK8zXlJQzhhxsyVn58WFRYXST99b3Nqnu+ttOtjds2Pl2grUHGpe2NzhPynuQ==} engines: {node: '>=18'} + '@humanwhocodes/object-schema@2.0.3': + resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==} + deprecated: Use @eslint/object-schema instead + '@humanwhocodes/retry@0.4.3': resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} engines: {node: '>=18.18'} @@ -2349,6 +2481,9 @@ packages: resolution: {integrity: sha512-/uejZt4dSere1bx12WLlPfv8GktzcaDtuJ7s42/HEZ5zGj9oxRaD4bj7qwSunXkf+pbAhFt2zjpHYUiT5lHf0Q==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@ungap/structured-clone@1.3.0': + resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + '@unrs/resolver-binding-android-arm-eabi@1.11.1': resolution: {integrity: sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==} cpu: [arm] @@ -3233,6 +3368,10 @@ packages: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} engines: {node: '>=0.10.0'} + doctrine@3.0.0: + resolution: {integrity: sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==} + engines: {node: '>=6.0.0'} + dot-case@3.0.4: resolution: {integrity: sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==} @@ -3502,6 +3641,10 @@ packages: resolution: {integrity: sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==} engines: {node: '>=8.0.0'} + eslint-scope@7.2.2: + resolution: {integrity: sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + eslint-scope@8.4.0: resolution: {integrity: sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -3536,6 +3679,12 @@ packages: resolution: {integrity: sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} + eslint@8.57.1: + resolution: {integrity: sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. + hasBin: true + eslint@9.39.4: resolution: {integrity: sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -3550,6 +3699,10 @@ packages: resolution: {integrity: sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + espree@9.6.1: + resolution: {integrity: sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + esprima@4.0.1: resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} engines: {node: '>=4'} @@ -3668,6 +3821,10 @@ packages: resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==} engines: {node: '>=8'} + file-entry-cache@6.0.1: + resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} + engines: {node: ^10.12.0 || >=12.0.0} + file-entry-cache@8.0.0: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} @@ -3715,6 +3872,10 @@ packages: find-yarn-workspace-root@2.0.0: resolution: {integrity: sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ==} + flat-cache@3.2.0: + resolution: {integrity: sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==} + engines: {node: ^10.12.0 || >=12.0.0} + flat-cache@4.0.1: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} @@ -4195,6 +4356,10 @@ packages: resolution: {integrity: sha512-NqCa4Sa2d+u7BWc6CukaObG3Fh+CU9bvixbpcXYhy2VvYS7vVGIdAgnIS5Ks3A/cqk4rebLJ9s8zBstT2aKnIA==} engines: {node: '>=4'} + is-path-inside@3.0.3: + resolution: {integrity: sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==} + engines: {node: '>=8'} + is-plain-obj@2.1.0: resolution: {integrity: sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==} engines: {node: '>=8'} @@ -5822,6 +5987,9 @@ packages: text-hex@1.0.0: resolution: {integrity: sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==} + text-table@0.2.0: + resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} + thirty-two@1.0.2: resolution: {integrity: sha512-OEI0IWCe+Dw46019YLl6V10Us5bi574EvlJEOcAkB29IzQ/mYD1A6RyNHLjZPiHCmuodxvgF6U+vZO1L15lxVA==} engines: {node: '>=0.2.6'} @@ -7240,6 +7408,11 @@ snapshots: esquery: 1.7.0 jsdoc-type-pratt-parser: 4.1.0 + '@eslint-community/eslint-utils@4.9.1(eslint@8.57.1)': + dependencies: + eslint: 8.57.1 + eslint-visitor-keys: 3.4.3 + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4)': dependencies: eslint: 9.39.4 @@ -7247,6 +7420,12 @@ snapshots: '@eslint-community/regexpp@4.12.2': {} + '@eslint/compat@1.4.1(eslint@8.57.1)': + dependencies: + '@eslint/core': 0.17.0 + optionalDependencies: + eslint: 8.57.1 + '@eslint/compat@1.4.1(eslint@9.39.4)': dependencies: '@eslint/core': 0.17.0 @@ -7288,6 +7467,20 @@ snapshots: '@eslint/css-tree': 3.6.9 '@eslint/plugin-kit': 0.3.5 + '@eslint/eslintrc@2.1.4': + dependencies: + ajv: 6.15.0 + debug: 4.4.3(supports-color@8.1.1) + espree: 9.6.1 + globals: 13.24.0 + ignore: 5.3.2 + import-fresh: 3.3.1 + js-yaml: 4.1.1 + minimatch: 3.1.5 + strip-json-comments: 3.1.1 + transitivePeerDependencies: + - supports-color + '@eslint/eslintrc@3.3.5': dependencies: ajv: 6.15.0 @@ -7302,6 +7495,8 @@ snapshots: transitivePeerDependencies: - supports-color + '@eslint/js@8.57.1': {} + '@eslint/js@9.39.4': {} '@eslint/json@0.13.2': @@ -7354,10 +7549,20 @@ snapshots: '@humanfs/types@0.15.0': {} + '@humanwhocodes/config-array@0.13.0': + dependencies: + '@humanwhocodes/object-schema': 2.0.3 + debug: 4.4.3(supports-color@8.1.1) + minimatch: 3.1.5 + transitivePeerDependencies: + - supports-color + '@humanwhocodes/module-importer@1.0.1': {} '@humanwhocodes/momoa@3.3.10': {} + '@humanwhocodes/object-schema@2.0.3': {} + '@humanwhocodes/retry@0.4.3': {} '@inquirer/ansi@1.0.2': {} @@ -8514,6 +8719,18 @@ snapshots: color: 5.0.3 text-hex: 1.0.0 + '@stylistic/eslint-plugin@3.1.0(eslint@8.57.1)(typescript@4.9.5)': + dependencies: + '@typescript-eslint/utils': 8.59.0(eslint@8.57.1)(typescript@4.9.5) + eslint: 8.57.1 + eslint-visitor-keys: 4.2.1 + espree: 10.4.0 + estraverse: 5.3.0 + picomatch: 4.0.4 + transitivePeerDependencies: + - supports-color + - typescript + '@stylistic/eslint-plugin@3.1.0(eslint@9.39.4)(typescript@4.9.5)': dependencies: '@typescript-eslint/utils': 8.59.0(eslint@9.39.4)(typescript@4.9.5) @@ -8538,6 +8755,16 @@ snapshots: - supports-color - typescript + '@stylistic/eslint-plugin@5.10.0(eslint@8.57.1)': + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@8.57.1) + '@typescript-eslint/types': 8.59.0 + eslint: 8.57.1 + eslint-visitor-keys: 4.2.1 + espree: 10.4.0 + estraverse: 5.3.0 + picomatch: 4.0.4 + '@stylistic/eslint-plugin@5.10.0(eslint@9.39.4)': dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4) @@ -8772,6 +8999,22 @@ snapshots: transitivePeerDependencies: - supports-color + '@typescript-eslint/eslint-plugin@8.59.0(@typescript-eslint/parser@8.59.0(eslint@8.57.1)(typescript@4.9.5))(eslint@8.57.1)(typescript@4.9.5)': + dependencies: + '@eslint-community/regexpp': 4.12.2 + '@typescript-eslint/parser': 8.59.0(eslint@8.57.1)(typescript@4.9.5) + '@typescript-eslint/scope-manager': 8.59.0 + '@typescript-eslint/type-utils': 8.59.0(eslint@8.57.1)(typescript@4.9.5) + '@typescript-eslint/utils': 8.59.0(eslint@8.57.1)(typescript@4.9.5) + '@typescript-eslint/visitor-keys': 8.59.0 + eslint: 8.57.1 + ignore: 7.0.5 + natural-compare: 1.4.0 + ts-api-utils: 2.5.0(typescript@4.9.5) + typescript: 4.9.5 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/eslint-plugin@8.59.0(@typescript-eslint/parser@8.59.0(eslint@9.39.4)(typescript@4.9.5))(eslint@9.39.4)(typescript@4.9.5)': dependencies: '@eslint-community/regexpp': 4.12.2 @@ -8830,6 +9073,18 @@ snapshots: transitivePeerDependencies: - supports-color + '@typescript-eslint/parser@8.59.0(eslint@8.57.1)(typescript@4.9.5)': + dependencies: + '@typescript-eslint/scope-manager': 8.59.0 + '@typescript-eslint/types': 8.59.0 + '@typescript-eslint/typescript-estree': 8.59.0(typescript@4.9.5) + '@typescript-eslint/visitor-keys': 8.59.0 + debug: 4.4.3(supports-color@8.1.1) + eslint: 8.57.1 + typescript: 4.9.5 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/parser@8.59.0(eslint@9.39.4)(typescript@4.9.5)': dependencies: '@typescript-eslint/scope-manager': 8.59.0 @@ -8936,6 +9191,18 @@ snapshots: transitivePeerDependencies: - supports-color + '@typescript-eslint/type-utils@8.59.0(eslint@8.57.1)(typescript@4.9.5)': + dependencies: + '@typescript-eslint/types': 8.59.0 + '@typescript-eslint/typescript-estree': 8.59.0(typescript@4.9.5) + '@typescript-eslint/utils': 8.59.0(eslint@8.57.1)(typescript@4.9.5) + debug: 4.4.3(supports-color@8.1.1) + eslint: 8.57.1 + ts-api-utils: 2.5.0(typescript@4.9.5) + typescript: 4.9.5 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/type-utils@8.59.0(eslint@9.39.4)(typescript@4.9.5)': dependencies: '@typescript-eslint/types': 8.59.0 @@ -9137,6 +9404,17 @@ snapshots: - supports-color - typescript + '@typescript-eslint/utils@8.59.0(eslint@8.57.1)(typescript@4.9.5)': + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@8.57.1) + '@typescript-eslint/scope-manager': 8.59.0 + '@typescript-eslint/types': 8.59.0 + '@typescript-eslint/typescript-estree': 8.59.0(typescript@4.9.5) + eslint: 8.57.1 + typescript: 4.9.5 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/utils@8.59.0(eslint@9.39.4)(typescript@4.9.5)': dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4) @@ -9179,6 +9457,8 @@ snapshots: '@typescript-eslint/types': 8.59.0 eslint-visitor-keys: 5.0.1 + '@ungap/structured-clone@1.3.0': {} + '@unrs/resolver-binding-android-arm-eabi@1.11.1': optional: true @@ -10090,6 +10370,10 @@ snapshots: dependencies: esutils: 2.0.3 + doctrine@3.0.0: + dependencies: + esutils: 2.0.3 + dot-case@3.0.4: dependencies: no-case: 3.0.4 @@ -10234,6 +10518,11 @@ snapshots: escape-string-regexp@4.0.0: {} + eslint-compat-utils@0.5.1(eslint@8.57.1): + dependencies: + eslint: 8.57.1 + semver: 7.7.4 + eslint-compat-utils@0.5.1(eslint@9.39.4): dependencies: eslint: 9.39.4 @@ -10281,6 +10570,15 @@ snapshots: - typescript - vue-eslint-parser + eslint-config-oclif@5.2.2(eslint@8.57.1): + dependencies: + eslint-config-xo-space: 0.35.0(eslint@8.57.1) + eslint-plugin-mocha: 10.5.0(eslint@8.57.1) + eslint-plugin-n: 15.7.0(eslint@8.57.1) + eslint-plugin-unicorn: 48.0.1(eslint@8.57.1) + transitivePeerDependencies: + - eslint + eslint-config-oclif@5.2.2(eslint@9.39.4): dependencies: eslint-config-xo-space: 0.35.0(eslint@9.39.4) @@ -10290,6 +10588,32 @@ snapshots: transitivePeerDependencies: - eslint + eslint-config-oclif@6.0.160(eslint@8.57.1)(typescript@4.9.5): + dependencies: + '@eslint/compat': 1.4.1(eslint@8.57.1) + '@eslint/eslintrc': 3.3.5 + '@eslint/js': 9.39.4 + '@stylistic/eslint-plugin': 3.1.0(eslint@8.57.1)(typescript@4.9.5) + '@typescript-eslint/eslint-plugin': 8.59.0(@typescript-eslint/parser@8.59.0(eslint@8.57.1)(typescript@4.9.5))(eslint@8.57.1)(typescript@4.9.5) + '@typescript-eslint/parser': 8.59.0(eslint@8.57.1)(typescript@4.9.5) + eslint-config-oclif: 5.2.2(eslint@8.57.1) + eslint-config-xo: 0.49.0(eslint@8.57.1) + eslint-config-xo-space: 0.35.0(eslint@8.57.1) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@8.57.1) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.0(eslint@8.57.1)(typescript@4.9.5))(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1) + eslint-plugin-jsdoc: 50.8.0(eslint@8.57.1) + eslint-plugin-mocha: 10.5.0(eslint@8.57.1) + eslint-plugin-n: 17.24.0(eslint@8.57.1)(typescript@4.9.5) + eslint-plugin-perfectionist: 4.15.1(eslint@8.57.1)(typescript@4.9.5) + eslint-plugin-unicorn: 56.0.1(eslint@8.57.1) + typescript-eslint: 8.59.0(eslint@8.57.1)(typescript@4.9.5) + transitivePeerDependencies: + - eslint + - eslint-import-resolver-webpack + - eslint-plugin-import-x + - supports-color + - typescript + eslint-config-oclif@6.0.160(eslint@9.39.4)(typescript@4.9.5): dependencies: '@eslint/compat': 1.4.1(eslint@9.39.4) @@ -10342,16 +10666,35 @@ snapshots: - supports-color - typescript + eslint-config-xo-space@0.35.0(eslint@8.57.1): + dependencies: + eslint: 8.57.1 + eslint-config-xo: 0.44.0(eslint@8.57.1) + eslint-config-xo-space@0.35.0(eslint@9.39.4): dependencies: eslint: 9.39.4 eslint-config-xo: 0.44.0(eslint@9.39.4) + eslint-config-xo@0.44.0(eslint@8.57.1): + dependencies: + confusing-browser-globals: 1.0.11 + eslint: 8.57.1 + eslint-config-xo@0.44.0(eslint@9.39.4): dependencies: confusing-browser-globals: 1.0.11 eslint: 9.39.4 + eslint-config-xo@0.49.0(eslint@8.57.1): + dependencies: + '@eslint/css': 0.10.0 + '@eslint/json': 0.13.2 + '@stylistic/eslint-plugin': 5.10.0(eslint@8.57.1) + confusing-browser-globals: 1.0.11 + eslint: 8.57.1 + globals: 16.5.0 + eslint-config-xo@0.49.0(eslint@9.39.4): dependencies: '@eslint/css': 0.10.0 @@ -10384,6 +10727,21 @@ snapshots: transitivePeerDependencies: - supports-color + eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@8.57.1): + dependencies: + '@nolyfill/is-core-module': 1.0.39 + debug: 4.4.3(supports-color@8.1.1) + eslint: 8.57.1 + get-tsconfig: 4.14.0 + is-bun-module: 2.0.0 + stable-hash: 0.0.5 + tinyglobby: 0.2.16 + unrs-resolver: 1.11.1 + optionalDependencies: + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.0(eslint@8.57.1)(typescript@4.9.5))(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1) + transitivePeerDependencies: + - supports-color + eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4): dependencies: '@nolyfill/is-core-module': 1.0.39 @@ -10421,6 +10779,17 @@ snapshots: transitivePeerDependencies: - supports-color + eslint-module-utils@2.12.1(@typescript-eslint/parser@8.59.0(eslint@8.57.1)(typescript@4.9.5))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1): + dependencies: + debug: 3.2.7 + optionalDependencies: + '@typescript-eslint/parser': 8.59.0(eslint@8.57.1)(typescript@4.9.5) + eslint: 8.57.1 + eslint-import-resolver-node: 0.3.10 + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@8.57.1) + transitivePeerDependencies: + - supports-color + eslint-module-utils@2.12.1(@typescript-eslint/parser@8.59.0(eslint@9.39.4)(typescript@4.9.5))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4): dependencies: debug: 3.2.7 @@ -10443,6 +10812,13 @@ snapshots: transitivePeerDependencies: - supports-color + eslint-plugin-es-x@7.8.0(eslint@8.57.1): + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@8.57.1) + '@eslint-community/regexpp': 4.12.2 + eslint: 8.57.1 + eslint-compat-utils: 0.5.1(eslint@8.57.1) + eslint-plugin-es-x@7.8.0(eslint@9.39.4): dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4) @@ -10450,6 +10826,12 @@ snapshots: eslint: 9.39.4 eslint-compat-utils: 0.5.1(eslint@9.39.4) + eslint-plugin-es@4.1.0(eslint@8.57.1): + dependencies: + eslint: 8.57.1 + eslint-utils: 2.1.0 + regexpp: 3.2.0 + eslint-plugin-es@4.1.0(eslint@9.39.4): dependencies: eslint: 9.39.4 @@ -10514,6 +10896,35 @@ snapshots: - eslint-import-resolver-webpack - supports-color + eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.59.0(eslint@8.57.1)(typescript@4.9.5))(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1): + dependencies: + '@rtsao/scc': 1.1.0 + array-includes: 3.1.9 + array.prototype.findlastindex: 1.2.6 + array.prototype.flat: 1.3.3 + array.prototype.flatmap: 1.3.3 + debug: 3.2.7 + doctrine: 2.1.0 + eslint: 8.57.1 + eslint-import-resolver-node: 0.3.10 + eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.59.0(eslint@8.57.1)(typescript@4.9.5))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1)(eslint@8.57.1) + hasown: 2.0.3 + is-core-module: 2.16.1 + is-glob: 4.0.3 + minimatch: 3.1.5 + object.fromentries: 2.0.8 + object.groupby: 1.0.3 + object.values: 1.2.1 + semver: 6.3.1 + string.prototype.trimend: 1.0.9 + tsconfig-paths: 3.15.0 + optionalDependencies: + '@typescript-eslint/parser': 8.59.0(eslint@8.57.1)(typescript@4.9.5) + transitivePeerDependencies: + - eslint-import-resolver-typescript + - eslint-import-resolver-webpack + - supports-color + eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.59.0(eslint@9.39.4)(typescript@4.9.5))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4): dependencies: '@rtsao/scc': 1.1.0 @@ -10572,6 +10983,22 @@ snapshots: - eslint-import-resolver-webpack - supports-color + eslint-plugin-jsdoc@50.8.0(eslint@8.57.1): + dependencies: + '@es-joy/jsdoccomment': 0.50.2 + are-docs-informative: 0.0.2 + comment-parser: 1.4.1 + debug: 4.4.3(supports-color@8.1.1) + escape-string-regexp: 4.0.0 + eslint: 8.57.1 + espree: 10.4.0 + esquery: 1.7.0 + parse-imports-exports: 0.2.4 + semver: 7.7.4 + spdx-expression-parse: 4.0.0 + transitivePeerDependencies: + - supports-color + eslint-plugin-jsdoc@50.8.0(eslint@9.39.4): dependencies: '@es-joy/jsdoccomment': 0.50.2 @@ -10588,6 +11015,13 @@ snapshots: transitivePeerDependencies: - supports-color + eslint-plugin-mocha@10.5.0(eslint@8.57.1): + dependencies: + eslint: 8.57.1 + eslint-utils: 3.0.0(eslint@8.57.1) + globals: 13.24.0 + rambda: 7.5.0 + eslint-plugin-mocha@10.5.0(eslint@9.39.4): dependencies: eslint: 9.39.4 @@ -10595,6 +11029,18 @@ snapshots: globals: 13.24.0 rambda: 7.5.0 + eslint-plugin-n@15.7.0(eslint@8.57.1): + dependencies: + builtins: 5.1.0 + eslint: 8.57.1 + eslint-plugin-es: 4.1.0(eslint@8.57.1) + eslint-utils: 3.0.0(eslint@8.57.1) + ignore: 5.3.2 + is-core-module: 2.16.1 + minimatch: 3.1.5 + resolve: 1.22.12 + semver: 7.7.4 + eslint-plugin-n@15.7.0(eslint@9.39.4): dependencies: builtins: 5.1.0 @@ -10607,6 +11053,21 @@ snapshots: resolve: 1.22.12 semver: 7.7.4 + eslint-plugin-n@17.24.0(eslint@8.57.1)(typescript@4.9.5): + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@8.57.1) + enhanced-resolve: 5.21.0 + eslint: 8.57.1 + eslint-plugin-es-x: 7.8.0(eslint@8.57.1) + get-tsconfig: 4.14.0 + globals: 15.15.0 + globrex: 0.1.2 + ignore: 5.3.2 + semver: 7.7.4 + ts-declaration-location: 1.0.7(typescript@4.9.5) + transitivePeerDependencies: + - typescript + eslint-plugin-n@17.24.0(eslint@9.39.4)(typescript@4.9.5): dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4) @@ -10657,6 +11118,16 @@ snapshots: - supports-color - typescript + eslint-plugin-perfectionist@4.15.1(eslint@8.57.1)(typescript@4.9.5): + dependencies: + '@typescript-eslint/types': 8.59.0 + '@typescript-eslint/utils': 8.59.0(eslint@8.57.1)(typescript@4.9.5) + eslint: 8.57.1 + natural-orderby: 5.0.0 + transitivePeerDependencies: + - supports-color + - typescript + eslint-plugin-perfectionist@4.15.1(eslint@9.39.4)(typescript@4.9.5): dependencies: '@typescript-eslint/types': 8.59.0 @@ -10677,6 +11148,25 @@ snapshots: - supports-color - typescript + eslint-plugin-unicorn@48.0.1(eslint@8.57.1): + dependencies: + '@babel/helper-validator-identifier': 7.28.5 + '@eslint-community/eslint-utils': 4.9.1(eslint@8.57.1) + ci-info: 3.9.0 + clean-regexp: 1.0.0 + eslint: 8.57.1 + esquery: 1.7.0 + indent-string: 4.0.0 + is-builtin-module: 3.2.1 + jsesc: 3.1.0 + lodash: 4.18.1 + pluralize: 8.0.0 + read-pkg-up: 7.0.1 + regexp-tree: 0.1.27 + regjsparser: 0.10.0 + semver: 7.7.4 + strip-indent: 3.0.0 + eslint-plugin-unicorn@48.0.1(eslint@9.39.4): dependencies: '@babel/helper-validator-identifier': 7.28.5 @@ -10696,6 +11186,26 @@ snapshots: semver: 7.7.4 strip-indent: 3.0.0 + eslint-plugin-unicorn@56.0.1(eslint@8.57.1): + dependencies: + '@babel/helper-validator-identifier': 7.28.5 + '@eslint-community/eslint-utils': 4.9.1(eslint@8.57.1) + ci-info: 4.4.0 + clean-regexp: 1.0.0 + core-js-compat: 3.49.0 + eslint: 8.57.1 + esquery: 1.7.0 + globals: 15.15.0 + indent-string: 4.0.0 + is-builtin-module: 3.2.1 + jsesc: 3.1.0 + pluralize: 8.0.0 + read-pkg-up: 7.0.1 + regexp-tree: 0.1.27 + regjsparser: 0.10.0 + semver: 7.7.4 + strip-indent: 3.0.0 + eslint-plugin-unicorn@56.0.1(eslint@9.39.4): dependencies: '@babel/helper-validator-identifier': 7.28.5 @@ -10721,6 +11231,11 @@ snapshots: esrecurse: 4.3.0 estraverse: 4.3.0 + eslint-scope@7.2.2: + dependencies: + esrecurse: 4.3.0 + estraverse: 5.3.0 + eslint-scope@8.4.0: dependencies: esrecurse: 4.3.0 @@ -10730,6 +11245,11 @@ snapshots: dependencies: eslint-visitor-keys: 1.3.0 + eslint-utils@3.0.0(eslint@8.57.1): + dependencies: + eslint: 8.57.1 + eslint-visitor-keys: 2.1.0 + eslint-utils@3.0.0(eslint@9.39.4): dependencies: eslint: 9.39.4 @@ -10745,6 +11265,49 @@ snapshots: eslint-visitor-keys@5.0.1: {} + eslint@8.57.1: + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@8.57.1) + '@eslint-community/regexpp': 4.12.2 + '@eslint/eslintrc': 2.1.4 + '@eslint/js': 8.57.1 + '@humanwhocodes/config-array': 0.13.0 + '@humanwhocodes/module-importer': 1.0.1 + '@nodelib/fs.walk': 1.2.8 + '@ungap/structured-clone': 1.3.0 + ajv: 6.15.0 + chalk: 4.1.2 + cross-spawn: 7.0.6 + debug: 4.4.3(supports-color@8.1.1) + doctrine: 3.0.0 + escape-string-regexp: 4.0.0 + eslint-scope: 7.2.2 + eslint-visitor-keys: 3.4.3 + espree: 9.6.1 + esquery: 1.7.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 6.0.1 + find-up: 5.0.0 + glob-parent: 6.0.2 + globals: 13.24.0 + graphemer: 1.4.0 + ignore: 5.3.2 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + is-path-inside: 3.0.3 + js-yaml: 4.1.1 + json-stable-stringify-without-jsonify: 1.0.1 + levn: 0.4.1 + lodash.merge: 4.6.2 + minimatch: 3.1.5 + natural-compare: 1.4.0 + optionator: 0.9.4 + strip-ansi: 6.0.1 + text-table: 0.2.0 + transitivePeerDependencies: + - supports-color + eslint@9.39.4: dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4) @@ -10790,6 +11353,12 @@ snapshots: acorn-jsx: 5.3.2(acorn@8.16.0) eslint-visitor-keys: 4.2.1 + espree@9.6.1: + dependencies: + acorn: 8.16.0 + acorn-jsx: 5.3.2(acorn@8.16.0) + eslint-visitor-keys: 3.4.3 + esprima@4.0.1: {} esquery@1.7.0: @@ -10924,6 +11493,10 @@ snapshots: dependencies: escape-string-regexp: 1.0.5 + file-entry-cache@6.0.1: + dependencies: + flat-cache: 3.2.0 + file-entry-cache@8.0.0: dependencies: flat-cache: 4.0.1 @@ -10971,6 +11544,12 @@ snapshots: dependencies: micromatch: 4.0.8 + flat-cache@3.2.0: + dependencies: + flatted: 3.4.2 + keyv: 4.5.4 + rimraf: 3.0.2 + flat-cache@4.0.1: dependencies: flatted: 3.4.2 @@ -11521,6 +12100,8 @@ snapshots: dependencies: symbol-observable: 1.2.0 + is-path-inside@3.0.3: {} + is-plain-obj@2.1.0: {} is-plain-obj@4.1.0: {} @@ -13534,6 +14115,8 @@ snapshots: text-hex@1.0.0: {} + text-table@0.2.0: {} + thirty-two@1.0.2: {} through2@2.0.5: @@ -13635,6 +14218,24 @@ snapshots: v8-compile-cache-lib: 3.0.1 yn: 3.1.1 + ts-node@10.9.2(@types/node@20.19.39)(typescript@4.9.5): + dependencies: + '@cspotcode/source-map-support': 0.8.1 + '@tsconfig/node10': 1.0.12 + '@tsconfig/node12': 1.0.11 + '@tsconfig/node14': 1.0.3 + '@tsconfig/node16': 1.0.4 + '@types/node': 20.19.39 + acorn: 8.16.0 + acorn-walk: 8.3.5 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.4 + make-error: 1.3.6 + typescript: 4.9.5 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + ts-node@10.9.2(@types/node@20.19.39)(typescript@5.9.3): dependencies: '@cspotcode/source-map-support': 0.8.1 @@ -13778,6 +14379,17 @@ snapshots: typedarray@0.0.6: {} + typescript-eslint@8.59.0(eslint@8.57.1)(typescript@4.9.5): + dependencies: + '@typescript-eslint/eslint-plugin': 8.59.0(@typescript-eslint/parser@8.59.0(eslint@8.57.1)(typescript@4.9.5))(eslint@8.57.1)(typescript@4.9.5) + '@typescript-eslint/parser': 8.59.0(eslint@8.57.1)(typescript@4.9.5) + '@typescript-eslint/typescript-estree': 8.59.0(typescript@4.9.5) + '@typescript-eslint/utils': 8.59.0(eslint@8.57.1)(typescript@4.9.5) + eslint: 8.57.1 + typescript: 4.9.5 + transitivePeerDependencies: + - supports-color + typescript-eslint@8.59.0(eslint@9.39.4)(typescript@4.9.5): dependencies: '@typescript-eslint/eslint-plugin': 8.59.0(@typescript-eslint/parser@8.59.0(eslint@9.39.4)(typescript@4.9.5))(eslint@9.39.4)(typescript@4.9.5)