Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions packages/contentstack-import/src/import/modules/base-class.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ export type ApiModuleType =
| 'create-entries'
| 'update-entries'
| 'publish-entries'
| 'publish-taxonomies'
| 'delete-entries'
| 'create-taxonomies'
| 'create-terms'
Expand Down Expand Up @@ -343,6 +344,8 @@ export default abstract class BaseClass {
if (
!apiData ||
(entity === 'publish-entries' && !apiData.entryUid) ||
(entity === 'publish-taxonomies' &&
(!apiData.environments?.length || !apiData.locales?.length || !apiData.items?.length)) ||
(entity === 'update-extensions' && !apiData.uid)
) {
return Promise.resolve();
Expand Down Expand Up @@ -489,6 +492,14 @@ export default abstract class BaseClass {
})
.then(onSuccess)
.catch(onReject);
case 'publish-taxonomies': {
const publishParams = this.importConfig.branchName ? { branch: this.importConfig.branchName } : {};
return (this.stack as any)
.taxonomy()
.publish(apiData, '3.2', publishParams)
.then(onSuccess)
.catch(onReject);
}
case 'delete-entries':
return this.stack
.contentType(apiData.cTUid)
Expand Down
261 changes: 243 additions & 18 deletions packages/contentstack-import/src/import/modules/taxonomies.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { join } from 'node:path';
import values from 'lodash/values';
import isEmpty from 'lodash/isEmpty';
import { log, handleAndLogError } from '@contentstack/cli-utilities';
import { log, handleAndLogError, CLIProgressManager } from '@contentstack/cli-utilities';
import { PATH_CONSTANTS } from '../../constants';

import BaseClass, { ApiOptions } from './base-class';
Expand All @@ -19,6 +19,8 @@ export default class ImportTaxonomies extends BaseClass {
private termsSuccessPath: string;
private termsFailsPath: string;
private localesFilePath: string;
private envUidMapperPath: string;
private envUidMapper: Record<string, string> = {};
private isLocaleBasedStructure: boolean = false;
public createdTaxonomies: Record<string, unknown> = {};
public failedTaxonomies: Record<string, unknown> = {};
Expand Down Expand Up @@ -46,8 +48,16 @@ export default class ImportTaxonomies extends BaseClass {
importConfig.modules.locales.dirName,
importConfig.modules.locales.fileName,
);
this.envUidMapperPath = join(
importConfig.backupDir,
PATH_CONSTANTS.MAPPER,
PATH_CONSTANTS.MAPPER_MODULES.ENVIRONMENTS,
PATH_CONSTANTS.FILES.UID_MAPPING,
);
}

// --- Lifecycle ---

/**
* @method start
* @returns {Promise<void>} Promise<void>
Expand All @@ -56,7 +66,7 @@ export default class ImportTaxonomies extends BaseClass {
try {
log.debug('Starting taxonomies import process...', this.importConfig.context);

const [taxonomiesCount] = await this.analyzeTaxonomies();
const [taxonomiesCount, publishJobCount] = await this.analyzeTaxonomies();
if (taxonomiesCount === 0) {
log.info('No taxonomies found to import', this.importConfig.context);
return;
Expand All @@ -67,8 +77,12 @@ export default class ImportTaxonomies extends BaseClass {
// Check if locale-based structure exists before import
this.isLocaleBasedStructure = this.detectAndScanLocaleStructure();

const progress = this.createSimpleProgress(this.currentModuleName, taxonomiesCount);
progress.updateStatus(PROCESS_STATUS[PROCESS_NAMES.TAXONOMIES_IMPORT].IMPORTING);
const progress = this.createNestedProgress(this.currentModuleName);
this.initializeTaxonomiesProgress(progress, taxonomiesCount, publishJobCount);

progress
.startProcess(PROCESS_NAMES.TAXONOMIES_IMPORT)
.updateStatus(PROCESS_STATUS[PROCESS_NAMES.TAXONOMIES_IMPORT].IMPORTING, PROCESS_NAMES.TAXONOMIES_IMPORT);
log.debug('Starting taxonomies import', this.importConfig.context);

if (this.isLocaleBasedStructure) {
Expand All @@ -79,6 +93,19 @@ export default class ImportTaxonomies extends BaseClass {
await this.importTaxonomiesLegacy();
}

progress.completeProcess(PROCESS_NAMES.TAXONOMIES_IMPORT, true);

if (publishJobCount > 0) {
progress
.startProcess(PROCESS_NAMES.TAXONOMIES_PUBLISH)
.updateStatus(
PROCESS_STATUS[PROCESS_NAMES.TAXONOMIES_PUBLISH].PUBLISHING,
PROCESS_NAMES.TAXONOMIES_PUBLISH,
);
await this.processTaxonomyPublishing();
progress.completeProcess(PROCESS_NAMES.TAXONOMIES_PUBLISH, true);
}

this.createSuccessAndFailedFile();
this.completeProgressWithMessage();
} catch (error) {
Expand All @@ -87,6 +114,8 @@ export default class ImportTaxonomies extends BaseClass {
}
}

// --- Import ---

/**
* create taxonomy and enter success & failure related data into taxonomies mapper file
* @method importTaxonomies
Expand Down Expand Up @@ -344,6 +373,191 @@ export default class ImportTaxonomies extends BaseClass {
return true;
}

// --- Progress ---

/**
* Registers nested progress for taxonomy import and optional taxonomy publish when publish jobs exist.
*/
initializeTaxonomiesProgress(progress: CLIProgressManager, taxonomyCount: number, publishJobCount: number): void {
progress.addProcess(PROCESS_NAMES.TAXONOMIES_IMPORT, taxonomyCount);
if (publishJobCount > 0) {
progress.addProcess(PROCESS_NAMES.TAXONOMIES_PUBLISH, publishJobCount);
}
}

// --- Publish ---

/**
* Reads source env UID → destination stack env UID map produced during environments import.
*/
private readEnvUidMapperSync(): Record<string, string> {
if (!fileHelper.fileExistsSync(this.envUidMapperPath)) {
log.debug(`Environment UID mapper not found at ${this.envUidMapperPath}`, this.importConfig.context);
return {};
}

try {
const raw = fsUtil.readFile(this.envUidMapperPath, true) as Record<string, unknown>;
const out: Record<string, string> = {};
for (const [k, v] of Object.entries(raw || {})) {
if (v !== undefined && v !== null && String(v).trim() !== '') {
out[k] = String(v);
}
}
return out;
} catch {
log.debug('Failed to read environment UID mapper', this.importConfig.context);
return {};
}
}

private countPublishEligibleTaxonomies(envMapper: Record<string, string>): number {
let count = 0;
for (const key of Object.keys(this.taxonomies || {})) {
const meta = this.taxonomies[key] as Record<string, any>;
const taxonomyUid = meta?.uid || key;
const filePath = this.findTaxonomyFilePath(taxonomyUid);
if (!filePath) continue;

const details = this.loadTaxonomyFile(filePath);
const tax = details?.taxonomy as Record<string, any> | undefined;
if (!tax?.publish_details?.length || !tax?.locale) continue;

const hasMapped = (tax.publish_details as any[]).some(
(p: any) => p?.environment && envMapper[String(p.environment)],
);
if (hasMapped) count++;
}
return count;
}

private collectTaxonomyPublishJobs(): Array<{ taxonomy: Record<string, any> }> {
const jobs: Array<{ taxonomy: Record<string, any> }> = [];
const seen = new Set<string>();

for (const key of Object.keys(this.taxonomies || {})) {
const meta = this.taxonomies[key] as Record<string, any>;
const taxonomyUid = meta?.uid || key;
if (seen.has(taxonomyUid)) continue;

const filePath = this.findTaxonomyFilePath(taxonomyUid);
if (!filePath) continue;

const details = this.loadTaxonomyFile(filePath);
const tax = details?.taxonomy as Record<string, any> | undefined;
if (!tax?.publish_details?.length || !tax?.locale) continue;

seen.add(taxonomyUid);
jobs.push({ taxonomy: tax });
}

return jobs;
}

private loadEnvUidMapper(): void {
this.envUidMapper = this.readEnvUidMapperSync();
if (isEmpty(this.envUidMapper)) {
log.warn(
'Environment UID mapper is empty; taxonomy publishing is skipped. Import environments first or ensure mapper/environments/uid-mapping.json exists.',
this.importConfig.context,
);
}
}

async processTaxonomyPublishing(): Promise<void> {
this.loadEnvUidMapper();
const jobs = this.collectTaxonomyPublishJobs();

if (jobs.length === 0) {
log.debug('No taxonomies with publish_details to publish', this.importConfig.context);
return;
}

log.info('Starting taxonomy publishing process', this.importConfig.context);

const onSuccess = ({ apiData }: any) => {
const taxonomyUid = apiData?.items?.[0]?.uid;
this.progressManager?.tick(
true,
`taxonomy published: ${taxonomyUid}`,
null,
PROCESS_NAMES.TAXONOMIES_PUBLISH,
);
log.success(`Published taxonomy '${taxonomyUid}'`, this.importConfig.context);
};

const onReject = ({ error, apiData }: any) => {
const taxonomyUid = apiData?.items?.[0]?.uid;
handleAndLogError(
error,
{ ...this.importConfig.context, taxonomyUid },
`Failed to publish taxonomy '${taxonomyUid}'`,
);
this.progressManager?.tick(
false,
`taxonomy publish: ${taxonomyUid}`,
(error as Error)?.message || `Failed to publish taxonomy '${taxonomyUid}'`,
PROCESS_NAMES.TAXONOMIES_PUBLISH,
);
};

await this.makeConcurrentCall(
{
apiContent: jobs as unknown as Record<string, any>[],
processName: 'publish taxonomies',
apiParams: {
serializeData: this.serializePublishTaxonomies.bind(this),
reject: onReject,
resolve: onSuccess,
entity: 'publish-taxonomies',
includeParamOnCompletion: true,
},
concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1,
},
undefined,
false,
);
}

/**
* Builds taxonomy publish payload: destination env UIDs from mapper, locales from taxonomy.locale, items: [{ uid }].
*/
serializePublishTaxonomies(apiOptions: ApiOptions): ApiOptions {
const job = apiOptions.apiData as { taxonomy?: Record<string, any> };
const taxonomy = job?.taxonomy;

if (!taxonomy?.publish_details?.length || !taxonomy?.locale) {
apiOptions.apiData = undefined;
return apiOptions;
}

const environments: string[] = [];
for (const pub of taxonomy.publish_details as any[]) {
const sourceEnvUid = pub?.environment;
if (!sourceEnvUid) continue;
const destUid = this.envUidMapper[String(sourceEnvUid)];
if (destUid && !environments.includes(destUid)) {
environments.push(destUid);
}
}

if (environments.length === 0) {
apiOptions.apiData = undefined;
return apiOptions;
}

const locales = [String(taxonomy.locale)];
apiOptions.apiData = {
environments,
locales,
items: [{ uid: taxonomy.uid }],
};

return apiOptions;
}

// --- Mapper output ---

/**
* create taxonomies success and fail in (mapper/taxonomies)
* create terms success and fail in (mapper/taxonomies/terms)
Expand Down Expand Up @@ -396,25 +610,36 @@ export default class ImportTaxonomies extends BaseClass {
}
}

private async analyzeTaxonomies(): Promise<[number]> {
// --- Analyze & prepare ---

private async analyzeTaxonomies(): Promise<[number, number]> {
return this.withLoadingSpinner('TAXONOMIES: Analyzing import data...', async () => {
log.debug('Checking for taxonomies folder existence', this.importConfig.context);

if (fileHelper.fileExistsSync(this.taxonomiesFolderPath)) {
log.debug(`Found taxonomies folder: ${this.taxonomiesFolderPath}`, this.importConfig.context);

this.taxonomies = fsUtil.readFile(join(this.taxonomiesFolderPath, 'taxonomies.json'), true) as Record<
string,
unknown
>;

const taxonomyCount = Object.keys(this.taxonomies || {}).length;
log.debug(`Loaded ${taxonomyCount} taxonomy items from file`, this.importConfig.context);
return [taxonomyCount];
} else {
if (!fileHelper.fileExistsSync(this.taxonomiesFolderPath)) {
log.info(`No Taxonomies Found! - '${this.taxonomiesFolderPath}'`, this.importConfig.context);
return [0];
return [0, 0];
}

log.debug(`Found taxonomies folder: ${this.taxonomiesFolderPath}`, this.importConfig.context);

this.taxonomies = fsUtil.readFile(join(this.taxonomiesFolderPath, 'taxonomies.json'), true) as Record<
string,
unknown
>;

this.isLocaleBasedStructure = this.detectAndScanLocaleStructure();

const taxonomyCount = Object.keys(this.taxonomies || {}).length;
const envMapper = this.readEnvUidMapperSync();
const publishJobCount = this.countPublishEligibleTaxonomies(envMapper);

log.debug(
`Loaded ${taxonomyCount} taxonomy items; ${publishJobCount} eligible for publish (mapped environments).`,
this.importConfig.context,
);

return [taxonomyCount, publishJobCount];
});
}

Expand Down
5 changes: 5 additions & 0 deletions packages/contentstack-import/src/utils/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ export const PROCESS_NAMES = {
CONTENT_TYPES_EXT_UPDATE: 'Content Types Ext Update',
WEBHOOKS_IMPORT: 'Webhooks Import',
TAXONOMIES_IMPORT: 'Taxonomies Import',
TAXONOMIES_PUBLISH: 'Taxonomies Publish',
PERSONALIZE_PROJECTS: 'Projects',
} as const;

Expand Down Expand Up @@ -267,6 +268,10 @@ export const PROCESS_STATUS = {
IMPORTING: 'Importing taxonomies...',
FAILED: 'Failed to import taxonomies.',
},
[PROCESS_NAMES.TAXONOMIES_PUBLISH]: {
PUBLISHING: 'Publishing taxonomies...',
FAILED: 'Failed to publish taxonomies.',
},
[PROCESS_NAMES.PERSONALIZE_PROJECTS]: {
IMPORTING: 'Importing personalization projects...',
FAILED: 'Failed to import personalization projects.',
Expand Down
Loading
Loading