From e64f9e747e85e90fb410c7e151dde8d944408797 Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Thu, 4 Jun 2026 17:49:14 +0530 Subject: [PATCH] feat: add pagination and test cases for AM package --- .talismanrc | 22 +- .../src/constants/index.ts | 2 + .../src/export/assets.ts | 4 +- .../src/export/base.ts | 10 +- .../src/export/spaces.ts | 2 + .../import-setup-asset-mappers.ts | 4 +- .../src/import/spaces.ts | 4 +- .../src/types/cs-assets-api.ts | 8 +- .../src/types/export-types.ts | 2 + .../src/utils/cs-assets-api-adapter.ts | 80 ++++- .../test/unit/export/base.test.ts | 30 ++ .../test/unit/import/asset-types.test.ts | 186 ++++++++++ .../test/unit/import/assets.test.ts | 239 +++++++++++++ .../test/unit/import/base.test.ts | 224 ++++++++++++ .../test/unit/import/fields.test.ts | 186 ++++++++++ .../test/unit/import/spaces.test.ts | 179 +++++++++- .../unit/utils/chunked-json-reader.test.ts | 143 ++++++++ .../unit/utils/cs-assets-api-adapter.test.ts | 336 +++++++++++++++++- .../contentstack-export/src/config/index.ts | 2 + .../src/export/modules/assets.ts | 2 + .../src/types/default-config.ts | 4 + 21 files changed, 1637 insertions(+), 32 deletions(-) create mode 100644 packages/contentstack-asset-management/test/unit/import/asset-types.test.ts create mode 100644 packages/contentstack-asset-management/test/unit/import/assets.test.ts create mode 100644 packages/contentstack-asset-management/test/unit/import/base.test.ts create mode 100644 packages/contentstack-asset-management/test/unit/import/fields.test.ts create mode 100644 packages/contentstack-asset-management/test/unit/utils/chunked-json-reader.test.ts diff --git a/.talismanrc b/.talismanrc index 2ec759fd6..00bf188a4 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,12 +1,14 @@ fileignoreconfig: - - filename: pnpm-lock.yaml - checksum: 2b0f2461ea1bb240a9210b9cf99dc403a756199712b7270f9792a590480451bd - - filename: packages/contentstack-import/test/unit/import/modules/base-class.test.ts - checksum: fe372852d5f2f3f57ef62c603406c30ccecdb444c17133ac0b21dda399b962c0 - - filename: packages/contentstack-bulk-operations/test/unit/commands/bulk-am-assets.test.ts - checksum: f8d21db7db0ca2eebe7cc40af0a59f02e74e1689efb6d50a1072dc5ca3e03e9b - - filename: packages/contentstack-export/src/export/modules/taxonomies.ts - checksum: b6d077118280bc88385405f504f921468a9fd490ac37a4a21f741be729fd1ca3 - - filename: packages/contentstack-export/test/unit/export/modules/taxonomies.test.ts - checksum: cab2ad4d897d23f04f988c1f018a9583ab7f0ee1815994d7bc9fce23dea70073 +- filename: packages/contentstack-asset-management/test/unit/import/asset-types.test.ts + checksum: fa4c08a47a52b8bd27353b9ba712e6128674315610d3372b53abe6af361ba0b7 +- filename: packages/contentstack-asset-management/test/unit/import/fields.test.ts + checksum: c2830111af2bf72c5a661fe2251a3520a865c42c3980289195a08d713703e4b1 +- filename: packages/contentstack-asset-management/test/unit/import/spaces.test.ts + checksum: f95e4b7ce04d4beb70c7f70c762f03541588c211a8c15ba68d426c98215a0769 +- filename: packages/contentstack-asset-management/test/unit/import/assets.test.ts + checksum: 37613b7f2b5812282eedf87e0b9f87efc8eebb5e3cf75afc5f0eaa3605be4b5d +- filename: packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts + checksum: 3d9d252ffa66a28380a015857fe7358fe7e2ba2efe90adaf715f9ff2a408450f +- filename: packages/contentstack-asset-management/test/unit/import/base.test.ts + checksum: c6ed81639052b5905f481e90d6c17e19b099b30916d4cf6bf7eabfe33fd15530 version: '1.0' diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index 9dbc66186..ec288b72a 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -4,6 +4,8 @@ export const FALLBACK_AM_CHUNK_FILE_SIZE_MB = 1; export const FALLBACK_AM_API_CONCURRENCY = 5; /** @deprecated Use FALLBACK_AM_API_CONCURRENCY */ export const DEFAULT_AM_API_CONCURRENCY = FALLBACK_AM_API_CONCURRENCY; +export const FALLBACK_AM_API_PAGE_SIZE = 100; +export const FALLBACK_AM_API_FETCH_CONCURRENCY = 5; /** Fallback strip lists when import options omit `fieldsImportInvalidKeys` / `assetTypesImportInvalidKeys`. */ export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [ diff --git a/packages/contentstack-asset-management/src/export/assets.ts b/packages/contentstack-asset-management/src/export/assets.ts index 6cc1129a3..c03acddfc 100644 --- a/packages/contentstack-asset-management/src/export/assets.ts +++ b/packages/contentstack-asset-management/src/export/assets.ts @@ -28,8 +28,8 @@ export default class ExportAssets extends CSAssetsExportAdapter { log.debug(`Fetching folders and assets for space ${workspace.space_uid}`, this.exportContext.context); const [folders, assetsData] = await Promise.all([ - this.getWorkspaceFolders(workspace.space_uid, workspace.uid), - this.getWorkspaceAssets(workspace.space_uid, workspace.uid), + this.getWorkspaceFolders(workspace.space_uid, workspace.uid, this.apiPageSize, this.apiFetchConcurrency), + this.getWorkspaceAssets(workspace.space_uid, workspace.uid, this.apiPageSize, this.apiFetchConcurrency), ]); const assetItems = getAssetItems(assetsData); diff --git a/packages/contentstack-asset-management/src/export/base.ts b/packages/contentstack-asset-management/src/export/base.ts index b9721685c..880fe8de3 100644 --- a/packages/contentstack-asset-management/src/export/base.ts +++ b/packages/contentstack-asset-management/src/export/base.ts @@ -5,7 +5,7 @@ import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack import type { CSAssetsAPIConfig } from '../types/cs-assets-api'; import type { ExportContext } from '../types/export-types'; import { CSAssetsAdapter } from '../utils/cs-assets-api-adapter'; -import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index'; +import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY, FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index'; export type { ExportContext }; @@ -82,6 +82,14 @@ export class CSAssetsExportAdapter extends CSAssetsAdapter { return this.exportContext.downloadAssetsConcurrency ?? this.apiConcurrency; } + protected get apiPageSize(): number { + return this.exportContext.pageSize ?? FALLBACK_AM_API_PAGE_SIZE; + } + + protected get apiFetchConcurrency(): number { + return this.exportContext.fetchConcurrency ?? FALLBACK_AM_API_FETCH_CONCURRENCY; + } + protected getAssetTypesDir(): string { return pResolve(this.exportContext.spacesRootPath, 'asset_types'); } diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts index 3a3459c3f..6d3b0ab13 100644 --- a/packages/contentstack-asset-management/src/export/spaces.ts +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -79,6 +79,8 @@ export class ExportSpaces { chunkFileSizeMb, apiConcurrency: this.options.apiConcurrency, downloadAssetsConcurrency: this.options.downloadAssetsConcurrency, + pageSize: this.options.pageSize, + fetchConcurrency: this.options.fetchConcurrency, }; const sharedFieldsDir = pResolve(spacesRootPath, 'fields'); diff --git a/packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts b/packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts index 5ea914e95..dca54810b 100644 --- a/packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts +++ b/packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts @@ -4,7 +4,7 @@ import { join, resolve } from 'node:path'; import { formatError, log } from '@contentstack/cli-utilities'; -import { IMPORT_ASSETS_MAPPER_FILES, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE, IMPORT_ASSETS_MAPPER_FILES, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; import type { CSAssetsAPIConfig, ImportContext } from '../types/cs-assets-api'; import type { AssetMapperImportSetupResult, RunAssetMapperImportSetupParams } from '../types/import-setup-asset-mapper'; import ImportAssets from '../import/assets'; @@ -25,7 +25,7 @@ export default class ImportSetupAssetMappers extends AssetManagementImportSetupA private async fetchExistingSpaceUidsInOrg(apiConfig: CSAssetsAPIConfig): Promise> { const adapter = new CSAssetsAdapter(apiConfig); await adapter.init(); - const { spaces } = await adapter.listSpaces(); + const { spaces } = await adapter.listSpaces(FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_API_FETCH_CONCURRENCY); const uids = new Set(); for (const s of spaces) { if (s.uid) { diff --git a/packages/contentstack-asset-management/src/import/spaces.ts b/packages/contentstack-asset-management/src/import/spaces.ts index 457c1f36d..faec9d13d 100644 --- a/packages/contentstack-asset-management/src/import/spaces.ts +++ b/packages/contentstack-asset-management/src/import/spaces.ts @@ -10,7 +10,7 @@ import type { ImportSpacesOptions, SpaceMapping, } from '../types/cs-assets-api'; -import { CS_ASSETS_MAIN_PROCESS_NAME, PROCESS_NAMES, getSpaceProcessName } from '../constants/index'; +import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_API_FETCH_CONCURRENCY, PROCESS_NAMES, getSpaceProcessName } from '../constants/index'; import { CSAssetsAdapter } from '../utils/cs-assets-api-adapter'; import ImportAssetTypes from './asset-types'; import ImportFields from './fields'; @@ -112,7 +112,7 @@ export class ImportSpaces { try { const adapterForList = new CSAssetsAdapter(apiConfig); await adapterForList.init(); - const { spaces } = await adapterForList.listSpaces(); + const { spaces } = await adapterForList.listSpaces(FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_API_FETCH_CONCURRENCY); for (const s of spaces) { if (s.uid) existingSpaceUids.add(s.uid); } diff --git a/packages/contentstack-asset-management/src/types/cs-assets-api.ts b/packages/contentstack-asset-management/src/types/cs-assets-api.ts index 96dbac1bd..2a47fbe32 100644 --- a/packages/contentstack-asset-management/src/types/cs-assets-api.ts +++ b/packages/contentstack-asset-management/src/types/cs-assets-api.ts @@ -167,11 +167,11 @@ export type SearchAssetsResponse = { export interface ICSAssetsAdapter { init(): Promise; - listSpaces(): Promise; + listSpaces(pageSize?: number, fetchConcurrency?: number): Promise; getSpace(spaceUid: string): Promise; getWorkspaceFields(spaceUid: string): Promise; - getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise; - getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise; + getWorkspaceAssets(spaceUid: string, workspaceUid?: string, pageSize?: number, fetchConcurrency?: number): Promise; + getWorkspaceFolders(spaceUid: string, workspaceUid?: string, pageSize?: number, fetchConcurrency?: number): Promise; getWorkspaceAssetTypes(spaceUid: string): Promise; searchAssets(params: SearchAssetsParams): Promise; bulkDeleteAssets( @@ -233,6 +233,8 @@ export type AssetManagementExportOptions = { * Max parallel asset file downloads per workspace. */ downloadAssetsConcurrency?: number; + pageSize?: number; + fetchConcurrency?: number; }; // --------------------------------------------------------------------------- diff --git a/packages/contentstack-asset-management/src/types/export-types.ts b/packages/contentstack-asset-management/src/types/export-types.ts index 865302a60..3e21682b4 100644 --- a/packages/contentstack-asset-management/src/types/export-types.ts +++ b/packages/contentstack-asset-management/src/types/export-types.ts @@ -5,6 +5,8 @@ export type ExportContext = { chunkFileSizeMb?: number; apiConcurrency?: number; downloadAssetsConcurrency?: number; + pageSize?: number; + fetchConcurrency?: number; }; /** diff --git a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts index f76fa4106..412ae542f 100644 --- a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts @@ -2,6 +2,9 @@ import { readFileSync } from 'node:fs'; import { basename } from 'node:path'; import { HttpClient, log, authenticationHandler, handleAndLogError } from '@contentstack/cli-utilities'; +import { chunkArray } from './concurrent-batch'; +import { FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE } from '../constants/index'; + import type { CSAssetsAPIConfig, AssetTypesResponse, @@ -189,11 +192,17 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { } } - async listSpaces(): Promise { + async listSpaces(pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { log.debug('Fetching all spaces in org', this.config.context); - const result = await this.getSpaceLevel('', '/api/spaces', {}); - log.debug(`Fetched ${result?.count ?? result?.spaces?.length ?? '?'} space(s)`, this.config.context); - return result; + const items = await this.fetchAllPages( + '', + '/api/spaces', + 'spaces', + pageSize, + fetchConcurrency, + ); + log.debug(`Fetched ${items.length} space(s)`, this.config.context); + return { spaces: items as Space[], count: items.length }; } async getSpace(spaceUid: string): Promise { @@ -230,22 +239,73 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { return result; } - async getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise { - return this.getWorkspaceCollection( + + /** + * Fetch all pages of a paginated collection by issuing the first request to determine + * the total count, then issuing remaining page requests with controlled concurrency. + */ + private async fetchAllPages( + spaceUid: string, + path: string, + itemsKey: string, + pageSize: number, + concurrency: number, + baseParams: Record = {}, + ): Promise { + const first = await this.getSpaceLevel>(spaceUid, path, { + ...baseParams, limit: String(pageSize), skip: '0', + }); + + const total: number = Number(first?.count ?? 0); + const firstItems: unknown[] = Array.isArray(first?.[itemsKey]) ? (first[itemsKey] as unknown[]) : []; + if (firstItems.length >= total) return firstItems; + + const skips = Array.from( + { length: Math.ceil(total / pageSize) - 1 }, + (_, i) => (i + 1) * pageSize, + ); + + const skipBatches = chunkArray(skips, concurrency); + const rest: unknown[] = []; + + for (const batch of skipBatches) { + const pages = await Promise.all( + batch.map((skip) => + this.getSpaceLevel>(spaceUid, path, { + ...baseParams, limit: String(pageSize), skip: String(skip), + }).then((r) => (Array.isArray(r?.[itemsKey]) ? (r[itemsKey] as unknown[]) : [])), + ), + ); + rest.push(...pages.flat()); + } + + return [...firstItems, ...rest]; + } + + async getWorkspaceAssets(spaceUid: string, workspaceUid?: string, pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { + const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; + const items = await this.fetchAllPages( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, 'assets', - workspaceUid ? { workspace: workspaceUid } : {}, + pageSize, + fetchConcurrency, + baseParams, ); + return { assets: items, count: items.length }; } - async getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise { - return this.getWorkspaceCollection( + async getWorkspaceFolders(spaceUid: string, workspaceUid?: string, pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { + const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; + const items = await this.fetchAllPages( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/folders`, 'folders', - workspaceUid ? { workspace: workspaceUid } : {}, + pageSize, + fetchConcurrency, + baseParams, ); + return { folders: items, count: items.length }; } async getWorkspaceAssetTypes(spaceUid: string): Promise { diff --git a/packages/contentstack-asset-management/test/unit/export/base.test.ts b/packages/contentstack-asset-management/test/unit/export/base.test.ts index 08993eddf..22f41d67e 100644 --- a/packages/contentstack-asset-management/test/unit/export/base.test.ts +++ b/packages/contentstack-asset-management/test/unit/export/base.test.ts @@ -35,6 +35,12 @@ class TestAdapter extends CSAssetsExportAdapter { public get spacesRootPathPublic() { return this.spacesRootPath; } + public get apiPageSizePublic() { + return this.apiPageSize; + } + public get apiFetchConcurrencyPublic() { + return this.apiFetchConcurrency; + } } describe('CSAssetsExportAdapter (base)', () => { @@ -192,6 +198,30 @@ describe('CSAssetsExportAdapter (base)', () => { }); }); + describe('apiPageSize', () => { + it('should return FALLBACK_AM_API_PAGE_SIZE (100) when pageSize is not set in exportContext', () => { + const adapter = new TestAdapter(apiConfig, exportContext); + expect(adapter.apiPageSizePublic).to.equal(100); + }); + + it('should return the configured pageSize when set in exportContext', () => { + const adapter = new TestAdapter(apiConfig, { ...exportContext, pageSize: 50 }); + expect(adapter.apiPageSizePublic).to.equal(50); + }); + }); + + describe('apiFetchConcurrency', () => { + it('should return FALLBACK_AM_API_FETCH_CONCURRENCY (5) when fetchConcurrency is not set', () => { + const adapter = new TestAdapter(apiConfig, exportContext); + expect(adapter.apiFetchConcurrencyPublic).to.equal(5); + }); + + it('should return the configured fetchConcurrency when set in exportContext', () => { + const adapter = new TestAdapter(apiConfig, { ...exportContext, fetchConcurrency: 10 }); + expect(adapter.apiFetchConcurrencyPublic).to.equal(10); + }); + }); + describe('writeItemsToChunkedJson', () => { it('should write {} to an empty file when items array is empty', async () => { const os = require('node:os'); diff --git a/packages/contentstack-asset-management/test/unit/import/asset-types.test.ts b/packages/contentstack-asset-management/test/unit/import/asset-types.test.ts new file mode 100644 index 000000000..c3252526d --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/import/asset-types.test.ts @@ -0,0 +1,186 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { FsUtility } from '@contentstack/cli-utilities'; + +import ImportAssetTypes from '../../../src/import/asset-types'; +import { CSAssetsImportAdapter } from '../../../src/import/base'; +import type { CSAssetsAPIConfig, ImportContext } from '../../../src/types/cs-assets-api'; + +describe('ImportAssetTypes', () => { + const apiConfig: CSAssetsAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + const importContext: ImportContext = { + spacesRootPath: '/tmp/import/spaces', + apiKey: 'api-key-1', + host: 'https://api.contentstack.io/v3', + org_uid: 'org-1', + }; + + let tickStub: sinon.SinonStub; + + beforeEach(() => { + sinon.stub(CSAssetsImportAdapter.prototype, 'init' as any).resolves(); + tickStub = sinon.stub(CSAssetsImportAdapter.prototype, 'tick' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'updateStatus' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'getAssetTypesDir' as any).returns('/tmp/import/spaces/asset_types'); + }); + + afterEach(() => sinon.restore()); + + const stubExistingAssetTypes = (assetTypes: any[]) => { + sinon.stub(CSAssetsImportAdapter.prototype, 'getWorkspaceAssetTypes' as any) + .resolves({ asset_types: assetTypes }); + }; + + const stubChunks = (records: Record[]) => { + const indexer = records.length > 0 ? { '0': true } : {}; + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => indexer); + if (records.length > 0) { + const chunk = Object.fromEntries(records.map((r) => [(r.uid as string), r])); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves(chunk), + })); + } + }; + + describe('when index file does not exist', () => { + it('ticks once and returns without calling createAssetType', async () => { + sinon.stub(require('node:fs'), 'existsSync').returns(false); + stubExistingAssetTypes([]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.callCount).to.equal(1); + expect(tickStub.firstCall.args[0]).to.equal(true); + }); + }); + + describe('when asset types exist in the export', () => { + beforeEach(() => { + sinon.stub(require('node:fs'), 'existsSync').returns(true); + }); + + it('creates a new asset type that does not exist in the target org', async () => { + const newType = { uid: 'type-new', label: 'New Type' }; + stubExistingAssetTypes([]); + stubChunks([newType]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(1); + const payload = createStub.firstCall.args[0]; + expect(payload.label).to.equal('New Type'); + }); + + it('skips asset types with is_system=true', async () => { + stubExistingAssetTypes([]); + stubChunks([{ uid: 'sys-type', is_system: true, label: 'System Type' }]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + const tickArgs = tickStub.lastCall.args[1] as string; + expect(tickArgs).to.include('skipped'); + }); + + it('skips (no create) when uid already exists in target with matching definition', async () => { + const existing = { uid: 'type-1', label: 'Type One', created_at: '2024-01-01' }; + const exported = { uid: 'type-1', label: 'Type One', created_at: '2024-01-01' }; + stubExistingAssetTypes([existing]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('skips (no create) when uid exists with a different definition in target', async () => { + const existing = { uid: 'type-1', label: 'Old Label' }; + const exported = { uid: 'type-1', label: 'New Label' }; + stubExistingAssetTypes([existing]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('strips invalid keys (created_at, updated_at, is_system) from the POST payload', async () => { + const exported = { + uid: 'type-clean', + label: 'Clean Type', + created_at: '2024-01-01', + updated_at: '2024-06-01', + is_system: false, + created_by: 'user-1', + updated_by: 'user-2', + }; + stubExistingAssetTypes([]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + const payload = createStub.firstCall.args[0]; + expect(payload).to.not.have.property('created_at'); + expect(payload).to.not.have.property('updated_at'); + expect(payload).to.not.have.property('is_system'); + expect(payload).to.not.have.property('created_by'); + expect(payload).to.not.have.property('updated_by'); + expect(payload.label).to.equal('Clean Type'); + }); + + it('handles createAssetType failure: increments failure count, final tick reflects failure', async () => { + stubExistingAssetTypes([]); + stubChunks([{ uid: 'type-bad', label: 'Bad Type' }]); + sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).rejects(new Error('API error')); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + const lastTickArgs = tickStub.lastCall.args; + expect(lastTickArgs[0]).to.equal(false); + expect(lastTickArgs[1]).to.include('1 failed'); + }); + + it('handles getWorkspaceAssetTypes failure: proceeds as if no existing types', async () => { + sinon.stub(CSAssetsImportAdapter.prototype, 'getWorkspaceAssetTypes' as any) + .rejects(new Error('API unavailable')); + stubChunks([{ uid: 'type-new', label: 'New Type' }]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(1); + }); + + it('final tick is success=true when all creates succeed', async () => { + stubExistingAssetTypes([]); + stubChunks([{ uid: 'type-ok', label: 'OK Type' }]); + sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + const lastTickArgs = tickStub.lastCall.args; + expect(lastTickArgs[0]).to.equal(true); + expect(lastTickArgs[1]).to.include('1 created'); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/import/assets.test.ts b/packages/contentstack-asset-management/test/unit/import/assets.test.ts new file mode 100644 index 000000000..d60a758ae --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/import/assets.test.ts @@ -0,0 +1,239 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import * as os from 'os'; +import * as path from 'path'; +import * as fsReal from 'fs'; +import { FsUtility } from '@contentstack/cli-utilities'; + +import ImportAssets from '../../../src/import/assets'; +import { CSAssetsImportAdapter } from '../../../src/import/base'; +import type { CSAssetsAPIConfig, ImportContext } from '../../../src/types/cs-assets-api'; + +describe('ImportAssets', () => { + const apiConfig: CSAssetsAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + const importContext: ImportContext = { + spacesRootPath: '/tmp/import/spaces', + apiKey: 'api-key-1', + host: 'https://api.contentstack.io/v3', + org_uid: 'org-1', + }; + + let tickStub: sinon.SinonStub; + + beforeEach(() => { + sinon.stub(CSAssetsImportAdapter.prototype, 'init' as any).resolves(); + tickStub = sinon.stub(CSAssetsImportAdapter.prototype, 'tick' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'updateStatus' as any); + }); + + afterEach(() => sinon.restore()); + + const makeSpaceDir = () => { + const dir = path.join(os.tmpdir(), `am-test-${Date.now()}`); + fsReal.mkdirSync(path.join(dir, 'assets'), { recursive: true }); + return dir; + }; + + const stubAssetChunks = (assets: Record[]) => { + const indexer = assets.length > 0 ? { '0': true } : {}; + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => indexer); + if (assets.length > 0) { + const chunk = Object.fromEntries(assets.map((a) => [(a.uid as string), a])); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves(chunk), + })); + } + sinon.stub(FsUtility.prototype, 'getPlainMeta').returns( + assets.length > 0 ? { 'chunk0': assets.map((a) => a.uid) } : {}, + ); + }; + + describe('buildIdentityMappersFromExport', () => { + it('returns empty maps when no assets.json index exists in spaceDir', async () => { + const spaceDir = makeSpaceDir(); + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.buildIdentityMappersFromExport(spaceDir); + + expect(result.uidMap).to.deep.equal({}); + expect(result.urlMap).to.deep.equal({}); + }); + + it('builds identity uid and url maps from chunked assets', async () => { + const spaceDir = makeSpaceDir(); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([ + { uid: 'asset-1', url: 'https://cdn.example.com/asset-1.png' }, + { uid: 'asset-2', url: 'https://cdn.example.com/asset-2.png' }, + ]); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.buildIdentityMappersFromExport(spaceDir); + + expect(result.uidMap).to.deep.equal({ 'asset-1': 'asset-1', 'asset-2': 'asset-2' }); + expect(result.urlMap).to.deep.equal({ + 'https://cdn.example.com/asset-1.png': 'https://cdn.example.com/asset-1.png', + 'https://cdn.example.com/asset-2.png': 'https://cdn.example.com/asset-2.png', + }); + }); + + it('handles assets with missing uid gracefully: only url is added to urlMap', async () => { + const spaceDir = makeSpaceDir(); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => ({ '0': true })); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves({ + 'asset-no-url': { uid: 'asset-no-url' }, + }), + })); + sinon.stub(FsUtility.prototype, 'getPlainMeta').returns({}); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.buildIdentityMappersFromExport(spaceDir); + + expect(result.uidMap).to.have.key('asset-no-url'); + expect(result.urlMap).to.deep.equal({}); + }); + }); + + describe('start', () => { + it('returns empty maps and ticks once for an empty space (no folders, no assets)', async () => { + const spaceDir = makeSpaceDir(); + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.start('new-space-uid', spaceDir); + + expect(result.uidMap).to.deep.equal({}); + expect(result.urlMap).to.deep.equal({}); + expect(tickStub.callCount).to.equal(1); + expect(tickStub.firstCall.args[0]).to.equal(true); + }); + + it('creates root-level folders and maps their uids', async () => { + const spaceDir = makeSpaceDir(); + const folders = [{ uid: 'folder-old', title: 'My Folder' }]; + fsReal.writeFileSync( + path.join(spaceDir, 'assets', 'folders.json'), + JSON.stringify({ folders }), + ); + stubAssetChunks([]); + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => ({})); + const createFolderStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createFolder' as any) + .resolves({ folder: { uid: 'folder-new' } }); + + const importer = new ImportAssets(apiConfig, importContext); + await importer.start('space-uid', spaceDir); + + expect(createFolderStub.callCount).to.equal(1); + const createArgs = createFolderStub.firstCall.args; + expect(createArgs[0]).to.equal('space-uid'); + expect(createArgs[1].title).to.equal('My Folder'); + }); + + it('imports nested folders in multi-pass: child waits for parent to be created', async () => { + const spaceDir = makeSpaceDir(); + const folders = [ + { uid: 'child-folder', title: 'Child', parent_uid: 'parent-folder' }, + { uid: 'parent-folder', title: 'Parent' }, + ]; + fsReal.writeFileSync( + path.join(spaceDir, 'assets', 'folders.json'), + JSON.stringify({ folders }), + ); + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => ({})); + let callOrder: string[] = []; + const createFolderStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createFolder' as any) + .callsFake(async (_spaceUid: string, payload: any) => { + callOrder.push(payload.title); + return { folder: { uid: `new-${payload.title.toLowerCase()}` } }; + }); + + const importer = new ImportAssets(apiConfig, importContext); + await importer.start('space-uid', spaceDir); + + expect(createFolderStub.callCount).to.equal(2); + expect(callOrder[0]).to.equal('Parent'); + expect(callOrder[1]).to.equal('Child'); + }); + + it('uploads assets: calls uploadAsset and builds uidMap and urlMap', async () => { + const spaceDir = makeSpaceDir(); + const assetUid = 'asset-old-uid'; + const assetFilename = 'photo.png'; + fsReal.mkdirSync(path.join(spaceDir, 'assets', 'files', assetUid), { recursive: true }); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'files', assetUid, assetFilename), 'fake-content'); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([{ uid: assetUid, url: 'https://old-cdn.com/photo.png', filename: assetFilename }]); + + const uploadStub = sinon.stub(CSAssetsImportAdapter.prototype, 'uploadAsset' as any) + .resolves({ asset: { uid: 'asset-new-uid', url: 'https://new-cdn.com/photo.png' } }); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.start('space-uid', spaceDir); + + expect(uploadStub.callCount).to.equal(1); + expect(result.uidMap[assetUid]).to.equal('asset-new-uid'); + expect(result.urlMap['https://old-cdn.com/photo.png']).to.equal('https://new-cdn.com/photo.png'); + }); + + it('skips an asset and ticks false when the file is not found on disk', async () => { + const spaceDir = makeSpaceDir(); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([{ uid: 'missing-asset', url: 'https://cdn.com/x.png', filename: 'x.png' }]); + const uploadStub = sinon.stub(CSAssetsImportAdapter.prototype, 'uploadAsset' as any).resolves(); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.start('space-uid', spaceDir); + + expect(uploadStub.callCount).to.equal(0); + expect(result.uidMap).to.deep.equal({}); + const failTick = tickStub.getCalls().find((c) => c.args[0] === false && c.args[2]); + expect(failTick).to.exist; + }); + + it('handles uploadAsset failure gracefully: continues, ticks false, omits from maps', async () => { + const spaceDir = makeSpaceDir(); + const assetUid = 'asset-fail'; + const filename = 'fail.png'; + fsReal.mkdirSync(path.join(spaceDir, 'assets', 'files', assetUid), { recursive: true }); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'files', assetUid, filename), 'data'); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([{ uid: assetUid, url: 'https://cdn.com/fail.png', filename }]); + + sinon.stub(CSAssetsImportAdapter.prototype, 'uploadAsset' as any).rejects(new Error('upload failed')); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.start('space-uid', spaceDir); + + expect(result.uidMap).to.deep.equal({}); + const failTick = tickStub.getCalls().find((c) => c.args[0] === false); + expect(failTick).to.exist; + }); + + it('maps asset parent_uid to the new folder uid when parent was imported', async () => { + const spaceDir = makeSpaceDir(); + fsReal.writeFileSync( + path.join(spaceDir, 'assets', 'folders.json'), + JSON.stringify({ folders: [{ uid: 'old-folder', title: 'Folder A' }] }), + ); + const assetUid = 'asset-in-folder'; + const filename = 'file.png'; + fsReal.mkdirSync(path.join(spaceDir, 'assets', 'files', assetUid), { recursive: true }); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'files', assetUid, filename), 'data'); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([{ uid: assetUid, parent_uid: 'old-folder', filename }]); + + sinon.stub(CSAssetsImportAdapter.prototype, 'createFolder' as any) + .resolves({ folder: { uid: 'new-folder-uid' } }); + const uploadStub = sinon.stub(CSAssetsImportAdapter.prototype, 'uploadAsset' as any) + .resolves({ asset: { uid: 'new-asset-uid' } }); + + const importer = new ImportAssets(apiConfig, importContext); + await importer.start('space-uid', spaceDir); + + const uploadArgs = uploadStub.firstCall.args; + expect(uploadArgs[2].parent_uid).to.equal('new-folder-uid'); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/import/base.test.ts b/packages/contentstack-asset-management/test/unit/import/base.test.ts new file mode 100644 index 000000000..ddbab234e --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/import/base.test.ts @@ -0,0 +1,224 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import { CSAssetsImportAdapter } from '../../../src/import/base'; +import type { CSAssetsAPIConfig, ImportContext } from '../../../src/types/cs-assets-api'; + +class TestImportAdapter extends CSAssetsImportAdapter { + public callCreateNestedProgress(name: string) { return this.createNestedProgress(name); } + public callTick(success: boolean, name: string, error: string | null, processName?: string) { + return this.tick(success, name, error, processName); + } + public callUpdateStatus(msg: string, processName?: string) { return this.updateStatus(msg, processName); } + public callCompleteProcess(name: string, success: boolean) { return this.completeProcess(name, success); } + public get progressOrParentPublic() { return this.progressOrParent; } + public get spacesRootPathPublic() { return this.spacesRootPath; } + public get apiConcurrencyPublic() { return this.apiConcurrency; } + public get uploadBatchPublic() { return this.uploadAssetsBatchConcurrency; } + public get foldersBatchPublic() { return this.importFoldersBatchConcurrency; } + public getAssetTypesDirPublic() { return this.getAssetTypesDir(); } + public getFieldsDirPublic() { return this.getFieldsDir(); } +} + +describe('CSAssetsImportAdapter (base)', () => { + const apiConfig: CSAssetsAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + const importContext: ImportContext = { + spacesRootPath: '/tmp/import/spaces', + apiKey: 'api-key-1', + host: 'https://api.contentstack.io/v3', + org_uid: 'org-1', + }; + + beforeEach(() => { + sinon.stub(CSAssetsImportAdapter.prototype, 'init' as any).resolves(); + }); + afterEach(() => sinon.restore()); + + describe('setParentProgressManager / progressOrParent', () => { + it('returns null when no progress manager is set', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(adapter.progressOrParentPublic).to.be.null; + }); + + it('returns the parent manager after setParentProgressManager', () => { + const fakeParent = { tick: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + expect(adapter.progressOrParentPublic).to.equal(fakeParent); + }); + + it('returns progressManager when parentProgressManager is not set', () => { + sinon.stub(configHandler, 'get').returns({}); + const fakeProgress = { tick: sinon.stub() } as any; + sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.callCreateNestedProgress('test-module'); + expect(adapter.progressOrParentPublic).to.equal(fakeProgress); + }); + }); + + describe('setProcessName', () => { + it('overrides the processName used in tick calls', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.setProcessName('custom-process'); + adapter.callTick(true, 'item', null); + expect(fakeParent.tick.firstCall.args[3]).to.equal('custom-process'); + }); + }); + + describe('createNestedProgress', () => { + it('creates a CLIProgressManager when no parent is set', () => { + sinon.stub(configHandler, 'get').returns({ showConsoleLogs: true }); + const fakeProgress = { tick: sinon.stub() } as any; + const createNestedStub = sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + const adapter = new TestImportAdapter(apiConfig, importContext); + const result = adapter.callCreateNestedProgress('my-module'); + expect(createNestedStub.firstCall.args[0]).to.equal('my-module'); + expect(result).to.equal(fakeProgress); + }); + + it('returns parent directly when parentProgressManager is set', () => { + const fakeParent = { tick: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + const result = adapter.callCreateNestedProgress('ignored'); + expect(result).to.equal(fakeParent); + }); + + it('defaults showConsoleLogs to false when log config is missing', () => { + sinon.stub(configHandler, 'get').returns(null); + const fakeProgress = { tick: sinon.stub() } as any; + const createNestedStub = sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.callCreateNestedProgress('test'); + expect(createNestedStub.firstCall.args[1]).to.be.false; + }); + }); + + describe('tick', () => { + it('forwards success, itemName, error to progress manager tick', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.callTick(true, 'my-item', 'some-error'); + expect(fakeParent.tick.firstCall.args[0]).to.equal(true); + expect(fakeParent.tick.firstCall.args[1]).to.equal('my-item'); + expect(fakeParent.tick.firstCall.args[2]).to.equal('some-error'); + }); + + it('uses explicit processName override when provided', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.callTick(false, 'item', null, 'override-process'); + expect(fakeParent.tick.firstCall.args[3]).to.equal('override-process'); + }); + + it('does not throw when progressOrParent is null', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(() => adapter.callTick(true, 'item', null)).to.not.throw(); + }); + }); + + describe('updateStatus', () => { + it('forwards status message to progress manager', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.callUpdateStatus('Importing...'); + expect(fakeParent.updateStatus.firstCall.args[0]).to.equal('Importing...'); + }); + + it('does not throw when progressOrParent is null', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(() => adapter.callUpdateStatus('msg')).to.not.throw(); + }); + }); + + describe('completeProcess', () => { + it('calls completeProcess on progressManager when no parent is set', () => { + sinon.stub(configHandler, 'get').returns({}); + const fakeProgress = { tick: sinon.stub(), completeProcess: sinon.stub() } as any; + sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.callCreateNestedProgress('test'); + adapter.callCompleteProcess('test-process', true); + expect(fakeProgress.completeProcess.firstCall.args).to.deep.equal(['test-process', true]); + }); + + it('does NOT call completeProcess when parentProgressManager is set', () => { + const fakeParent = { tick: sinon.stub(), completeProcess: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.callCompleteProcess('test-process', true); + expect(fakeParent.completeProcess.callCount).to.equal(0); + }); + }); + + describe('path and concurrency getters', () => { + it('spacesRootPath returns the value from importContext', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(adapter.spacesRootPathPublic).to.equal('/tmp/import/spaces'); + }); + + it('apiConcurrency defaults to FALLBACK_AM_API_CONCURRENCY (5) when not set', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(adapter.apiConcurrencyPublic).to.equal(5); + }); + + it('apiConcurrency uses importContext.apiConcurrency when set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, apiConcurrency: 10 }); + expect(adapter.apiConcurrencyPublic).to.equal(10); + }); + + it('uploadAssetsBatchConcurrency falls back to apiConcurrency when uploadAssetsConcurrency not set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, apiConcurrency: 8 }); + expect(adapter.uploadBatchPublic).to.equal(8); + }); + + it('uploadAssetsBatchConcurrency uses uploadAssetsConcurrency when set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, uploadAssetsConcurrency: 3 }); + expect(adapter.uploadBatchPublic).to.equal(3); + }); + + it('importFoldersBatchConcurrency falls back to apiConcurrency when not set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, apiConcurrency: 6 }); + expect(adapter.foldersBatchPublic).to.equal(6); + }); + + it('importFoldersBatchConcurrency uses importFoldersConcurrency when set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, importFoldersConcurrency: 2 }); + expect(adapter.foldersBatchPublic).to.equal(2); + }); + + it('getAssetTypesDir defaults to spacesRootPath/asset_types', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + const expected = require('node:path').join('/tmp/import/spaces', 'asset_types'); + expect(adapter.getAssetTypesDirPublic()).to.equal(expected); + }); + + it('getAssetTypesDir uses custom assetTypesDir when set in importContext', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, assetTypesDir: 'custom_at' }); + const expected = require('node:path').join('/tmp/import/spaces', 'custom_at'); + expect(adapter.getAssetTypesDirPublic()).to.equal(expected); + }); + + it('getFieldsDir defaults to spacesRootPath/fields', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + const expected = require('node:path').join('/tmp/import/spaces', 'fields'); + expect(adapter.getFieldsDirPublic()).to.equal(expected); + }); + + it('getFieldsDir uses custom fieldsDir when set in importContext', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, fieldsDir: 'custom_fields' }); + const expected = require('node:path').join('/tmp/import/spaces', 'custom_fields'); + expect(adapter.getFieldsDirPublic()).to.equal(expected); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/import/fields.test.ts b/packages/contentstack-asset-management/test/unit/import/fields.test.ts new file mode 100644 index 000000000..c45ef17c8 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/import/fields.test.ts @@ -0,0 +1,186 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { FsUtility } from '@contentstack/cli-utilities'; + +import ImportFields from '../../../src/import/fields'; +import { CSAssetsImportAdapter } from '../../../src/import/base'; +import type { CSAssetsAPIConfig, ImportContext } from '../../../src/types/cs-assets-api'; + +describe('ImportFields', () => { + const apiConfig: CSAssetsAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + const importContext: ImportContext = { + spacesRootPath: '/tmp/import/spaces', + apiKey: 'api-key-1', + host: 'https://api.contentstack.io/v3', + org_uid: 'org-1', + }; + + let tickStub: sinon.SinonStub; + + beforeEach(() => { + sinon.stub(CSAssetsImportAdapter.prototype, 'init' as any).resolves(); + tickStub = sinon.stub(CSAssetsImportAdapter.prototype, 'tick' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'updateStatus' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'getFieldsDir' as any).returns('/tmp/import/spaces/fields'); + }); + + afterEach(() => sinon.restore()); + + const stubExistingFields = (fields: any[]) => { + sinon.stub(CSAssetsImportAdapter.prototype, 'getWorkspaceFields' as any) + .resolves({ fields }); + }; + + const stubChunks = (records: Record[]) => { + const indexer = records.length > 0 ? { '0': true } : {}; + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => indexer); + if (records.length > 0) { + const chunk = Object.fromEntries(records.map((r) => [(r.uid as string), r])); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves(chunk), + })); + } + }; + + describe('when index file does not exist', () => { + it('ticks once and returns without calling createField', async () => { + sinon.stub(require('node:fs'), 'existsSync').returns(false); + stubExistingFields([]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.callCount).to.equal(1); + expect(tickStub.firstCall.args[0]).to.equal(true); + }); + }); + + describe('when fields exist in the export', () => { + beforeEach(() => { + sinon.stub(require('node:fs'), 'existsSync').returns(true); + }); + + it('creates a new field that does not exist in the target org', async () => { + const newField = { uid: 'field-new', label: 'New Field', type: 'text' }; + stubExistingFields([]); + stubChunks([newField]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(1); + const payload = createStub.firstCall.args[0]; + expect(payload.label).to.equal('New Field'); + }); + + it('skips fields with is_system=true', async () => { + stubExistingFields([]); + stubChunks([{ uid: 'sys-field', is_system: true, label: 'System Field' }]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('silently skips (no create) when uid exists with matching definition after stripping invalid keys', async () => { + const existing = { uid: 'field-1', label: 'Field One', created_at: '2024-01-01', asset_types_count: 3 }; + const exported = { uid: 'field-1', label: 'Field One', created_at: '2024-01-01', asset_types_count: 5 }; + stubExistingFields([existing]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('skips (no create) when uid exists with a different definition', async () => { + const existing = { uid: 'field-1', label: 'Old Label', type: 'text' }; + const exported = { uid: 'field-1', label: 'New Label', type: 'text' }; + stubExistingFields([existing]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('strips invalid keys (created_at, updated_at, is_system, asset_types_count) from POST payload', async () => { + const exported = { + uid: 'field-clean', + label: 'Clean Field', + created_at: '2024-01-01', + updated_at: '2024-06-01', + is_system: false, + asset_types_count: 10, + created_by: 'user-1', + updated_by: 'user-2', + }; + stubExistingFields([]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + const payload = createStub.firstCall.args[0]; + expect(payload).to.not.have.property('created_at'); + expect(payload).to.not.have.property('updated_at'); + expect(payload).to.not.have.property('is_system'); + expect(payload).to.not.have.property('asset_types_count'); + expect(payload.label).to.equal('Clean Field'); + }); + + it('handles createField failure: final tick reflects failure count', async () => { + stubExistingFields([]); + stubChunks([{ uid: 'field-bad', label: 'Bad Field' }]); + sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).rejects(new Error('API error')); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + const lastTickArgs = tickStub.lastCall.args; + expect(lastTickArgs[0]).to.equal(false); + expect(lastTickArgs[1]).to.include('1 failed'); + }); + + it('handles getWorkspaceFields failure: proceeds as if no existing fields', async () => { + sinon.stub(CSAssetsImportAdapter.prototype, 'getWorkspaceFields' as any) + .rejects(new Error('API unavailable')); + stubChunks([{ uid: 'field-new', label: 'New Field' }]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(1); + }); + + it('final tick is success=true when all creates succeed and none fail', async () => { + stubExistingFields([]); + stubChunks([{ uid: 'field-ok', label: 'OK Field' }]); + sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + const lastTickArgs = tickStub.lastCall.args; + expect(lastTickArgs[0]).to.equal(true); + expect(lastTickArgs[1]).to.include('1 created'); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/import/spaces.test.ts b/packages/contentstack-asset-management/test/unit/import/spaces.test.ts index 5cff18b66..d77f7e943 100644 --- a/packages/contentstack-asset-management/test/unit/import/spaces.test.ts +++ b/packages/contentstack-asset-management/test/unit/import/spaces.test.ts @@ -30,7 +30,10 @@ describe('ImportSpaces', () => { }; beforeEach(() => { - sinon.stub(configHandler, 'get').returns({ showConsoleLogs: false }); + sinon.stub(configHandler, 'get').callsFake((key: string) => { + if (key === 'log') return { showConsoleLogs: false }; + return undefined; + }); sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress as any); // init and listSpaces live on AssetManagementAdapter (the common base). // Stubbing the base once covers both the adapter used for listSpaces and ImportWorkspace. @@ -173,4 +176,178 @@ describe('ImportSpaces', () => { expect(result.spaceUidMap).to.deep.equal({}); }); }); + + describe('bootstrap failure', () => { + it('should mark all space rows as failed and re-throw when ImportFields throws', async () => { + stubSpaceDirs(['am-space-1']); + sinon.stub(ImportWorkspace.prototype, 'start').resolves({ + oldSpaceUid: 'am-space-1', newSpaceUid: 'new-space', workspaceUid: 'main', + isDefault: false, uidMap: {}, urlMap: {}, + }); + (ImportFields.prototype.start as sinon.SinonStub).rejects(new Error('fields-bootstrap-error')); + + const importer = new ImportSpaces(baseOptions); + try { + await importer.start(); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.equal('fields-bootstrap-error'); + } + + const completeCalls = fakeProgress.completeProcess.getCalls().map((c) => c.args); + expect(completeCalls).to.deep.include([PROCESS_NAMES.AM_IMPORT_FIELDS, false]); + }); + + it('should mark all space rows as failed and re-throw when ImportAssetTypes throws', async () => { + stubSpaceDirs(['am-space-1']); + (ImportAssetTypes.prototype.start as sinon.SinonStub).rejects(new Error('at-bootstrap-error')); + + const importer = new ImportSpaces(baseOptions); + try { + await importer.start(); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.equal('at-bootstrap-error'); + } + + const completeCalls = fakeProgress.completeProcess.getCalls().map((c) => c.args); + expect(completeCalls).to.deep.include([PROCESS_NAMES.AM_IMPORT_ASSET_TYPES, false]); + }); + }); + + describe('per-space failure resilience', () => { + it('should continue importing remaining spaces when one space fails', async () => { + stubSpaceDirs(['am-space-1', 'am-space-2']); + const startStub = sinon.stub(ImportWorkspace.prototype, 'start'); + startStub.onFirstCall().rejects(new Error('space-1-error')); + startStub.onSecondCall().resolves({ + oldSpaceUid: 'am-space-2', newSpaceUid: 'new-space-2', workspaceUid: 'main', + isDefault: false, uidMap: {}, urlMap: {}, + }); + + const importer = new ImportSpaces(baseOptions); + const result = await importer.start(); + + expect(startStub.callCount).to.equal(2); + expect(result.spaceMappings).to.have.lengthOf(1); + expect(result.spaceMappings[0].oldSpaceUid).to.equal('am-space-2'); + }); + }); + + describe('backupDir mapper file writing', () => { + it('should write uid, url, and space-uid mapping files when backupDir is set', async () => { + const os = require('node:os'); + const path = require('node:path'); + const fsReal = require('node:fs'); + const tmpDir = path.join(os.tmpdir(), `import-spaces-backup-${Date.now()}`); + fsReal.mkdirSync(tmpDir, { recursive: true }); + + stubSpaceDirs(['am-space-1']); + sinon.stub(ImportWorkspace.prototype, 'start').resolves({ + oldSpaceUid: 'am-space-1', newSpaceUid: 'new-space-1', workspaceUid: 'main', + isDefault: false, + uidMap: { 'old-uid': 'new-uid' }, + urlMap: { 'old-url': 'new-url' }, + }); + + const options: ImportSpacesOptions = { ...baseOptions, backupDir: tmpDir }; + const importer = new ImportSpaces(options); + await importer.start(); + + const mapperDir = path.join(tmpDir, 'mapper', 'assets'); + expect(fsReal.existsSync(path.join(mapperDir, 'uid-mapping.json'))).to.be.true; + expect(fsReal.existsSync(path.join(mapperDir, 'url-mapping.json'))).to.be.true; + expect(fsReal.existsSync(path.join(mapperDir, 'space-uid-mapping.json'))).to.be.true; + + const uidMap = JSON.parse(fsReal.readFileSync(path.join(mapperDir, 'uid-mapping.json'), 'utf8')); + expect(uidMap).to.deep.equal({ 'old-uid': 'new-uid' }); + }); + }); + + describe('listSpaces error handling and uid filtering', () => { + it('should pass existing org space uids to ImportWorkspace when listSpaces returns spaces', async () => { + (CSAssetsAdapter.prototype.listSpaces as sinon.SinonStub).resolves({ spaces: [{ uid: 'org-space-uid' }] }); + stubSpaceDirs(['am-space-1']); + const startStub = sinon.stub(ImportWorkspace.prototype, 'start').resolves({ + oldSpaceUid: 'am-space-1', newSpaceUid: 'new-space', workspaceUid: 'main', + isDefault: false, uidMap: {}, urlMap: {}, + }); + + const importer = new ImportSpaces(baseOptions); + await importer.start(); + + expect(startStub.callCount).to.equal(1); + const existingSpaceUids: Set = startStub.firstCall.args[2]; + expect(existingSpaceUids.has('org-space-uid')).to.be.true; + }); + + it('should continue (disable reuse-by-uid) when listSpaces throws', async () => { + (CSAssetsAdapter.prototype.listSpaces as sinon.SinonStub).rejects(new Error('network error')); + stubSpaceDirs(['am-space-1']); + sinon.stub(ImportWorkspace.prototype, 'start').resolves({ + oldSpaceUid: 'am-space-1', newSpaceUid: 'new-uid', workspaceUid: 'main', + isDefault: false, uidMap: {}, urlMap: {}, + }); + + const importer = new ImportSpaces(baseOptions); + const result = await importer.start(); + + expect(result.spaceMappings).to.have.lengthOf(1); + }); + + it('should return false for a directory entry when statSync throws', async () => { + const fsMock = require('node:fs'); + const pResolve = require('node:path').resolve; + const join = require('node:path').join; + const spacesRoot = pResolve('/tmp/import', 'spaces'); + const origStatSync = fsMock.statSync.bind(fsMock); + sinon.stub(fsMock, 'readdirSync').returns(['am-bad-entry'] as any); + sinon.stub(fsMock, 'statSync').callsFake((p: string) => { + if (p === join(spacesRoot, 'am-bad-entry')) throw new Error('permission denied'); + return origStatSync(p); + }); + + const importer = new ImportSpaces(baseOptions); + const result = await importer.start(); + + expect(result.spaceMappings).to.deep.equal([]); + }); + + it('should log warning and return empty dirs when readdirSync throws', async () => { + const fsMock = require('node:fs'); + const pResolve = require('node:path').resolve; + const spacesRoot = pResolve('/tmp/import', 'spaces'); + const origReaddir = fsMock.readdirSync.bind(fsMock); + sinon.stub(fsMock, 'readdirSync').callsFake((p: string) => { + if (p === spacesRoot) throw new Error('ENOENT: no such file or directory'); + return origReaddir(p); + }); + sinon.stub(fsMock, 'statSync').returns({ isDirectory: () => true } as any); + + const importer = new ImportSpaces(baseOptions); + const result = await importer.start(); + + expect(result.spaceMappings).to.deep.equal([]); + }); + }); + + describe('setParentProgressManager', () => { + it('should use parent progress manager instead of creating a new CLIProgressManager', async () => { + const fakeParent = { + addProcess: sinon.stub().returnsThis(), + startProcess: sinon.stub().returnsThis(), + updateStatus: sinon.stub().returnsThis(), + tick: sinon.stub(), + completeProcess: sinon.stub(), + }; + stubSpaceDirs([]); + + const importer = new ImportSpaces(baseOptions); + importer.setParentProgressManager(fakeParent as any); + await importer.start(); + + expect((CLIProgressManager.createNested as sinon.SinonStub).callCount).to.equal(0); + expect(fakeParent.addProcess.callCount).to.be.greaterThan(0); + }); + }); }); diff --git a/packages/contentstack-asset-management/test/unit/utils/chunked-json-reader.test.ts b/packages/contentstack-asset-management/test/unit/utils/chunked-json-reader.test.ts new file mode 100644 index 000000000..309b24776 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/utils/chunked-json-reader.test.ts @@ -0,0 +1,143 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import * as os from 'os'; +import * as fsReal from 'fs'; +import * as path from 'path'; +import { FsUtility } from '@contentstack/cli-utilities'; + +import { forEachChunkedJsonStore, forEachChunkRecordsFromFs } from '../../../src/utils/chunked-json-reader'; + +describe('chunked-json-reader', () => { + afterEach(() => sinon.restore()); + + const makeFakeFs = (indexer: Record, chunks: unknown[]): FsUtility => { + let idx = 0; + return { + indexFileContent: indexer, + readChunkFiles: { next: async () => chunks[idx++] ?? null }, + getPlainMeta: () => ({}), + } as unknown as FsUtility; + }; + + describe('forEachChunkRecordsFromFs', () => { + it('does nothing when indexer is empty', async () => { + const onChunk = sinon.stub(); + await forEachChunkRecordsFromFs(makeFakeFs({}, []), { chunkReadLogLabel: 'test' }, onChunk); + expect(onChunk.callCount).to.equal(0); + }); + + it('calls onChunk with Object.values of each chunk record', async () => { + const r1 = { uid: 'uid-1', url: 'https://a.com' }; + const r2 = { uid: 'uid-2', url: 'https://b.com' }; + const collected: unknown[] = []; + await forEachChunkRecordsFromFs( + makeFakeFs({ '0': true }, [{ 'uid-1': r1, 'uid-2': r2 }]), + { chunkReadLogLabel: 'assets' }, + async (records) => { collected.push(...records); }, + ); + expect(collected).to.deep.equal([r1, r2]); + }); + + it('processes multiple chunks in order', async () => { + const order: string[] = []; + await forEachChunkRecordsFromFs( + makeFakeFs({ '0': true, '1': true }, [ + { 'uid-A': { uid: 'uid-A' } }, + { 'uid-B': { uid: 'uid-B' } }, + ]), + { chunkReadLogLabel: 'test' }, + async (records: any[]) => { order.push(...records.map((r) => r.uid)); }, + ); + expect(order).to.deep.equal(['uid-A', 'uid-B']); + }); + + it('skips a chunk when readChunkFiles.next() rejects', async () => { + const onChunk = sinon.stub(); + const fakeFs = { + indexFileContent: { '0': true }, + readChunkFiles: { next: sinon.stub().rejects(new Error('disk error')) }, + } as unknown as FsUtility; + await forEachChunkRecordsFromFs(fakeFs, { chunkReadLogLabel: 'test' }, onChunk); + expect(onChunk.callCount).to.equal(0); + }); + + it('skips null chunks returned by readChunkFiles', async () => { + const onChunk = sinon.stub(); + await forEachChunkRecordsFromFs( + makeFakeFs({ '0': true }, [null]), + { chunkReadLogLabel: 'test' }, + onChunk, + ); + expect(onChunk.callCount).to.equal(0); + }); + }); + + describe('forEachChunkedJsonStore', () => { + it('calls onOpenError and does not call onEmptyIndexer or onChunk when FsUtility constructor throws', async () => { + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => { + throw new Error('constructor error'); + }); + const onOpenError = sinon.stub(); + const onEmptyIndexer = sinon.stub(); + const onChunk = sinon.stub(); + + await forEachChunkedJsonStore( + '/nonexistent/path', + 'index.json', + { chunkReadLogLabel: 'test', onOpenError, onEmptyIndexer }, + onChunk, + ); + + expect(onOpenError.callCount).to.equal(1); + expect(onEmptyIndexer.callCount).to.equal(0); + expect(onChunk.callCount).to.equal(0); + }); + + it('calls onEmptyIndexer when the index file exists but has no entries', async () => { + const tmpDir = path.join(os.tmpdir(), `cjr-empty-${Date.now()}`); + fsReal.mkdirSync(tmpDir, { recursive: true }); + fsReal.writeFileSync(path.join(tmpDir, 'index.json'), '{}'); + + const onOpenError = sinon.stub(); + const onEmptyIndexer = sinon.stub(); + const onChunk = sinon.stub(); + + await forEachChunkedJsonStore( + tmpDir, + 'index.json', + { chunkReadLogLabel: 'test', onOpenError, onEmptyIndexer }, + onChunk, + ); + + expect(onEmptyIndexer.callCount).to.equal(1); + expect(onChunk.callCount).to.equal(0); + }); + + it('calls onChunk with records when the index has entries', async () => { + const tmpDir = path.join(os.tmpdir(), `cjr-chunks-${Date.now()}`); + fsReal.mkdirSync(tmpDir, { recursive: true }); + fsReal.writeFileSync(path.join(tmpDir, 'index.json'), '{"0": true}'); + + const record = { uid: 'field-1', name: 'My Field' }; + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => ({ '0': true })); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves({ 'field-1': record }), + })); + + const onOpenError = sinon.stub(); + const onEmptyIndexer = sinon.stub(); + const collected: unknown[] = []; + + await forEachChunkedJsonStore( + tmpDir, + 'index.json', + { chunkReadLogLabel: 'fields', onOpenError, onEmptyIndexer }, + async (records) => { collected.push(...records); }, + ); + + expect(onOpenError.callCount).to.equal(0); + expect(onEmptyIndexer.callCount).to.equal(0); + expect(collected).to.deep.equal([record]); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts b/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts index 20f122d82..0cfaa7417 100644 --- a/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts +++ b/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts @@ -186,7 +186,7 @@ describe('CSAssetsAdapter', () => { const atResponse = { count: 1, relation: 'org', asset_types: [{ uid: 'at1' }] }; getStub.resolves({ status: 200, data: atResponse }); const adapter = new CSAssetsAdapter(baseConfig); - const result = await adapter.getWorkspaceAssetTypes('sp-1'); + const result: unknown = await adapter.getWorkspaceAssetTypes('sp-1'); const path = getStub.firstCall.args[0] as string; expect(path).to.include('/api/asset_types'); @@ -216,4 +216,338 @@ describe('CSAssetsAdapter', () => { expect(path).to.not.include('?'); }); }); + + describe('listSpaces (paginated)', () => { + it('should return all spaces in a single page when count <= pageSize', async () => { + const spaces = [{ uid: 'sp-1' }, { uid: 'sp-2' }]; + getStub.resolves({ status: 200, data: { spaces, count: 2 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(100, 5); + + expect(getStub.callCount).to.equal(1); + expect(getStub.firstCall.args[0]).to.include('/api/spaces'); + expect(getStub.firstCall.args[0]).to.include('limit=100'); + expect(getStub.firstCall.args[0]).to.include('skip=0'); + expect(result.spaces).to.deep.equal(spaces); + expect(result.count).to.equal(2); + }); + + it('should issue additional page requests when total exceeds first page', async () => { + const page1 = Array.from({ length: 2 }, (_, i) => ({ uid: `sp-${i}` })); + const page2 = Array.from({ length: 1 }, (_, i) => ({ uid: `sp-${i + 2}` })); + getStub.onCall(0).resolves({ status: 200, data: { spaces: page1, count: 3 } }); + getStub.onCall(1).resolves({ status: 200, data: { spaces: page2, count: 3 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(2, 5); + + expect(getStub.callCount).to.equal(2); + expect(getStub.secondCall.args[0]).to.include('skip=2'); + expect(result.spaces).to.have.lengthOf(3); + expect(result.count).to.equal(3); + }); + + it('should return empty spaces when count is 0', async () => { + getStub.resolves({ status: 200, data: { spaces: [], count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(); + + expect(getStub.callCount).to.equal(1); + expect(result.spaces).to.deep.equal([]); + expect(result.count).to.equal(0); + }); + + it('should batch additional page requests by fetchConcurrency', async () => { + // 5 total, pageSize=1, concurrency=2 → 4 additional pages in 2 batches + const pages = Array.from({ length: 5 }, (_, i) => [{ uid: `sp-${i}` }]); + getStub.onCall(0).resolves({ status: 200, data: { spaces: pages[0], count: 5 } }); + getStub.onCall(1).resolves({ status: 200, data: { spaces: pages[1], count: 5 } }); + getStub.onCall(2).resolves({ status: 200, data: { spaces: pages[2], count: 5 } }); + getStub.onCall(3).resolves({ status: 200, data: { spaces: pages[3], count: 5 } }); + getStub.onCall(4).resolves({ status: 200, data: { spaces: pages[4], count: 5 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(1, 2); + + expect(getStub.callCount).to.equal(5); + expect(result.spaces).to.have.lengthOf(5); + }); + }); + + describe('getWorkspaceAssets (paginated)', () => { + it('should fetch all assets across multiple pages', async () => { + const page1 = [{ uid: 'a-1' }, { uid: 'a-2' }]; + const page2 = [{ uid: 'a-3' }]; + getStub.onCall(0).resolves({ status: 200, data: { assets: page1, count: 3 } }); + getStub.onCall(1).resolves({ status: 200, data: { assets: page2, count: 3 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getWorkspaceAssets('sp-1', undefined, 2, 5) as any; + + expect(result.assets).to.have.lengthOf(3); + expect(result.count).to.equal(3); + }); + + it('should include workspace query param when workspaceUid is provided', async () => { + getStub.resolves({ status: 200, data: { assets: [], count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.getWorkspaceAssets('sp-1', 'ws-main', 100, 5); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.include('workspace=ws-main'); + }); + + it('should NOT include workspace param when workspaceUid is undefined', async () => { + getStub.resolves({ status: 200, data: { assets: [], count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.getWorkspaceAssets('sp-1', undefined, 100, 5); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.not.include('workspace='); + }); + }); + + describe('getWorkspaceFolders (paginated)', () => { + it('should fetch all folders across multiple pages', async () => { + const page1 = [{ uid: 'f-1' }]; + const page2 = [{ uid: 'f-2' }]; + getStub.onCall(0).resolves({ status: 200, data: { folders: page1, count: 2 } }); + getStub.onCall(1).resolves({ status: 200, data: { folders: page2, count: 2 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getWorkspaceFolders('sp-1', undefined, 1, 5) as any; + + expect(result.folders).to.have.lengthOf(2); + expect(result.count).to.equal(2); + }); + + it('should include workspace param when workspaceUid is provided', async () => { + getStub.resolves({ status: 200, data: { folders: [], count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.getWorkspaceFolders('sp-1', 'ws-main', 100, 5); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.include('workspace=ws-main'); + }); + }); + + describe('POST methods (createSpace, createFolder, createField, createAssetType, bulkDelete, bulkMove)', () => { + let fetchStub: sinon.SinonStub; + + beforeEach(() => { + fetchStub = sinon.stub(global, 'fetch' as any); + }); + + const okJsonResponse = (data: unknown) => ({ + ok: true, + json: async () => data, + text: async () => JSON.stringify(data), + }); + + const failResponse = (status: number, body = 'error body') => ({ + ok: false, + status, + json: async () => ({}), + text: async () => body, + }); + + describe('createSpace', () => { + it('POSTs to /api/spaces and returns the created space', async () => { + const created = { space: { uid: 'new-space-uid', title: 'My Space' } }; + fetchStub.resolves(okJsonResponse(created)); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.createSpace({ title: 'My Space' }); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces'); + expect(opts.method).to.equal('POST'); + expect(result).to.deep.equal(created); + }); + + it('throws when POST returns non-ok status', async () => { + fetchStub.resolves(failResponse(400, 'bad request')); + const adapter = new CSAssetsAdapter(baseConfig); + + try { + await adapter.createSpace({ title: 'Bad Space' }); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('400'); + } + }); + }); + + describe('createFolder', () => { + it('POSTs to /api/spaces/{spaceUid}/folders with space_key header', async () => { + const created = { folder: { uid: 'folder-new' } }; + fetchStub.resolves(okJsonResponse(created)); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.createFolder('sp-1', { title: 'Docs' }); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces/sp-1/folders'); + expect(opts.headers['space_key']).to.equal('sp-1'); + expect(result).to.deep.equal(created); + }); + + it('URL-encodes spaceUid with special characters', async () => { + fetchStub.resolves(okJsonResponse({ folder: { uid: 'f1' } })); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.createFolder('sp uid/1', { title: 'X' }); + + const [url] = fetchStub.firstCall.args; + expect(url).to.include('sp%20uid%2F1'); + }); + }); + + describe('createField', () => { + it('POSTs to /api/fields and returns the created field', async () => { + const created = { field: { uid: 'field-1' } }; + fetchStub.resolves(okJsonResponse(created)); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.createField({ uid: 'field-1', label: 'My Field' } as any); + + const [url] = fetchStub.firstCall.args; + expect(url).to.include('/api/fields'); + expect(result).to.deep.equal(created); + }); + }); + + describe('createAssetType', () => { + it('POSTs to /api/asset_types and returns the created asset type', async () => { + const created = { asset_type: { uid: 'at-1' } }; + fetchStub.resolves(okJsonResponse(created)); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.createAssetType({ uid: 'at-1' } as any); + + const [url] = fetchStub.firstCall.args; + expect(url).to.include('/api/asset_types'); + expect(result).to.deep.equal(created); + }); + }); + + describe('bulkDeleteAssets', () => { + it('POSTs to the bulk delete endpoint with workspace query param', async () => { + fetchStub.resolves(okJsonResponse({ deleted: 2 })); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.bulkDeleteAssets('sp-1', 'ws-main', { asset_uids: ['a1', 'a2'] } as any); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces/sp-1/assets/bulk/delete'); + expect(url).to.include('workspace=ws-main'); + expect(opts.headers['space_key']).to.equal('sp-1'); + }); + + it('uses "main" as default workspace uid', async () => { + fetchStub.resolves(okJsonResponse({})); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.bulkDeleteAssets('sp-1', undefined as any, {} as any); + + const [url] = fetchStub.firstCall.args; + expect(url).to.include('workspace=main'); + }); + }); + + describe('bulkMoveAssets', () => { + it('POSTs to the bulk-move endpoint with workspace query param', async () => { + fetchStub.resolves(okJsonResponse({ moved: 1 })); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.bulkMoveAssets('sp-1', 'ws-main', { asset_uids: ['a1'], folder_uid: 'f1' } as any); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces/sp-1/assets/bulk-move'); + expect(url).to.include('workspace=ws-main'); + expect(opts.headers['space_key']).to.equal('sp-1'); + }); + }); + + describe('postJson error handling', () => { + it('wraps non-API errors in a consistent error message', async () => { + fetchStub.rejects(new Error('network failure')); + const adapter = new CSAssetsAdapter(baseConfig); + + try { + await adapter.createField({} as any); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('CS Assets API POST failed'); + expect(err.message).to.include('network failure'); + } + }); + }); + + describe('uploadAsset', () => { + const os = require('os'); + const path = require('path'); + const fsReal = require('fs'); + + it('reads the file, builds multipart form, and POSTs to /api/spaces/{uid}/assets', async () => { + const tmpFile = path.join(os.tmpdir(), `upload-test-${Date.now()}.png`); + fsReal.writeFileSync(tmpFile, 'fake-image-content'); + fetchStub.resolves(okJsonResponse({ asset: { uid: 'new-asset', url: 'https://cdn.com/x.png' } })); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.uploadAsset('sp-1', tmpFile, { title: 'My Image' }); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces/sp-1/assets'); + expect(opts.method).to.equal('POST'); + expect(opts.headers['space_key']).to.equal('sp-1'); + expect(result).to.deep.equal({ asset: { uid: 'new-asset', url: 'https://cdn.com/x.png' } }); + + fsReal.unlinkSync(tmpFile); + }); + + it('appends description and parent_uid to the form when provided', async () => { + const tmpFile = path.join(os.tmpdir(), `upload-test-desc-${Date.now()}.png`); + fsReal.writeFileSync(tmpFile, 'data'); + const formAppendSpy = sinon.spy(FormData.prototype, 'append'); + fetchStub.resolves(okJsonResponse({ asset: { uid: 'a1', url: 'https://cdn.com/a1.png' } })); + + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.uploadAsset('sp-1', tmpFile, { + title: 'T', description: 'Desc', parent_uid: 'folder-uid', + }); + + const appendCalls = formAppendSpy.getCalls().map((c) => c.args[0]); + expect(appendCalls).to.include('description'); + expect(appendCalls).to.include('parent_uid'); + + fsReal.unlinkSync(tmpFile); + }); + + it('throws when multipart POST returns non-ok status', async () => { + const tmpFile = path.join(os.tmpdir(), `upload-fail-${Date.now()}.png`); + fsReal.writeFileSync(tmpFile, 'data'); + fetchStub.resolves(failResponse(413, 'file too large')); + + const adapter = new CSAssetsAdapter(baseConfig); + try { + await adapter.uploadAsset('sp-1', tmpFile, { title: 'Big File' }); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('413'); + } + + fsReal.unlinkSync(tmpFile); + }); + + it('wraps network errors from multipart fetch in a consistent error message', async () => { + const tmpFile = path.join(os.tmpdir(), `upload-net-${Date.now()}.png`); + fsReal.writeFileSync(tmpFile, 'data'); + fetchStub.rejects(new Error('connection reset')); + + const adapter = new CSAssetsAdapter(baseConfig); + try { + await adapter.uploadAsset('sp-1', tmpFile, { title: 'File' }); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('CS Assets API multipart POST failed'); + expect(err.message).to.include('connection reset'); + } + + fsReal.unlinkSync(tmpFile); + }); + }); + }); }); diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 3df8b5847..373a592a3 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -122,6 +122,8 @@ const config: DefaultConfig = { chunkFileSizeMb: 1, apiConcurrency: 5, downloadAssetsConcurrency: 5, + pageSize: 100, + fetchConcurrency: 5, }, content_types: { dirName: 'content_types', diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index a5a529ad9..583f9d8b6 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -108,6 +108,8 @@ export default class ExportAssets extends BaseClass { chunkFileSizeMb: csAssetsModuleConfig?.chunkFileSizeMb, apiConcurrency: csAssetsModuleConfig?.apiConcurrency, downloadAssetsConcurrency: csAssetsModuleConfig?.downloadAssetsConcurrency, + pageSize: csAssetsModuleConfig?.pageSize, + fetchConcurrency: csAssetsModuleConfig?.fetchConcurrency, }); exporter.setParentProgressManager(progress); await exporter.start(); diff --git a/packages/contentstack-export/src/types/default-config.ts b/packages/contentstack-export/src/types/default-config.ts index ba03e91ee..0c684b702 100644 --- a/packages/contentstack-export/src/types/default-config.ts +++ b/packages/contentstack-export/src/types/default-config.ts @@ -110,6 +110,10 @@ export default interface DefaultConfig { apiConcurrency: number; /** Parallel downloads per AM workspace export. */ downloadAssetsConcurrency: number; + /** Items per page for paginated GET requests (assets, folders, spaces). */ + pageSize: number; + /** Parallel page fetches for paginated GET requests. */ + fetchConcurrency: number; dependencies?: Modules[]; }; content_types: {