CMG-824 | Delta Migration Support for Wordpress CMS

.github/workflows/test.yml

@@ -40,6 +40,7 @@ jobs:
           cache: npm
           cache-dependency-path: ui/package-lock.json
       - run: npm ci --legacy-peer-deps
+      - run: npm install @rollup/rollup-linux-x64-gnu --no-save
       - run: npm run test:coverage
       - uses: actions/upload-artifact@v4
         if: always()

api/package.json

@@ -51,7 +51,7 @@
     "html-to-json-parser": "^2.0.1",
     "jsdom": "^24.1.0",
     "jsonwebtoken": "^9.0.3",
-    "lodash": "^4.17.21",
+    "lodash": "^4.18.1",
     "lowdb": "^7.0.1",
     "mkdirp": "^3.0.1",
     "mysql2": "^3.16.2",

api/src/services/contentful.service.ts

@@ -99,6 +99,36 @@ const mapLocales = ({ masterLocale, locale, locales, isNull = false }: any) => {
   }
 }
 
+function resolveEntryFieldKey(entry: Record<string, unknown>, baseKey: string): string | undefined {
+  if (baseKey in entry) return baseKey;
+  const snake = baseKey.replace(/([A-Z])/g, (m) => `_${m.toLowerCase()}`);
+  if (snake in entry) return snake;
+  return undefined;
+}
+
+/**
+ * Maps Contentful content type id → field id → whether that field is localized in the export schema.
+ * Used so we only fan out values for fields with `localized: false`, not for localized fields that
+ * happen to have a single locale in the entry (missing translations).
+ */
+function buildContentfulFieldLocalizedByContentType(
+  contentTypesFromPackage: any[]
+): Map<string, Map<string, boolean>> {
+  const byCt = new Map<string, Map<string, boolean>>();
+  for (const ct of contentTypesFromPackage ?? []) {
+    const ctId = ct?.sys?.id;
+    if (!ctId) continue;
+    const byField = new Map<string, boolean>();
+    for (const f of ct?.fields ?? []) {
+      if (f?.id != null) {
+        byField.set(f.id, f.localized === true);
+      }
+    }
+    byCt.set(ctId, byField);
+  }
+  return byCt;
+}
+
 const transformCloudinaryObject = (input: any) => {
   const result: any = [];
   if (!Array.isArray(input)) {
@@ -777,6 +807,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
     const data = await fs.promises.readFile(packagePath, "utf8");
     const entries = JSON.parse(data)?.entries;
     const content = JSON.parse(data)?.contentTypes;
+    const cfFieldLocalizedByCt = buildContentfulFieldLocalizedByContentType(content);
     const LocaleMapper = { masterLocale: project?.master_locale ?? LOCALE_MAPPER?.masterLocale, ...project?.locales ?? {} };
     if (entries && entries.length > 0) {
       const assetId = await readFile(assetsSave, ASSETS_SCHEMA_FILE) ?? [];
@@ -814,7 +845,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
               entryData[name][lang] ??= {};
               entryData[name][lang][id] ??= {};
               locales.push(lang);
-              const fieldData = currentCT?.fieldMapping?.find?.((item: any) => (key === item?.uid) && (!["text", "url"]?.includes?.(item?.backupFieldType)));
+              const fieldData = currentCT?.fieldMapping?.find?.((item: any) => key === item?.uid);
               const newId = fieldData?.contentstackFieldUid ?? `${key}`?.replace?.(/[^a-zA-Z0-9]+/g, "_");
               entryData[name][lang][id][newId] = processField(
                 langValue,
@@ -860,18 +891,73 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
               );
             });
           });
+
+          // Non-localized Contentful fields (`localized: false` in the content type) only appear under
+          // one locale in exports. Copy them to every other locale branch so each slice is complete.
+          // Do not infer non-localized-ness from a single locale key — localized fields can legitimately
+          // have only one locale when translations are missing.
+          const entryLocaleKeys = new Set<string>();
+          for (const [, v] of Object?.entries?.(fields)) {
+            for (const lang of Object.keys(v as object)) {
+              entryLocaleKeys.add(lang);
+            }
+          }
+          const ct = contentTypes?.find((c: any) => c?.otherCmsUid === name);
+          for (const [key, value] of Object?.entries?.(fields)) {
+            const langs = Object?.keys(value as object);
+            if (langs?.length !== 1) continue;
+            const fd = ct?.fieldMapping?.find?.((item: any) => key === item?.uid);
+            const localizedInCf = cfFieldLocalizedByCt.get(name)?.get(key);
+            const explicitlyNonLocalized =
+              localizedInCf === false ||
+              (localizedInCf === undefined && fd?.advanced?.nonLocalizable === true);
+            if (!explicitlyNonLocalized) continue;
+            const srcLang = langs[0];
+            const newId = fd?.contentstackFieldUid ?? `${key}`?.replace?.(/[^a-zA-Z0-9]+/g, "_");
+            const srcEntry = entryData[name][srcLang]?.[id] as Record<string, unknown> | undefined;
+            if (!srcEntry) continue;
+            const fk = resolveEntryFieldKey(srcEntry, newId);
+            if (fk === undefined) continue;
+            for (const tgtLang of entryLocaleKeys) {
+              if (tgtLang === srcLang) continue;
+              entryData[name][tgtLang] ??= {};
+              entryData[name][tgtLang][id] ??= {};
+              const tgt = entryData[name][tgtLang][id] as Record<string, unknown>;
+              if (tgt[fk] === undefined) {
+                tgt[fk] = srcEntry[fk];
+              }
+            }
+          }
+
           return entryData;
         },
         {}
       );
       for await (const [newKey, values] of Object.entries(result)) {
         const currentCT = contentTypes?.find((ct: any) => ct?.otherCmsUid === newKey);
         const ctName = currentCT?.contentstackUid in mapperKeys ?
-          mapperKeys?.[currentCT?.contentstackUid] : (currentCT?.contentstackUid ?? newKey.replace(/([A-Z])/g, "_$1").toLowerCase());
-        for await (const [localeKey, localeValues] of Object.entries(
-          values as { [key: string]: any }
-        )) {
-          const localeCode = mapLocales({ masterLocale: master_locale, locale: localeKey, locales: LocaleMapper, isNull: true });
+          mapperKeys?.[currentCT?.contentstackUid] : (currentCT?.contentstackUid ?? newKey?.replace?.(/([A-Z])/g, "_$1")?.toLowerCase?.());
+        const valuesByCfLocale = values as { [key: string]: { [uid: string]: Record<string, unknown> } };
+        const mergedByDestinationLocale: { [localeCode: string]: { [uid: string]: Record<string, unknown> } } = {};
+        for (const localeKey of Object.keys(valuesByCfLocale)) {
+          const localeValues = valuesByCfLocale[localeKey];
+          if (!localeValues) continue;
+          const localeCode = mapLocales({
+            masterLocale: master_locale,
+            locale: localeKey,
+            locales: LocaleMapper,
+            isNull: true,
+          });
+          if (!localeCode) continue;
+          mergedByDestinationLocale[localeCode] ??= {};
+          for (const [uid, entry] of Object.entries(localeValues)) {
+            mergedByDestinationLocale[localeCode][uid] = {
+              ...(mergedByDestinationLocale[localeCode][uid] ?? {}),
+              ...(entry ?? {}),
+            };
+          }
+        }
+        for await (const [localeCode, localeValues] of Object.entries(mergedByDestinationLocale)) {
           const chunks = makeChunks(localeValues);
           for (const [entryKey, entryValue] of Object.entries(localeValues)) {
             const message = getLogMessage(
@@ -883,18 +969,12 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
           }
           const refs: { [key: string]: any } = {};
           let chunkIndex = 1;
-          if (localeCode) {
-            const filePath = path.join(
-              entriesSave,
-              ctName,
-              localeCode
-            );
-            for await (const [chunkId, chunkData] of Object.entries(chunks)) {
-              refs[chunkIndex++] = `${chunkId}-entries.json`;
-              await writeFile(filePath, `${chunkId}-entries.json`, chunkData);
-            }
-            await writeFile(filePath, ENTRIES_MASTER_FILE, refs);
+          const filePath = path.join(entriesSave, ctName, localeCode);
+          for await (const [chunkId, chunkData] of Object.entries(chunks)) {
+            refs[chunkIndex++] = `${chunkId}-entries.json`;
+            await writeFile(filePath, `${chunkId}-entries.json`, chunkData);
           }
+          await writeFile(filePath, ENTRIES_MASTER_FILE, refs);
         }
       }
     } else {

api/src/services/wordpress.service.ts

@@ -518,6 +518,11 @@ async function saveEntry(fields: any, entry: any,  file_path: string, assetData
   const $ = cheerio.load(xmlData, { xmlMode: true });
   const items = $('item');
   const entryData: Record<string, any> = {};
+  const fieldList = Array.isArray(fields) ? fields : [];
+  const hasField = (uid: string) =>
+    fieldList.some(
+      (field: any) => field?.contentstackFieldUid === uid || field?.uid === uid
+    );
 
   try {
     if(entry ){
@@ -584,16 +589,20 @@ async function saveEntry(fields: any, entry: any,  file_path: string, assetData
           // Pass individual content to createSchema
           entryData[uid] = await createSchema(fields, blocksJson, item?.title, uid, assetData, duplicateBlockMappings);
           const categoryReference = extractCategoryReference(item?.['category']);
-          if (categoryReference?.length > 0) {
-            entryData[uid]['taxonomies'] = taxonomies;
+          if (hasField("taxonomies") && categoryReference?.length > 0) {
+            entryData[uid]["taxonomies"] = taxonomies;
           }
           const termsReference = extractTermsReference(item?.['category']);
-          if(termsReference?.length > 0) {
-            entryData[uid]['terms'] = terms;
+          if (hasField("terms") && termsReference?.length > 0) {
+            entryData[uid]["terms"] = terms;
+          }
+          entryData[uid]["tags"] = tags?.map((tag: any) => tag?.text) || [];
+          if (hasField("author")) {
+            entryData[uid]["author"] =
+              authorData?.filter((author: any) => author?.uid) || [];
           }
-          entryData[uid]['tags'] = tags?.map((tag: any) => tag?.text);
-          entryData[uid]['author'] = authorData;
           entryData[uid]['locale'] = locale;
+          entryData[uid]["publish_details"] = [];
           entryData[uid]['publish_details'] = [];
 
 

api/src/utils/market-app.utils.ts

@@ -1,13 +1,13 @@
-import contentstack from '@contentstack/marketplace-sdk';
+import {client} from '@contentstack/marketplace-sdk';
 import { DEVURLS } from '../constants/index.js';
 
 
 
 
 export const getAllApps = async ({ organizationUid, authtoken, region }: any) => {
   try {
-    const client = contentstack.client({ authtoken, host: DEVURLS?.[region] ?? DEVURLS?.NA });
-    const data = await client.marketplace(organizationUid).findAllApps();
+    const contentstackclient = client({ authtoken, host: DEVURLS?.[region] ?? DEVURLS?.NA });
+    const data = await contentstackclient.marketplace(organizationUid).findAllApps();
     return data?.items;
   } catch (err) {
     console.info("🚀 ~ getAllApps ~ err:", err)
@@ -16,8 +16,8 @@ export const getAllApps = async ({ organizationUid, authtoken, region }: any) =>
 
 export const getAppManifestAndAppConfig = async ({ organizationUid, authtoken, region, manifestUid }: any) => {
   try {
-    const client = contentstack.client({ authtoken, host: DEVURLS?.[region] ?? DEVURLS?.NA });
-    const data = await client.marketplace(organizationUid).app(manifestUid).fetch();
+    const contentstackclient = client({ authtoken, host: DEVURLS?.[region] ?? DEVURLS?.NA });
+    const data = await contentstackclient.marketplace(organizationUid).app(manifestUid).fetch();
     return data;
   } catch (err: any) {
     console.info("🚀 ~ getAppManifestAndAppConfig ~ err:", err)

api/src/validators/file-format.validator.ts

@@ -18,13 +18,14 @@ export default checkSchema({
       errorMessage: VALIDATION_ERRORS.LENGTH_LIMIT.replace(/\$/g, "file_format"),
       options: {
         min: 1,
-        max: 400,
+        max: 200,
       },
       bail: true,
     },
   },
   file_path: {
     in: "body",
+    optional: { options: { nullable: true } },
     isString: {
       errorMessage: VALIDATION_ERRORS.STRING_REQUIRED.replace(/\$/g, "file_path"),
       bail: true,

api/sso.utils.js

@@ -367,13 +367,13 @@ module.exports = async ({
     };
 
     if (ENCRYPT_KEY) {
-      if (appData.oauthData) {
-        appData.oauthData.client_id = encrypt(appData.oauthData.client_id);
-        appData.oauthData.client_secret = encrypt(appData.oauthData.client_secret);
+      if (appData?.oauthData) {
+        appData?.oauthData?.client_id = encrypt(appData?.oauthData?.client_id);
+        appData?.oauthData?.client_secret = encrypt(appData?.oauthData?.client_secret);
       }
-      if (appData.pkce) {
-        appData.pkce.code_verifier = encrypt(appData.pkce.code_verifier);
-        appData.pkce.code_challenge = encrypt(appData.pkce.code_challenge);
+      if (appData?.pkce) {
+        appData?.pkce?.code_verifier = encrypt(appData?.pkce?.code_verifier);
+        appData?.pkce?.code_challenge = encrypt(appData?.pkce?.code_challenge);
       }
     } else {
       console.warn("WARNING: MANIFEST_ENCRYPT_KEY not set — app.json will contain plaintext credentials");