Skip to content
Merged

Dev #1035

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions api/src/constants/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -326,3 +326,6 @@ export const RESERVED_FIELD_MAPPINGS: Record<string, string> = {
locale: 'cm_locale',
// Add other reserved fields if needed
};

export const MEDIA_BLOCK_NAMES = ['core/image', 'core/video', 'core/audio', 'core/file'];
export const WORDPRESS_MISSSING_BLOCKS = 'core/missing';
114 changes: 97 additions & 17 deletions api/src/services/contentful.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,36 @@ const mapLocales = ({ masterLocale, locale, locales, isNull = false }: any) => {
}
}

function resolveEntryFieldKey(entry: Record<string, unknown>, baseKey: string): string | undefined {
if (baseKey in entry) return baseKey;
const snake = baseKey.replace(/([A-Z])/g, (m) => `_${m.toLowerCase()}`);
if (snake in entry) return snake;
return undefined;
}

/**
* Maps Contentful content type id → field id → whether that field is localized in the export schema.
* Used so we only fan out values for fields with `localized: false`, not for localized fields that
* happen to have a single locale in the entry (missing translations).
*/
function buildContentfulFieldLocalizedByContentType(
contentTypesFromPackage: any[]
): Map<string, Map<string, boolean>> {
const byCt = new Map<string, Map<string, boolean>>();
for (const ct of contentTypesFromPackage ?? []) {
const ctId = ct?.sys?.id;
if (!ctId) continue;
const byField = new Map<string, boolean>();
for (const f of ct?.fields ?? []) {
if (f?.id != null) {
byField.set(f.id, f.localized === true);
}
}
byCt.set(ctId, byField);
}
return byCt;
}

const transformCloudinaryObject = (input: any) => {
const result: any = [];
if (!Array.isArray(input)) {
Expand Down Expand Up @@ -777,6 +807,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
const data = await fs.promises.readFile(packagePath, "utf8");
const entries = JSON.parse(data)?.entries;
const content = JSON.parse(data)?.contentTypes;
const cfFieldLocalizedByCt = buildContentfulFieldLocalizedByContentType(content);
const LocaleMapper = { masterLocale: project?.master_locale ?? LOCALE_MAPPER?.masterLocale, ...project?.locales ?? {} };
if (entries && entries.length > 0) {
const assetId = await readFile(assetsSave, ASSETS_SCHEMA_FILE) ?? [];
Expand Down Expand Up @@ -814,7 +845,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
entryData[name][lang] ??= {};
entryData[name][lang][id] ??= {};
locales.push(lang);
const fieldData = currentCT?.fieldMapping?.find?.((item: any) => (key === item?.uid) && (!["text", "url"]?.includes?.(item?.backupFieldType)));
const fieldData = currentCT?.fieldMapping?.find?.((item: any) => key === item?.uid);
const newId = fieldData?.contentstackFieldUid ?? `${key}`?.replace?.(/[^a-zA-Z0-9]+/g, "_");
entryData[name][lang][id][newId] = processField(
langValue,
Expand Down Expand Up @@ -860,18 +891,73 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
);
});
});

// Non-localized Contentful fields (`localized: false` in the content type) only appear under
// one locale in exports. Copy them to every other locale branch so each slice is complete.
// Do not infer non-localized-ness from a single locale key — localized fields can legitimately
// have only one locale when translations are missing.
const entryLocaleKeys = new Set<string>();
for (const [, v] of Object?.entries?.(fields)) {
for (const lang of Object.keys(v as object)) {
entryLocaleKeys.add(lang);
}
}
const ct = contentTypes?.find((c: any) => c?.otherCmsUid === name);
for (const [key, value] of Object?.entries?.(fields)) {
const langs = Object?.keys(value as object);
if (langs?.length !== 1) continue;
const fd = ct?.fieldMapping?.find?.((item: any) => key === item?.uid);
const localizedInCf = cfFieldLocalizedByCt.get(name)?.get(key);
const explicitlyNonLocalized =
localizedInCf === false ||
(localizedInCf === undefined && fd?.advanced?.nonLocalizable === true);
if (!explicitlyNonLocalized) continue;
const srcLang = langs[0];
const newId = fd?.contentstackFieldUid ?? `${key}`?.replace?.(/[^a-zA-Z0-9]+/g, "_");
const srcEntry = entryData[name][srcLang]?.[id] as Record<string, unknown> | undefined;
if (!srcEntry) continue;
const fk = resolveEntryFieldKey(srcEntry, newId);
if (fk === undefined) continue;
for (const tgtLang of entryLocaleKeys) {
if (tgtLang === srcLang) continue;
entryData[name][tgtLang] ??= {};
entryData[name][tgtLang][id] ??= {};
const tgt = entryData[name][tgtLang][id] as Record<string, unknown>;
if (tgt[fk] === undefined) {
tgt[fk] = srcEntry[fk];
}
}
}

return entryData;
},
{}
);
for await (const [newKey, values] of Object.entries(result)) {
const currentCT = contentTypes?.find((ct: any) => ct?.otherCmsUid === newKey);
const ctName = currentCT?.contentstackUid in mapperKeys ?
mapperKeys?.[currentCT?.contentstackUid] : (currentCT?.contentstackUid ?? newKey.replace(/([A-Z])/g, "_$1").toLowerCase());
for await (const [localeKey, localeValues] of Object.entries(
values as { [key: string]: any }
)) {
const localeCode = mapLocales({ masterLocale: master_locale, locale: localeKey, locales: LocaleMapper, isNull: true });
mapperKeys?.[currentCT?.contentstackUid] : (currentCT?.contentstackUid ?? newKey?.replace?.(/([A-Z])/g, "_$1")?.toLowerCase?.());
const valuesByCfLocale = values as { [key: string]: { [uid: string]: Record<string, unknown> } };
const mergedByDestinationLocale: { [localeCode: string]: { [uid: string]: Record<string, unknown> } } = {};
for (const localeKey of Object.keys(valuesByCfLocale)) {
const localeValues = valuesByCfLocale[localeKey];
if (!localeValues) continue;
const localeCode = mapLocales({
masterLocale: master_locale,
locale: localeKey,
locales: LocaleMapper,
isNull: true,
});
if (!localeCode) continue;
mergedByDestinationLocale[localeCode] ??= {};
for (const [uid, entry] of Object.entries(localeValues)) {
mergedByDestinationLocale[localeCode][uid] = {
...(mergedByDestinationLocale[localeCode][uid] ?? {}),
...(entry ?? {}),
};
}
}
for await (const [localeCode, localeValues] of Object.entries(mergedByDestinationLocale)) {
const chunks = makeChunks(localeValues);
for (const [entryKey, entryValue] of Object.entries(localeValues)) {
const message = getLogMessage(
Expand All @@ -883,18 +969,12 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
}
const refs: { [key: string]: any } = {};
let chunkIndex = 1;
if (localeCode) {
const filePath = path.join(
entriesSave,
ctName,
localeCode
);
for await (const [chunkId, chunkData] of Object.entries(chunks)) {
refs[chunkIndex++] = `${chunkId}-entries.json`;
await writeFile(filePath, `${chunkId}-entries.json`, chunkData);
}
await writeFile(filePath, ENTRIES_MASTER_FILE, refs);
const filePath = path.join(entriesSave, ctName, localeCode);
for await (const [chunkId, chunkData] of Object.entries(chunks)) {
refs[chunkIndex++] = `${chunkId}-entries.json`;
await writeFile(filePath, `${chunkId}-entries.json`, chunkData);
}
await writeFile(filePath, ENTRIES_MASTER_FILE, refs);
}
}
} else {
Expand Down
39 changes: 29 additions & 10 deletions api/src/services/wordpress.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { orgService } from "./org.service.js";
import * as cheerio from 'cheerio';
import { setupWordPressBlocks, stripHtmlTags } from "../utils/wordpressParseUtil.js";
import { getMimeTypeFromExtension } from "../utils/mimeTypes.js";
import { MEDIA_BLOCK_NAMES, WORDPRESS_MISSSING_BLOCKS } from "../constants/index.js";

const { JSDOM } = jsdom;

Expand Down Expand Up @@ -124,6 +125,16 @@ function getLastUid(uid : string) {
return uid?.split?.('.')?.[uid?.split?.('.')?.length - 1];
}


const resolvedBlockName = (block: any) => {
if (block?.attrs?.metadata?.name) return block?.attrs?.metadata?.name;
if (block?.blockName === WORDPRESS_MISSSING_BLOCKS) {
return block?.attrs?.originalName || 'body';
}
if (MEDIA_BLOCK_NAMES?.includes?.(block?.blockName)) return 'media';
return block?.blockName;
}

async function createSchema(fields: any, blockJson : any, title: string, uid: string, assetData: any, duplicateBlockMappings?: Record<string, string>) {
const schema : any = {
title: title,
Expand Down Expand Up @@ -159,13 +170,13 @@ async function createSchema(fields: any, blockJson : any, title: string, uid: st
// Process each block in blockJson to see if it matches any modular block child
for (const block of blockJson) {
try {
const blockName = (block?.attrs?.metadata?.name?.toLowerCase() || getFieldName(block?.blockName?.toLowerCase()));
const blockName = getFieldName(resolvedBlockName(block));

// Find which modular block child this block matches
let matchingChildField = fields.find((childField: any) => {
const fieldName = childField?.otherCmsField?.toLowerCase() ;

return (childField?.contentstackFieldType !== 'modular_blocks_child') && (blockName === fieldName)
const fieldName = childField?.otherCmsField?.toLowerCase();
const fieldType = childField?.otherCmsType?.toLowerCase();
return (childField?.contentstackFieldType !== 'modular_blocks_child') && (blockName === fieldName || blockName === fieldType)
});

let matchingModularBlockChild = modularBlockChildren.find((childField: any) => {
Expand All @@ -187,7 +198,8 @@ async function createSchema(fields: any, blockJson : any, title: string, uid: st
//if (!matchingChildField) {
matchingChildField = fields.find((childField: any) => {
const fieldName = childField?.otherCmsField?.toLowerCase();
return (childField?.contentstackFieldType !== 'modular_blocks_child') && (mappedName === fieldName);
const fieldType = childField?.otherCmsType?.toLowerCase();
return (childField?.contentstackFieldType !== 'modular_blocks_child') && (mappedName === fieldName || mappedName === fieldType);
});

// }
Expand All @@ -206,12 +218,13 @@ async function createSchema(fields: any, blockJson : any, title: string, uid: st
const childFieldUid = matchingModularBlockChild?.contentstackFieldUid || getLastUid(matchingModularBlockChild?.contentstackUid);
const childField = fields.find((f: any) => {
const fUid = f?.contentstackFieldUid || '';
const fOtherCmsField = f?.otherCmsType?.toLowerCase();
const childBlockName = matchingChildField ? matchingChildField?.otherCmsField?.toLowerCase() : (child?.attrs?.metadata?.name?.toLowerCase() || getFieldName(child?.blockName?.toLowerCase()));
const fOtherCmsType = f?.otherCmsType?.toLowerCase();
const fOtherCmsField = f?.otherCmsField?.toLowerCase();
const childBlockName = matchingChildField ? matchingChildField?.otherCmsField?.toLowerCase() : (getFieldName(resolvedBlockName(child))?.toLowerCase() || getFieldName(resolvedBlockName(child)?.toLowerCase()));
const childKey = getLastUid(f?.contentstackFieldUid);
const alreadyPopulated = childrenObject[childKey] !== undefined && childrenObject[childKey] !== null;
return fUid.startsWith(childFieldUid + '.') &&
(fOtherCmsField === childBlockName) && (!alreadyPopulated || f?.advanced?.multiple === true);
(fOtherCmsType === childBlockName || fOtherCmsField === childBlockName) && (!alreadyPopulated || f?.advanced?.multiple === true);
});

if (childField) {
Expand Down Expand Up @@ -313,8 +326,9 @@ function processNestedGroup(child: any, childField: any, allFields: any[]): Reco
child?.innerBlocks?.forEach((nestedChild: any, nestedIndex: number) => {
try {

const nestedBlockName = (getFieldName(resolvedBlockName(nestedChild))?.toLowerCase() ?? getFieldName(resolvedBlockName(nestedChild)?.toLowerCase()))?.toLowerCase();
const nestedChildField = nestedFields?.find((field: any) =>
field?.otherCmsType?.toLowerCase() === (nestedChild?.attrs?.metadata?.name?.toLowerCase() ?? getFieldName(nestedChild?.blockName?.toLowerCase()))?.toLowerCase() && !nestedChildrenObject[getLastUid(field?.contentstackFieldUid)]?.length
(field?.otherCmsType?.toLowerCase() === nestedBlockName || field?.otherCmsField?.toLowerCase() === nestedBlockName) && !nestedChildrenObject[getLastUid(field?.contentstackFieldUid)]?.length
);

if (!nestedChildField) {
Expand Down Expand Up @@ -387,7 +401,7 @@ function formatChildByType(child: any, field: any, assetData: any) {

// Process attributes based on field type configuration
//if (child?.attributes && typeof child.attributes === 'object') {
const attrKey = getFieldName(child?.attr?.metadata?.name?.toLowerCase() || child?.blockName?.toLowerCase());
const attrKey = getFieldName(getFieldName(resolvedBlockName(child))?.toLowerCase() || getFieldName(resolvedBlockName(child)?.toLowerCase()));
try {
const attrValue = child?.attrs?.innerHTML;

Expand Down Expand Up @@ -477,6 +491,10 @@ function formatChildByType(child: any, field: any, assetData: any) {
formatted = asset;
break;
}

case 'markdown':
formatted = stripHtmlTags(child?.innerHTML);
break;
default:
// Default formatting - preserve original structure with null check
formatted = attrValue;
Expand Down Expand Up @@ -580,6 +598,7 @@ async function saveEntry(fields: any, entry: any, file_path: string, assetData
// Extract individual content encoded for this specific item
const contentEncoded = $(xmlItem)?.find("content\\:encoded")?.text() || '';
const blocksJson = await setupWordPressBlocks(contentEncoded);

customLogger(project?.id, destinationStackId,'info', `Processed blocks for entry ${uid}`);


Expand Down
9 changes: 5 additions & 4 deletions ui/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion ui/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
"sass": "^1.68.0",
"socket.io-client": "^4.7.5",
"typescript": "^4.9.5",
"vite": "^7.3.1",
"vite": "^7.3.2",
"vite-tsconfig-paths": "^6.1.1"
},
"scripts": {
Expand Down
5 changes: 3 additions & 2 deletions ui/src/components/Common/AddStack/addStack.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ const AddStack = (props: any): JSX.Element => {
locale: formData?.locale?.value || props?.defaultValues?.locale
});

if (resp) {
if (resp === true) {
Notification({
notificationContent: { text: 'Stack created successfully' },
notificationProps: {
Expand All @@ -67,8 +67,9 @@ const AddStack = (props: any): JSX.Element => {
});
props?.closeModal();
} else {

Notification({
notificationContent: { text: 'Stack creation failed. Please try again.' },
notificationContent: { text: resp },
notificationProps: {
position: 'bottom-center',
hideProgressBar: true
Expand Down
2 changes: 1 addition & 1 deletion ui/src/components/ContentMapper/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ const Fields: MappingFields = {
'taxonomy':{
label: 'Taxonomy',
options: {'Taxonomy':'taxonomy'},
type:'taxonomy'
type:''
}
}
type contentMapperProps = {
Expand Down
32 changes: 30 additions & 2 deletions ui/src/components/DestinationStack/Actions/LoadStacks.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,15 @@ interface LoadFileFormatProps {
handleStepChange: (stepIndex: number, closeStep?: boolean) => void;
}

interface ErrorObject {
error_message?: string;
errors?: Errors;
}
interface Errors {
org_uid?: string[];
}


const defaultStack = {
description: 'Created from Migration Destination Stack Step',
locale: '',
Expand Down Expand Up @@ -108,6 +117,21 @@ const LoadStacks = (props: LoadFileFormatProps) => {
// setAllStack(newMigrationData?.destination_stack?.stackArray)
}, [newMigrationData?.destination_stack?.selectedStack]);

/**
* Function to format the error message
*/
const formatErrorMessage = (errorData: ErrorObject) => {
let message = errorData.error_message;

if (errorData.errors) {
Object.entries(errorData.errors).forEach(([key, value]) => {
message += `\n${key}: ${(value as string[]).join(", ")}`;
});
}

return message;
}

//Handle new stack details
const handleOnSave = async (data: Stack) => {
try {
Expand Down Expand Up @@ -159,8 +183,12 @@ const LoadStacks = (props: LoadFileFormatProps) => {
setIsStackLoading(false);
return true;
}
} catch (error) {
return error;
else {
const errorMessage = formatErrorMessage(resp?.data?.data);
return errorMessage;
}
} catch (error: any) {
return error?.response?.data;
}
};

Expand Down
Loading
Loading