From 2f3fe21dd4e1276b86e80fee87240945d079131f Mon Sep 17 00:00:00 2001 From: AaronAgility Date: Wed, 19 Nov 2025 11:45:01 -0500 Subject: [PATCH 01/19] Reset processed page IDs --- src/core/logs.ts | 4 --- .../pushers/page-pusher/process-sitemap.ts | 7 ++++++ src/lib/pushers/page-pusher/push-pages.ts | 23 ++++++++++++----- .../pushers/page-pusher/sitemap-hierarchy.ts | 25 +++++++++++++------ 4 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/core/logs.ts b/src/core/logs.ts index 8732117..51511ed 100644 --- a/src/core/logs.ts +++ b/src/core/logs.ts @@ -922,10 +922,6 @@ export class Logs { // Log to file using logger summary this.summary(this.operationType, totalSuccessful, totalFailed, 0); - // Console output - console.log(ansiColors.cyan("\nSummary:")); - console.log(`Processed ${results.length} GUID/locale combinations`); - console.log(`${totalSuccessful} successful, ${totalFailed} failed`); console.log(`Total time: ${timeDisplay}`); // Success/failure message diff --git a/src/lib/pushers/page-pusher/process-sitemap.ts b/src/lib/pushers/page-pusher/process-sitemap.ts index db2f98d..8437015 100644 --- a/src/lib/pushers/page-pusher/process-sitemap.ts +++ b/src/lib/pushers/page-pusher/process-sitemap.ts @@ -36,6 +36,13 @@ interface Props { // This is separate from pagesInProgress which tracks concurrent processing const processedPageIDs = new Set(); +/** + * Reset the processed page IDs tracking - should be called at the start of each pushPages operation + */ +export function resetProcessedPageIDs(): void { + processedPageIDs.clear(); +} + export async function processSitemap({ channel, pageMapper, diff --git a/src/lib/pushers/page-pusher/push-pages.ts b/src/lib/pushers/page-pusher/push-pages.ts index a5ac973..468c9ce 100644 --- a/src/lib/pushers/page-pusher/push-pages.ts +++ b/src/lib/pushers/page-pusher/push-pages.ts @@ -3,7 +3,7 @@ import { state, getApiClient, getLoggerForGuid } from "core/state"; import { PusherResult } from "../../../types/sourceData"; import { SitemapHierarchy } from "lib/pushers/page-pusher/sitemap-hierarchy"; import { PageMapper } from "lib/mappers/page-mapper"; -import { processSitemap } from "./process-sitemap"; +import { processSitemap, resetProcessedPageIDs } from "./process-sitemap"; import ansiColors from "ansi-colors"; export async function pushPages( @@ -24,6 +24,9 @@ export async function pushPages( const sitemapHierarchy = new SitemapHierarchy(); + // Reset processed page IDs tracking for this locale + resetProcessedPageIDs(); + const sitemaps = sitemapHierarchy.loadAllSitemaps(sourceGuid[0], locale); const channels = Object.keys(sitemaps); @@ -40,6 +43,12 @@ export async function pushPages( for (const channel of channels) { const sitemap = sitemaps[channel]; + // Skip if sitemap is null or empty + if (!sitemap || sitemap.length === 0) { + console.log(ansiColors.yellow(`⚠️ Skipping channel ${channel} - no sitemap data found for locale ${locale}`)); + continue; + } + const { sourceGuid, targetGuid, overwrite } = state; const apiClient = getApiClient(); @@ -59,12 +68,14 @@ export async function pushPages( logger }) - successful = res.successful; - failed = res.failed; - skipped = res.skipped; - publishableIds = res.publishableIds; + successful += res.successful; + failed += res.failed; + skipped += res.skipped; + if (res.publishableIds && res.publishableIds.length > 0) { + publishableIds.push(...res.publishableIds); + } - if (failed > 0) { + if (res.failed > 0) { status = "error"; } diff --git a/src/lib/pushers/page-pusher/sitemap-hierarchy.ts b/src/lib/pushers/page-pusher/sitemap-hierarchy.ts index dc73ca7..b4cd5eb 100644 --- a/src/lib/pushers/page-pusher/sitemap-hierarchy.ts +++ b/src/lib/pushers/page-pusher/sitemap-hierarchy.ts @@ -24,13 +24,24 @@ export class SitemapHierarchy { const sitemaps: { [key: string]: SitemapNode[] | null } = {}; - fs.readdirSync(sitemapDir).forEach(fileName => { - if (!fileName.endsWith('.json')) { - return; // Skip non-JSON files - } - const channel = path.basename(fileName, '.json'); - sitemaps[channel] = this.loadNestedSitemap(path.join(sitemapDir, fileName)); - }); + // Check if directory exists before trying to read it + if (!fs.existsSync(sitemapDir)) { + console.warn(`⚠️ Nested sitemap directory not found for locale ${locale}: ${sitemapDir}`); + return sitemaps; // Return empty object if directory doesn't exist + } + + try { + fs.readdirSync(sitemapDir).forEach(fileName => { + if (!fileName.endsWith('.json')) { + return; // Skip non-JSON files + } + const channel = path.basename(fileName, '.json'); + sitemaps[channel] = this.loadNestedSitemap(path.join(sitemapDir, fileName)); + }); + } catch (error: any) { + console.error(`Error reading sitemap directory ${sitemapDir}: ${error.message}`); + return sitemaps; // Return empty object on error + } return sitemaps; } From 31970cfd6f9a359336524ae61964f373696b5633 Mon Sep 17 00:00:00 2001 From: AaronAgility Date: Wed, 19 Nov 2025 11:46:51 -0500 Subject: [PATCH 02/19] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index aeefb4f..2dd9daf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agility/cli", - "version": "1.0.0", + "version": "1.0.0-beta.10", "description": "Agility CLI for working with your content. (Public Beta)", "repository": { "type": "git", From b4d2225e87a0bbf66217352bf8bcbdc58bf3e32f Mon Sep 17 00:00:00 2001 From: AaronAgility Date: Wed, 19 Nov 2025 11:50:46 -0500 Subject: [PATCH 03/19] Update orchestrate-pushers.ts --- src/lib/pushers/orchestrate-pushers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/pushers/orchestrate-pushers.ts b/src/lib/pushers/orchestrate-pushers.ts index 0896718..a849ca4 100644 --- a/src/lib/pushers/orchestrate-pushers.ts +++ b/src/lib/pushers/orchestrate-pushers.ts @@ -287,7 +287,7 @@ export class Pushers { (Array.isArray(elementData) && elementData.length === 0) || !elements.some((element) => config.elements.includes(element)) ) { - console.log(ansiColors.gray(`Skipping ${config.description} - no data or not requested`)); + console.log(ansiColors.yellow(`⚠️ Skipping ${config.description} for locale ${locale} - no data or filtered by --locales`)); return; } From 2e55dcad0bfa348f4e82d596ba367f7613bcd30f Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 23 Dec 2025 11:49:50 -0500 Subject: [PATCH 04/19] Fix gallery mapper and model pusher issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove debugger statement from gallery-mapper.ts - Fix RichTextArea special case logic in model-pusher.ts to check for missing mapping and existing target model 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- src/lib/mappers/gallery-mapper.ts | 1 - src/lib/pushers/model-pusher.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/mappers/gallery-mapper.ts b/src/lib/mappers/gallery-mapper.ts index 09c734b..227a5a3 100644 --- a/src/lib/mappers/gallery-mapper.ts +++ b/src/lib/mappers/gallery-mapper.ts @@ -29,7 +29,6 @@ export class GalleryMapper { } getGalleryMapping(gallery: mgmtApi.assetMediaGrouping, type: 'source' | 'target'): GalleryMapping | null { - debugger; const mapping = this.mappings.find((m: GalleryMapping) => type === 'source' ? m.sourceMediaGroupingID === gallery.mediaGroupingID : m.targetMediaGroupingID === gallery.mediaGroupingID ); diff --git a/src/lib/pushers/model-pusher.ts b/src/lib/pushers/model-pusher.ts index df0f1d8..384a9bd 100644 --- a/src/lib/pushers/model-pusher.ts +++ b/src/lib/pushers/model-pusher.ts @@ -47,7 +47,7 @@ export async function pushModels(sourceData: mgmtApi.Model[], targetData: mgmtAp // special case for the default RichTextArea model - const defaultRichTextArea = model.referenceName === 'RichTextArea' && hasSourceChanged && hasTargetChanged; + const defaultRichTextArea = model.referenceName === 'RichTextArea' && !mapping && targetModel; if(defaultRichTextArea){ // force create the mapping for the default RichTextArea model referenceMapper.addMapping(model, targetModel); From 63f7571eadc5a08395f7e119683ece232a7f01c9 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 23 Dec 2025 12:22:39 -0500 Subject: [PATCH 05/19] version bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2dd9daf..246159b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agility/cli", - "version": "1.0.0-beta.10", + "version": "1.0.0-beta.11", "description": "Agility CLI for working with your content. (Public Beta)", "repository": { "type": "git", From 01c79cff7f559cfa379c3971f1ad1cfc3cab3d8d Mon Sep 17 00:00:00 2001 From: AaronAgility Date: Tue, 6 Jan 2026 13:18:11 -0500 Subject: [PATCH 06/19] Update package.json --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 246159b..9c533ef 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agility/cli", - "version": "1.0.0-beta.11", + "version": "1.0.0-beta.12", "description": "Agility CLI for working with your content. (Public Beta)", "repository": { "type": "git", @@ -76,4 +76,4 @@ "ts-node": "^10.9.2", "typescript": "^5.8.3" } -} +} \ No newline at end of file From 9586a7a5beea10b24507c8f12b61f5cd897324cd Mon Sep 17 00:00:00 2001 From: AaronAgility Date: Thu, 15 Jan 2026 16:42:34 -0500 Subject: [PATCH 07/19] Full workflow service, tests, integration with sync --- jest.config.js | 13 +- package-lock.json | 28 +- package.json | 9 +- src/core/auth.ts | 36 +- src/core/batch-workflows.ts | 139 ++++++++ src/core/index.ts | 38 ++- src/core/pull.ts | 42 ++- src/core/push.ts | 64 +++- src/core/state.ts | 54 ++- src/core/system-args.ts | 50 ++- src/index.ts | 108 +++++- src/lib/mappers/content-item-mapper.ts | 28 ++ src/lib/mappers/mapping-reader.ts | 143 ++++++++ src/lib/mappers/mapping-version-updater.ts | 309 ++++++++++++++++++ src/lib/mappers/page-mapper.ts | 28 ++ src/lib/publishers/index.ts | 55 +++- src/lib/pushers/content-pusher/util/types.ts | 2 +- src/lib/pushers/model-pusher.ts | 15 +- src/lib/pushers/orchestrate-pushers.ts | 2 +- src/lib/pushers/page-pusher/push-pages.ts | 2 +- src/lib/pushers/template-pusher.ts | 19 +- src/lib/shared/get-fetch-api-status.ts | 88 +++++ src/lib/shared/index.ts | 23 ++ .../shared/source-publish-status-checker.ts | 153 +++++++++ src/lib/workflows/index.ts | 24 ++ src/lib/workflows/list-mappings.ts | 38 +++ src/lib/workflows/process-batches.ts | 237 ++++++++++++++ src/lib/workflows/refresh-mappings.ts | 173 ++++++++++ src/lib/workflows/workflow-helpers.ts | 67 ++++ src/lib/workflows/workflow-operation.ts | 282 ++++++++++++++++ src/lib/workflows/workflow-options.ts | 89 +++++ src/lib/workflows/workflow-orchestrator.ts | 76 +++++ src/tests/setup.ts | 20 ++ .../fetch-api-status.integration.test.ts | 120 +++++++ .../batch-workflows.integration.test.ts | 155 +++++++++ src/types/index.ts | 5 +- src/types/sourceData.ts | 2 +- src/types/workflows.ts | 164 ++++++++++ yarn.lock | 21 +- 39 files changed, 2820 insertions(+), 101 deletions(-) create mode 100644 src/core/batch-workflows.ts create mode 100644 src/lib/mappers/mapping-reader.ts create mode 100644 src/lib/mappers/mapping-version-updater.ts create mode 100644 src/lib/shared/get-fetch-api-status.ts create mode 100644 src/lib/shared/source-publish-status-checker.ts create mode 100644 src/lib/workflows/index.ts create mode 100644 src/lib/workflows/list-mappings.ts create mode 100644 src/lib/workflows/process-batches.ts create mode 100644 src/lib/workflows/refresh-mappings.ts create mode 100644 src/lib/workflows/workflow-helpers.ts create mode 100644 src/lib/workflows/workflow-operation.ts create mode 100644 src/lib/workflows/workflow-options.ts create mode 100644 src/lib/workflows/workflow-orchestrator.ts create mode 100644 src/tests/setup.ts create mode 100644 src/tests/shared/fetch-api-status.integration.test.ts create mode 100644 src/tests/workflows/batch-workflows.integration.test.ts create mode 100644 src/types/workflows.ts diff --git a/jest.config.js b/jest.config.js index 7a425f4..e5d8be5 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,6 +1,15 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', - testMatch: ['**/src/tests/**/*.ts'], - testPathIgnorePatterns: ['/node_modules/', '/dist/', '/src/index.ts'], + // Default: unit tests only (exclude integration tests) + testMatch: ['**/src/tests/**/*.test.ts'], + testPathIgnorePatterns: ['/node_modules/', '/dist/', '/src/index.ts', 'integration\\.test\\.ts'], + setupFilesAfterEnv: ['/src/tests/setup.ts'], + // Map TypeScript path aliases to actual paths + moduleNameMapper: { + '^core/(.*)$': '/src/core/$1', + '^core$': '/src/core', + '^lib/(.*)$': '/src/lib/$1', + '^types/(.*)$': '/src/types/$1', + }, }; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 5c64474..7a9be0a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,22 +1,23 @@ { "name": "@agility/cli", - "version": "1.0.0-beta.9.16", + "version": "1.0.0-beta.13", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@agility/cli", - "version": "1.0.0-beta.9.16", + "version": "1.0.0-beta.13", "license": "ISC", "dependencies": { "@agility/content-fetch": "^2.0.10", "@agility/content-sync": "^1.2.0", - "@agility/management-sdk": "^0.1.35", + "@agility/management-sdk": "^0.1.38", "ansi-colors": "^4.1.3", "blessed": "^0.1.81", "blessed-contrib": "^4.11.0", "cli-progress": "^3.11.2", "date-fns": "^4.1.0", + "form-data": "^4.0.5", "fuzzy": "^0.1.3", "inquirer": "^8.0.0", "inquirer-checkbox-plus-prompt": "^1.4.2", @@ -82,7 +83,6 @@ "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", "license": "MIT", - "peer": true, "dependencies": { "follow-redirects": "^1.14.0" } @@ -101,9 +101,9 @@ } }, "node_modules/@agility/management-sdk": { - "version": "0.1.35", - "resolved": "https://registry.npmjs.org/@agility/management-sdk/-/management-sdk-0.1.35.tgz", - "integrity": "sha512-js4EYPm6FQtmao0kDT3y6w3Azh+PsHaGOMpjEdt/thtDm+T1szeZ0CRlONlpkN6qi4bNlWIA6/SeCoqhpOkVaA==", + "version": "0.1.38", + "resolved": "https://registry.npmjs.org/@agility/management-sdk/-/management-sdk-0.1.38.tgz", + "integrity": "sha512-g6/hNgCjf+uzcJbkPaxeWqwaAid50tMnRW0KYQ4J/fGuJqtcZG0et0X3m76Ev3yUqnkpIO/+C5zMIQMghN3/oQ==", "license": "MIT", "dependencies": { "axios": "^0.27.2" @@ -154,7 +154,6 @@ "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.27.1", @@ -1463,7 +1462,6 @@ "integrity": "sha512-hcxGs9TfQGghOM8atpRT+bBMUX7V8WosdYt98bQ59wUToJck55eCOlemJ+0FpOZOQw5ff7LSi9+IO56KvYEFyQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~5.26.4" } @@ -1952,7 +1950,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001726", "electron-to-chromium": "^1.5.173", @@ -3020,9 +3017,9 @@ } }, "node_modules/form-data": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.3.tgz", - "integrity": "sha512-qsITQPfmvMOSAdeyZ+12I1c+CKSstAFAwu+97zrnWAbIr5u8wfsExUzCesVLC8NgHuRUqNN4Zy6UPWUTRGslcA==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -3397,7 +3394,6 @@ "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.6.tgz", "integrity": "sha512-M1WuAmb7pn9zdFRtQYk26ZBoY043Sse0wVDdk4Bppr+JOXyQYybdtvK+l9wUibhtjdjvtoiNy8tk+EgsYIUqKg==", "license": "MIT", - "peer": true, "dependencies": { "ansi-escapes": "^4.2.1", "chalk": "^4.1.1", @@ -4465,7 +4461,6 @@ "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -6103,7 +6098,6 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", "license": "MIT", - "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -7596,7 +7590,6 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -7704,7 +7697,6 @@ "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/package.json b/package.json index 9c533ef..2d10907 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agility/cli", - "version": "1.0.0-beta.12", + "version": "1.0.0-beta.13", "description": "Agility CLI for working with your content. (Public Beta)", "repository": { "type": "git", @@ -17,6 +17,8 @@ "postbuild": "chmod +x dist/index.js", "refresh": "rm -rf ./node_modules ./package-lock.json && npm install", "test": "jest", + "test:unit": "jest", + "test:integration": "jest --testMatch=\"**/*.integration.test.ts\" --testPathIgnorePatterns=\"/node_modules/|/dist/|/src/index.ts\"", "debug": "node --inspect-brk -r ts-node/register src/index.ts" }, "keywords": [ @@ -45,12 +47,13 @@ "dependencies": { "@agility/content-fetch": "^2.0.10", "@agility/content-sync": "^1.2.0", - "@agility/management-sdk": "^0.1.35", + "@agility/management-sdk": "^0.1.38", "ansi-colors": "^4.1.3", "blessed": "^0.1.81", "blessed-contrib": "^4.11.0", "cli-progress": "^3.11.2", "date-fns": "^4.1.0", + "form-data": "^4.0.5", "fuzzy": "^0.1.3", "inquirer": "^8.0.0", "inquirer-checkbox-plus-prompt": "^1.4.2", @@ -76,4 +79,4 @@ "ts-node": "^10.9.2", "typescript": "^5.8.3" } -} \ No newline at end of file +} diff --git a/src/core/auth.ts b/src/core/auth.ts index fab976a..9d60d92 100644 --- a/src/core/auth.ts +++ b/src/core/auth.ts @@ -119,16 +119,33 @@ export class Auth { async logout() { const env = this.getEnv(); - const key = this.getEnvKey(env); + const auth0Key = this.getEnvKey(env); + const patKey = `cli-pat-token:${env}`; + + let removedAny = false; + try { - const removed = await keytar.deletePassword(SERVICE_NAME, key); - if (removed) { - console.log(`Logged out from ${env} environment.`); + // Remove Auth0 token + const removedAuth0 = await keytar.deletePassword(SERVICE_NAME, auth0Key); + if (removedAuth0) { + console.log(`✓ Removed Auth0 token for ${env} environment.`); + removedAny = true; + } + + // Remove PAT token + const removedPAT = await keytar.deletePassword(SERVICE_NAME, patKey); + if (removedPAT) { + console.log(`✓ Removed Personal Access Token for ${env} environment.`); + removedAny = true; + } + + if (removedAny) { + console.log(ansiColors.green(`\n🔓 Successfully logged out from ${env} environment.`)); } else { - console.log(`No token found in ${env} environment.`); + console.log(ansiColors.yellow(`No tokens found in ${env} environment.`)); } } catch (err) { - console.error(`❌ Failed to delete token:`, err); + console.error(`❌ Failed to delete tokens:`, err); } exit(); } @@ -871,13 +888,6 @@ export class Auth { async validateCommand(commandType: "pull" | "sync" | "clean" | "interactive" | "push"): Promise { const missingFields: string[] = []; - // Validate that --publish flag is only used with sync command - if (state.publish && commandType !== "sync") { - console.log(ansiColors.red(`\n❌ The --publish flag is only available for sync commands.`)); - console.log(ansiColors.yellow(`💡 Use: agility sync --sourceGuid="source" --targetGuid="target" --publish`)); - return false; - } - // Check command-specific requirements switch (commandType) { case "pull": diff --git a/src/core/batch-workflows.ts b/src/core/batch-workflows.ts new file mode 100644 index 0000000..69dde20 --- /dev/null +++ b/src/core/batch-workflows.ts @@ -0,0 +1,139 @@ +/** + * Batch Workflows Core Service + * + * Core batch workflow operations using the SDK's + * BatchWorkflowContent and BatchWorkflowPages methods. + * + * Supports: Publish, Unpublish, Approve, Decline, RequestApproval + */ + +import { state, getApiClient } from './state'; +import ansiColors from 'ansi-colors'; +import { WorkflowOperationType, BatchWorkflowResult } from '../types'; +import { getOperationName } from '../lib/workflows/workflow-helpers'; + +// Re-export types for convenience +export { WorkflowOperationType, BatchWorkflowResult }; + +// Re-export helpers from workflows folder +export { getOperationName, getOperationVerb, getOperationIcon } from '../lib/workflows/workflow-helpers'; +export { parseWorkflowOptions, parseOperationType } from '../lib/workflows/workflow-options'; + +/** + * Batch size for processing - prevents API throttling + */ +const BATCH_SIZE = 250; + +/** + * Extract detailed error message from various error formats + */ +function extractErrorDetails(error: any): string { + // Check for nested error structures (common in SDK exceptions) + if (error.innerError) { + return extractErrorDetails(error.innerError); + } + + // Check for response data from API + if (error.response?.data) { + if (typeof error.response.data === 'string') { + return error.response.data; + } + if (error.response.data.message) { + return error.response.data.message; + } + if (error.response.data.error) { + return error.response.data.error; + } + return JSON.stringify(error.response.data); + } + + // Check for status code + if (error.response?.status) { + return `HTTP ${error.response.status}: ${error.response.statusText || 'Unknown error'}`; + } + + // Check for message property + if (error.message) { + return error.message; + } + + // Fallback + return String(error) || 'Unknown workflow error'; +} + +/** + * Item type for batch workflow operations + */ +export type BatchItemType = 'content' | 'pages'; + +/** + * Unified batch workflow operation for content items or pages + * + * @param ids - Array of IDs to process + * @param locale - Target locale + * @param operation - Workflow operation type + * @param type - Item type: 'content' or 'pages' + * @returns Promise with batch result + */ +export async function batchWorkflow( + ids: number[], + locale: string, + operation: WorkflowOperationType, + type: BatchItemType +): Promise { + const label = type === 'content' ? 'content items' : 'pages'; + + try { + const apiClient = getApiClient(); + const targetGuid = state.targetGuid; + + if (!apiClient) { + throw new Error('API client not available in state'); + } + if (!targetGuid || targetGuid.length === 0) { + throw new Error('Target GUID not available in state'); + } + if (!locale) { + throw new Error('Locale not available in state'); + } + if (!ids || ids.length === 0) { + throw new Error(`${label} IDs array is empty`); + } + + // const operationName = getOperationName(operation); + + // Log the attempt for debugging + // if (state.verbose) { + // console.log(ansiColors.gray(`${operationName}ing ${ids.length} ${label} to ${targetGuid[0]} (${locale})...`)); + // } + + // Call appropriate SDK method based on type + const processedIds = type === 'content' + ? await apiClient.contentMethods.batchWorkflowContent(ids, targetGuid[0], locale, operation, false) + : await apiClient.pageMethods.batchWorkflowPages(ids, targetGuid[0], locale, operation, false); + + return { + success: true, + processedIds, + failedCount: 0 + }; + } catch (error: any) { + return { + success: false, + processedIds: [], + failedCount: ids.length, + error: extractErrorDetails(error) + }; + } +} + +/** + * Create batches of items for processing + */ +export function createBatches(items: T[], batchSize: number = BATCH_SIZE): T[][] { + const batches: T[][] = []; + for (let i = 0; i < items.length; i += batchSize) { + batches.push(items.slice(i, i + batchSize)); + } + return batches; +} diff --git a/src/core/index.ts b/src/core/index.ts index 4b64c15..4845c7c 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -15,6 +15,42 @@ export { normalizeProcessArgs, normalizeArgv } from './arg-normalizer'; // Publishing service export { PublishService, type PublishResult, type PublishOptions } from './publish'; +// Workflow operation standalone module +export { WorkflowOperation } from '../lib/workflows'; + +// Batch workflows service - core batch operations +export { + batchWorkflow, + type BatchItemType, + createBatches +} from './batch-workflows'; + +// Workflow module - orchestration, options, helpers +export { + workflowOrchestrator, + parseWorkflowOptions, + parseOperationType, + getOperationName, + getOperationVerb, + getOperationIcon +} from '../lib/workflows'; + +// Re-export all workflow types from central types folder +export { + WorkflowOperationType, + BatchWorkflowResult, + WorkflowOrchestratorResult, + WorkflowOptions, + WorkflowOperationResult, + ContentMapping, + PageMapping, + MappingReadResult, + MappingUpdateResult, + ItemState, + SourceItemData, + PublishStatusResult +} from '../types'; + // Content and data services export { content } from './content'; export { assets } from './assets'; @@ -22,4 +58,4 @@ export { fileOperations } from './fileOperations'; export { getApiClient } from './state'; // File system integration -// Note: store-interface-filesystem uses module.exports, import directly if needed +// Note: store-interface-filesystem uses module.exports, import directly if needed diff --git a/src/core/pull.ts b/src/core/pull.ts index 361ba54..d35c793 100644 --- a/src/core/pull.ts +++ b/src/core/pull.ts @@ -3,6 +3,7 @@ import * as fs from "fs"; import { getState, initializeLogger, finalizeLogger, getLogger } from "./state"; import ansiColors from "ansi-colors"; import { markPullStart, clearTimestamps } from "../lib/incremental"; +import { waitForFetchApiSync } from "../lib/shared/get-fetch-api-status"; import { Downloader } from "../lib/downloaders/orchestrate-downloaders"; @@ -65,6 +66,20 @@ export class Pull { const totalStartTime = Date.now(); try { + // Wait for Fetch API sync to complete before pulling (only for standalone pull operations) + // This ensures we're pulling the latest data from the CDN + // Skip when called from push - the refresh-mappings workflow handles this separately + if (!fromPush) { + for (const guid of allGuids) { + try { + await waitForFetchApiSync(guid, 'fetch', false); + } catch (error: any) { + // Log warning but don't fail the pull - the API might not support this endpoint yet + console.log(ansiColors.yellow(`⚠️ Could not check Fetch API status for ${guid}: ${error.message}`)); + } + } + } + // Execute concurrent downloads for all GUIDs, locales and channels (sitemaps) const results = await this.downloader.instanceOrchestrator(fromPush); @@ -84,21 +99,20 @@ export class Pull { const success = totalFailed === 0; - // Use the orchestrator summary function to handle all completion logic - const logger = getLogger(); - if (logger) { - // Collect log file paths - const logFilePaths = results - .map(res => res.logFilePath) - .filter(path => path); - - logger.orchestratorSummary(results, totalElapsedTime, success, logFilePaths); - } - - finalizeLogger(); // Finalize global logger if it exists - - // Only exit if not called from push operation + // Only show completion summary and finalize logger for standalone pull operations + // When called from push/sync, the parent operation handles its own summary if (!fromPush) { + const logger = getLogger(); + if (logger) { + // Collect log file paths + const logFilePaths = results + .map(res => res.logFilePath) + .filter(path => path); + + logger.orchestratorSummary(results, totalElapsedTime, success, logFilePaths); + } + + finalizeLogger(); // Finalize global logger if it exists process.exit(success ? 0 : 1); } diff --git a/src/core/push.ts b/src/core/push.ts index 1b8ad6b..6e12a30 100644 --- a/src/core/push.ts +++ b/src/core/push.ts @@ -1,6 +1,6 @@ import * as path from "path"; import * as fs from "fs"; -import { getState, initializeLogger, finalizeLogger, getLogger, state } from "./state"; +import { getState, initializeLogger, finalizeLogger, getLogger, state, setState } from "./state"; import ansiColors from "ansi-colors"; import { markPushStart, clearTimestamps } from "../lib/incremental"; @@ -16,7 +16,7 @@ export class Push { } async pushInstances(fromSync: boolean = false): Promise<{ success: boolean; results: any[]; elapsedTime: number }> { - const { isSync, sourceGuid, targetGuid, models, modelsWithDeps } = state; + const { isSync, sourceGuid, targetGuid, models, modelsWithDeps, autoPublish } = state; // Initialize logger for push operation // Determine if this is a sync operation by checking if both source and target GUIDs exist @@ -107,6 +107,11 @@ export class Push { } finalizeLogger(); // Finalize global logger if it exists + + // Auto-publish if enabled and sync was successful + if (isSync && autoPublish && success) { + await this.executeAutoPublish(results, autoPublish); + } // Only exit if not called from another operation @@ -127,6 +132,61 @@ export class Push { } } + /** + * Execute auto-publish after sync completes + */ + private async executeAutoPublish(results: PushResults[], autoPublishMode: string): Promise { + // Collect all publishable IDs from sync results + const allContentIds: number[] = []; + const allPageIds: number[] = []; + + for (const result of results) { + if (result.publishableContentIds && result.publishableContentIds.length > 0) { + allContentIds.push(...result.publishableContentIds); + } + if (result.publishablePageIds && result.publishablePageIds.length > 0) { + allPageIds.push(...result.publishablePageIds); + } + } + + // Determine what to publish based on mode + const publishContent = autoPublishMode === 'content' || autoPublishMode === 'both'; + const publishPages = autoPublishMode === 'pages' || autoPublishMode === 'both'; + + const contentIdsToPublish = publishContent ? allContentIds : []; + const pageIdsToPublish = publishPages ? allPageIds : []; + + // Check if there's anything to publish + if (contentIdsToPublish.length === 0 && pageIdsToPublish.length === 0) { + console.log(ansiColors.yellow('\n⚠️ Auto-publish: No items to publish from sync operation')); + return; + } + + console.log(ansiColors.cyan('\n' + '═'.repeat(50))); + console.log(ansiColors.cyan('🚀 AUTO-PUBLISH')); + console.log(ansiColors.cyan('═'.repeat(50))); + console.log(ansiColors.gray(`Mode: ${autoPublishMode}`)); + console.log(ansiColors.gray(`Content items to publish: ${contentIdsToPublish.length}`)); + console.log(ansiColors.gray(`Pages to publish: ${pageIdsToPublish.length}`)); + + try { + // Set explicit IDs in state for the workflow operation + setState({ + explicitContentIDs: contentIdsToPublish, + explicitPageIDs: pageIdsToPublish, + operationType: 'publish' + }); + + // Import and execute workflow operation + const { WorkflowOperation } = await import('../lib/workflows'); + const workflowOp = new WorkflowOperation(); + await workflowOp.executeFromMappings(); + + } catch (error: any) { + console.error(ansiColors.red(`\n❌ Auto-publish failed: ${error.message}`)); + } + } + private async handleResetFlag(guid: string): Promise { const state = getState(); const guidFolderPath = path.join(process.cwd(), state.rootPath, guid); diff --git a/src/core/state.ts b/src/core/state.ts index 45dc6af..8adb934 100644 --- a/src/core/state.ts +++ b/src/core/state.ts @@ -7,6 +7,7 @@ import * as mgmtApi from '@agility/management-sdk'; import fs from 'fs'; import path from 'path'; import { Logs, OperationType, EntityType } from './logs'; +import { Options } from '@agility/management-sdk'; export interface State { // Environment modes @@ -45,8 +46,14 @@ export interface State { reset: boolean; update: boolean; - // Publishing control - publish: boolean; + // Workflow operation control + operationType?: string; // Workflow operation: publish, unpublish, approve, decline, requestApproval + dryRun: boolean; // Preview mode - show what would be processed without executing + autoPublish: string; // Auto-publish after sync: 'content', 'pages', 'both', or '' (disabled) + + // Explicit ID overrides (bypass mappings lookup) + explicitContentIDs: number[]; // Target content IDs to process directly + explicitPageIDs: number[]; // Target page IDs to process directly // Model-specific models: string; @@ -127,9 +134,12 @@ export const state: State = { force: false, reset: false, update: true, + dryRun: false, + autoPublish: '', // Empty string = disabled - // Publishing control - publish: false, + // Explicit ID overrides (bypass mappings lookup) + explicitContentIDs: [], + explicitPageIDs: [], // Model-specific models: "", @@ -230,8 +240,23 @@ export function setState(argv: any) { if (argv.reset !== undefined) state.reset = argv.reset; if (argv.update !== undefined) state.update = argv.update; - // Publishing control - if (argv.publish !== undefined) state.publish = argv.publish; + // Workflow operation control + if (argv.operationType !== undefined) state.operationType = argv.operationType; + if (argv.dryRun !== undefined) state.dryRun = argv.dryRun; + + // Explicit ID overrides - parse comma-separated strings into number arrays + if (argv.contentIDs !== undefined && argv.contentIDs !== "") { + state.explicitContentIDs = String(argv.contentIDs) + .split(',') + .map((id: string) => parseInt(id.trim(), 10)) + .filter((id: number) => !isNaN(id) && id > 0); + } + if (argv.pageIDs !== undefined && argv.pageIDs !== "") { + state.explicitPageIDs = String(argv.pageIDs) + .split(',') + .map((id: string) => parseInt(id.trim(), 10)) + .filter((id: number) => !isNaN(id) && id > 0); + } // Model-specific if (argv.models !== undefined) state.models = argv.models; @@ -450,8 +475,13 @@ export function resetState() { state.reset = false; state.update = true; - // Publishing control - state.publish = false; + // Workflow operation control + state.operationType = undefined; + state.dryRun = false; + + // Explicit ID overrides + state.explicitContentIDs = []; + state.explicitPageIDs = []; // Model-specific state.models = ""; @@ -497,9 +527,15 @@ export function getApiClient(): mgmtApi.ApiClient { // Create new client using current auth state if (!state.mgmtApiOptions) { - throw new Error('Management API options not initialized. Call auth.init() first.'); + // throw new Error('Management API options not initialized. Call auth.init() first.'); } + if(!state.mgmtApiOptions && !state.token) { + throw new Error('Management API options not initialized. Call auth.init() first.'); + } else if (!state.mgmtApiOptions && state.token) { + state.mgmtApiOptions = new Options(); + state.mgmtApiOptions.token = state.token; + } // Create and cache the client state.cachedApiClient = new mgmtApi.ApiClient(state.mgmtApiOptions); return state.cachedApiClient; diff --git a/src/core/system-args.ts b/src/core/system-args.ts index 359c3b3..c9c2f7a 100644 --- a/src/core/system-args.ts +++ b/src/core/system-args.ts @@ -127,6 +127,29 @@ export const systemArgs = { type: "boolean" as const, default: false, }, + dryRun: { + describe: "Dry run mode: show what items would be processed without executing the operation. Useful for previewing workflow operations.", + demandOption: false, + type: "boolean" as const, + alias: ["dry-run", "dryrun", "DryRun", "DRY_RUN"], + default: false, + }, + + // **Explicit ID Override for Workflow Operations** + contentIDs: { + describe: "Comma-separated list of target content IDs to process. Bypasses mappings lookup when provided (e.g., --contentIDs=121,1221,345).", + demandOption: false, + alias: ["content-ids", "contentIds", "ContentIDs", "CONTENTIDS"], + type: "string" as const, + default: "", + }, + pageIDs: { + describe: "Comma-separated list of target page IDs to process. Bypasses mappings lookup when provided (e.g., --pageIDs=12,11,45).", + demandOption: false, + alias: ["page-ids", "pageIds", "PageIDs", "PAGEIDS"], + type: "string" as const, + default: "", + }, // Instance identification args sourceGuid: { @@ -167,12 +190,20 @@ export const systemArgs = { default: false }, - // Publishing args - publish: { - describe: "For sync commands only: automatically publish synced content items and pages after successful sync operation. Enables batch publishing for streamlined deployment workflow. Default: false.", - type: "boolean" as const, - alias: ["publish", "Publish", "PUBLISH"], - default: false + // Auto-publish after sync + autoPublish: { + describe: "Automatically publish content and/or pages after sync completes. Options: 'content' (publish only content), 'pages' (publish only pages), 'both' (publish content and pages). Default: both when flag is provided.", + demandOption: false, + alias: ["auto-publish", "autoPublish", "AutoPublish", "AUTO_PUBLISH"], + type: "string" as const, + coerce: (value: string | boolean) => { + // Handle --autoPublish without value (defaults to 'both') + if (value === true || value === '') return 'both'; + if (value === false) return ''; + const lower = String(value).toLowerCase(); + if (['content', 'pages', 'both'].includes(lower)) return lower; + return 'both'; // Default to 'both' for any other value + } }, }; @@ -190,11 +221,12 @@ export interface SystemArgs { sync?: boolean; clean?: boolean; generate?: boolean; - publish?: boolean; + operationType?: string; // Workflow operation: publish, unpublish, approve, decline, requestApproval test?: boolean; + dryRun?: boolean; // Preview mode - show what would be processed without executing verbose?: boolean; overwrite?: boolean; - force?: boolean; // New: Override target safety conflicts + force?: boolean; // Override target safety conflicts update?: boolean; legacyFolders?: boolean; elements?: string; @@ -205,4 +237,6 @@ export interface SystemArgs { channel?: string; preview?: boolean; rootPath?: string; + contentIDs?: string; // Explicit content IDs (bypasses mappings) + pageIDs?: string; // Explicit page IDs (bypasses mappings) } diff --git a/src/index.ts b/src/index.ts index 656f8da..1d19e85 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,6 +22,7 @@ inquirer.registerPrompt("search-list", searchList); import { Auth, state, setState, resetState, primeFromEnv, systemArgs, normalizeProcessArgs, normalizeArgv } from "./core"; import { Pull } from "./core/pull"; import { Push } from "./core/push"; +import { WorkflowOperation } from "./lib/workflows"; import { initializeLogger, getLogger, finalizeLogger, finalizeAllGuidLoggers } from "./core/state"; @@ -38,9 +39,10 @@ yargs.command({ describe: "Default command - shows available commands", handler: function () { console.log(colors.cyan("\nAvailable commands:")); - console.log(colors.white(" pull - Pull your Agility instance locally")); - console.log(colors.white(" push - Push your instance to a target instance")); - console.log(colors.white(" sync - Sync your instance (alias for push with updates enabled)")); + console.log(colors.white(" pull - Pull your Agility instance locally")); + console.log(colors.white(" push - Push your instance to a target instance")); + console.log(colors.white(" sync - Sync your instance (alias for push with updates enabled)")); + console.log(colors.white(" workflowOperation - Perform workflow operations (publish, unpublish, approve, decline)")); console.log(colors.white("\nFor more information, use: --help")); console.log(""); }, @@ -206,6 +208,106 @@ yargs.command({ } }) +// Workflow operation command - performs workflow operations on content/pages from existing mappings +yargs.command({ + command: "workflows", + aliases: ["workflow"], + describe: "Perform workflow operations (publish, unpublish, approve, decline, requestApproval) on content and pages from existing mappings.", + builder: { + sourceGuid: { + describe: "Source instance GUID (from the original sync).", + demandOption: true, + type: "string", + }, + targetGuid: { + describe: "Target instance GUID to perform workflow operation on.", + demandOption: true, + type: "string", + }, + list: { + describe: "List available mapping pairs instead of running operation.", + type: "boolean", + default: false, + }, + // Workflow operation type for batch workflow operations + operationType: { + describe: "Workflow operation to perform: publish, unpublish, approve, decline, or requestApproval. Used with workflowOperation command.", + type: "string" as const, + alias: ["operation-type", "operationType", "OperationType", "OPERATION_TYPE", "op","type"], + choices: ["publish", "unpublish", "approve", "decline", "requestApproval"], + // default: "publish", + coerce: (value: string) => { + if (!value) return "publish"; + const lower = String(value).toLowerCase(); + // Normalize various input formats + switch (lower) { + case "publish": + case "pub": + return "publish"; + case "unpublish": + case "unpub": + return "unpublish"; + case "approve": + case "app": + return "approve"; + case "decline": + case "dec": + return "decline"; + case "requestapproval": + case "request-approval": + case "request_approval": + case "req": + return "requestApproval"; + default: + return "publish"; + } + } + }, + // System args (commonly repeated across commands) + ...systemArgs + }, + handler: async function (argv) { + resetState(); // Clear any previous command state + + // Normalize argv to handle rich text editor character conversions + argv = normalizeArgv(argv); + + // Prime state from .env file before applying command line args + const envPriming = primeFromEnv(); + if (envPriming.hasEnvFile && envPriming.primedValues.length > 0) { + console.log(colors.cyan(`📄 Found .env file, primed: ${envPriming.primedValues.join(', ')}`)); + } + + setState(argv); + + // If --list flag, just list available mappings + if (argv.list) { + const workflowOp = new WorkflowOperation(); + workflowOp.listMappings(); + return; + } + + auth = new Auth(); + const isAuthorized = await auth.init(); + if (!isAuthorized) { + return; + } + + // Validate command requirements + const isValidCommand = await auth.validateCommand('push'); + if (!isValidCommand) { + return; + } + + const workflowOp = new WorkflowOperation(); + const result = await workflowOp.executeFromMappings(); + + if (!result.success) { + process.exit(1); + } + } +}) + // Normalize process.argv to handle rich text editor character conversions // (e.g., em dashes, curly quotes from Word/Notepad) normalizeProcessArgs(); diff --git a/src/lib/mappers/content-item-mapper.ts b/src/lib/mappers/content-item-mapper.ts index 853fb1b..a949d0d 100644 --- a/src/lib/mappers/content-item-mapper.ts +++ b/src/lib/mappers/content-item-mapper.ts @@ -138,5 +138,33 @@ export class ContentItemMapper { return targetContentItem.properties.versionID > mapping.targetVersionID; } + /** + * Update only the target versionID in a mapping (used after publishing) + * Does NOT update sourceVersionID - that should only change during sync operations + * + * @returns Object with success status and old/new version IDs + */ + updateTargetVersionID(targetContentID: number, newVersionID: number): { + success: boolean; + oldVersionID?: number; + newVersionID?: number; + } { + const mapping = this.getContentItemMappingByContentID(targetContentID, 'target'); + if (!mapping) return { success: false }; + + const oldVersionID = mapping.targetVersionID; + + // Only update if version actually changed + if (oldVersionID !== newVersionID) { + mapping.targetVersionID = newVersionID; + this.saveMapping(); + } + + return { + success: true, + oldVersionID, + newVersionID + }; + } } \ No newline at end of file diff --git a/src/lib/mappers/mapping-reader.ts b/src/lib/mappers/mapping-reader.ts new file mode 100644 index 0000000..fc92f36 --- /dev/null +++ b/src/lib/mappers/mapping-reader.ts @@ -0,0 +1,143 @@ +/** + * Mapping Reader Utility + * + * Reads content and page mappings from the file system to extract target IDs + * for workflow operations. Uses fileOperations for consistent filesystem access. + */ + +import { fileOperations } from '../../core'; +import { state } from '../../core/state'; +import { ContentMapping, PageMapping, MappingReadResult } from '../../types'; + +// Re-export types for convenience +export { ContentMapping, PageMapping, MappingReadResult }; + +/** + * Read all mappings for a source/target GUID pair across all locales + * Uses fileOperations for consistent filesystem access + */ +export function readMappingsForGuidPair( + sourceGuid: string, + targetGuid: string, + locales: string[] +): MappingReadResult { + const result: MappingReadResult = { + contentIds: [], + pageIds: [], + contentMappings: [], + pageMappings: [], + errors: [] + }; + + for (const locale of locales) { + // Use fileOperations for consistent access + const fileOps = new fileOperations(targetGuid, locale); + + // Read content mappings using fileOperations getMappingFile + const contentMappings = fileOps.getMappingFile('item', sourceGuid, targetGuid, locale); + if (contentMappings && contentMappings.length > 0) { + result.contentMappings.push(...contentMappings as ContentMapping[]); + result.contentIds.push(...contentMappings.map((m: ContentMapping) => m.targetContentID)); + } + + // Read page mappings using fileOperations getMappingFile + const pageMappings = fileOps.getMappingFile('page', sourceGuid, targetGuid, locale); + if (pageMappings && pageMappings.length > 0) { + result.pageMappings.push(...pageMappings as PageMapping[]); + result.pageIds.push(...pageMappings.map((m: PageMapping) => m.targetPageID)); + } + } + + // Deduplicate IDs (same content/page might appear in multiple locales) + result.contentIds = Array.from(new Set(result.contentIds)); + result.pageIds = Array.from(new Set(result.pageIds)); + + return result; +} + +/** + * List available mapping directories to discover source/target pairs + * Uses fileOperations for consistent filesystem access + */ +export function listAvailableMappingPairs(): Array<{ sourceGuid: string; targetGuid: string; locales: string[] }> { + const fileOps = new fileOperations('', ''); + const mappingsDir = fileOps.getMappingFilePath('', ''); + + // Get the root mappings folder from state + const rootMappingsPath = `${state.rootPath}/mappings`; + + if (!fileOps.fileExists(rootMappingsPath)) { + return []; + } + + const pairs: Array<{ sourceGuid: string; targetGuid: string; locales: string[] }> = []; + + try { + const dirs = fileOps.getFolderContents(rootMappingsPath); + + for (const dir of dirs) { + // Directory format: {sourceGuid}-{targetGuid} + const fullPath = `${rootMappingsPath}/${dir}`; + + // Find locales in this directory + const locales: string[] = []; + try { + const contents = fileOps.getFolderContents(fullPath); + + for (const item of contents) { + // Check if it looks like a locale (e.g., en-us, es-us) + if (/^[a-z]{2}-[a-z]{2}$/i.test(item)) { + locales.push(item); + } + } + } catch (e) { + // Skip directories we can't read + continue; + } + + if (locales.length > 0) { + // Parse the directory name to extract source and target GUIDs + // Format: {sourceGuid}-{targetGuid} where GUIDs are like "c39c63bd-us2" + const guidPattern = /^([a-zA-Z0-9]+-[a-zA-Z0-9]+)-([a-zA-Z0-9]+-[a-zA-Z0-9]+)$/; + const match = dir.match(guidPattern); + + if (match) { + pairs.push({ + sourceGuid: match[1], + targetGuid: match[2], + locales + }); + } + } + } + } catch (error: any) { + console.error(`Error listing mapping directories: ${error.message}`); + } + + return pairs; +} + +/** + * Get mapping summary for display + */ +export function getMappingSummary( + sourceGuid: string, + targetGuid: string, + locales: string[] +): { totalContent: number; totalPages: number; localesFound: string[] } { + const result = readMappingsForGuidPair(sourceGuid, targetGuid, locales); + const fileOps = new fileOperations('', ''); + + const localesFound = locales.filter(locale => { + const ops = new fileOperations(targetGuid, locale); + const contentMappings = ops.getMappingFile('item', sourceGuid, targetGuid, locale); + const pageMappings = ops.getMappingFile('page', sourceGuid, targetGuid, locale); + return (contentMappings && contentMappings.length > 0) || (pageMappings && pageMappings.length > 0); + }); + + return { + totalContent: result.contentIds.length, + totalPages: result.pageIds.length, + localesFound + }; +} diff --git a/src/lib/mappers/mapping-version-updater.ts b/src/lib/mappers/mapping-version-updater.ts new file mode 100644 index 0000000..3f187be --- /dev/null +++ b/src/lib/mappers/mapping-version-updater.ts @@ -0,0 +1,309 @@ +/** + * Mapping Version Updater + * + * After publishing, updates the mappings with the new versionIDs + * by reading the refreshed data from the filesystem using fileOperations. + */ + +import { fileOperations } from '../../core'; +import { getLogger } from '../../core/state'; +import { ContentItemMapper } from './content-item-mapper'; +import { PageMapper } from './page-mapper'; +import { getContentItemsFromFileSystem } from '../getters/filesystem/get-content-items'; +import { getPagesFromFileSystem } from '../getters/filesystem/get-pages'; +import ansiColors from 'ansi-colors'; +import { MappingUpdateResult } from '../../types'; + +// Re-export type for convenience +export { MappingUpdateResult }; + +/** + * Version change detail for logging + */ +export interface VersionChangeDetail { + id: number; + oldVersion: number; + newVersion: number; + changed: boolean; + name?: string; // Content title/name or page title + refName?: string; // Content referenceName or page path + modelName?: string; // Content model (definitionName) +} + +/** + * Helper to log to both logger and capture lines + */ +function logLine(line: string, logLines: string[]): void { + const logger = getLogger(); + if (logger) { + logger.info(line); + } else { + console.log(line); + } + logLines.push(line); +} + +/** + * Update content item mappings with new targetVersionID after publishing + * Only updates targetVersionID - sourceVersionID should only change during sync operations + */ +export async function updateContentMappingsAfterPublish( + publishedContentIds: number[], + sourceGuid: string, + targetGuid: string, + locale: string +): Promise<{ updated: number; errors: string[]; changes: VersionChangeDetail[] }> { + const errors: string[] = []; + const changes: VersionChangeDetail[] = []; + let updated = 0; + + // Deduplicate IDs - API may return duplicates for nested content + const uniqueContentIds = Array.from(new Set(publishedContentIds)); + + if (uniqueContentIds.length === 0) { + return { updated: 0, errors: [], changes: [] }; + } + + try { + // Create file operations for target (we only need target data for versionID) + const targetFileOps = new fileOperations(targetGuid, locale); + + // Load content items from target filesystem (refreshed after pull) + const targetContentItems = getContentItemsFromFileSystem(targetFileOps); + + // Create content item mapper + const contentMapper = new ContentItemMapper(sourceGuid, targetGuid, locale); + + // Create lookup map for quick access + const targetContentMap = new Map( + targetContentItems.map(item => [item.contentID, item]) + ); + + // Update targetVersionID for each published content item + for (const targetContentId of uniqueContentIds) { + const targetItem = targetContentMap.get(targetContentId); + if (!targetItem) { + errors.push(`Target content item ${targetContentId} not found in filesystem`); + continue; + } + + // Update only the target versionID in the mapping + const result = contentMapper.updateTargetVersionID( + targetContentId, + targetItem.properties.versionID + ); + + if (result.success) { + updated++; + // Track all version updates with display info + changes.push({ + id: targetContentId, + oldVersion: result.oldVersionID!, + newVersion: result.newVersionID!, + changed: result.oldVersionID !== result.newVersionID, + name: targetItem.fields?.title || targetItem.fields?.name || `Item ${targetContentId}`, + refName: targetItem.properties?.referenceName, + modelName: targetItem.properties?.definitionName + }); + } else { + errors.push(`No mapping found for target content ID ${targetContentId}`); + } + } + + return { updated, errors, changes }; + } catch (error: any) { + errors.push(`Content mapping update failed: ${error.message}`); + return { updated, errors, changes: [] }; + } +} + +/** + * Update page mappings with new targetVersionID after publishing + * Only updates targetVersionID - sourceVersionID should only change during sync operations + */ +export async function updatePageMappingsAfterPublish( + publishedPageIds: number[], + sourceGuid: string, + targetGuid: string, + locale: string +): Promise<{ updated: number; errors: string[]; changes: VersionChangeDetail[] }> { + const errors: string[] = []; + const changes: VersionChangeDetail[] = []; + let updated = 0; + + // Deduplicate IDs - API may return duplicates + const uniquePageIds = Array.from(new Set(publishedPageIds)); + + if (uniquePageIds.length === 0) { + return { updated: 0, errors: [], changes: [] }; + } + + try { + // Create file operations for target (we only need target data for versionID) + const targetFileOps = new fileOperations(targetGuid, locale); + + // Load pages from target filesystem (refreshed after pull) + const targetPages = getPagesFromFileSystem(targetFileOps); + + // Create page mapper + const pageMapper = new PageMapper(sourceGuid, targetGuid, locale); + + // Create lookup map for quick access + const targetPageMap = new Map( + targetPages.map(page => [page.pageID, page]) + ); + + // Update targetVersionID for each published page + for (const targetPageId of uniquePageIds) { + const targetPage = targetPageMap.get(targetPageId); + if (!targetPage) { + errors.push(`Target page ${targetPageId} not found in filesystem`); + continue; + } + + // Update only the target versionID in the mapping + const result = pageMapper.updateTargetVersionID( + targetPageId, + targetPage.properties.versionID + ); + + if (result.success) { + updated++; + // Track all version updates with display info + changes.push({ + id: targetPageId, + oldVersion: result.oldVersionID!, + newVersion: result.newVersionID!, + changed: result.oldVersionID !== result.newVersionID, + name: targetPage.title || targetPage.name || `Page ${targetPageId}`, + refName: targetPage.name ? `/${targetPage.name}` : undefined + }); + } else { + errors.push(`No mapping found for target page ID ${targetPageId}`); + } + } + + return { updated, errors, changes }; + } catch (error: any) { + errors.push(`Page mapping update failed: ${error.message}`); + return { updated, errors, changes: [] }; + } +} + +/** + * Format version change for display + * Format: ● [guid][locale] content ID: {id} - Name (Type) v1565 → v1593 mapping updated + */ +function formatVersionChange( + change: VersionChangeDetail, + entityType: string, + targetGuid: string, + locale: string +): string { + const symbol = change.changed ? ansiColors.green('●') : ansiColors.yellow('○'); + const guidDisplay = change.changed ? ansiColors.green(`[${targetGuid}]`) : ansiColors.yellow(`[${targetGuid}]`); + const localeDisplay = ansiColors.gray(`[${locale}]`); + const entityDisplay = ansiColors.white(entityType); + const idDisplay = ansiColors.cyan.underline(String(change.id)); + const nameDisplay = ansiColors.white(change.name || ''); + + // Build the type display (model name for content, path for pages) + let typeDisplay = ''; + if (change.modelName) { + typeDisplay = ansiColors.gray(` (${change.modelName})`); + } else if (change.refName) { + typeDisplay = ansiColors.gray(` (${change.refName})`); + } + + if (change.changed) { + const versionDisplay = ansiColors.gray(`v${change.oldVersion} → v${change.newVersion}`); + const action = ansiColors.green('mapping updated'); + // Format: ● [guid][locale] content ID: {id} - Name (Type) v1565 → v1593 mapping updated + return `${symbol} ${guidDisplay}${localeDisplay} ${entityDisplay} ID: ${idDisplay} - ${nameDisplay}${typeDisplay} ${versionDisplay} ${action}`; + } else { + const versionDisplay = ansiColors.gray(`v${change.newVersion}`); + return `${symbol} ${guidDisplay}${localeDisplay} ${entityDisplay} ID: ${idDisplay} - ${nameDisplay}${typeDisplay} ${versionDisplay} ${ansiColors.gray('unchanged')}`; + } +} + +/** + * Display version changes with summary and full details + * Returns formatted lines for logging + */ +function displayVersionChanges( + label: string, + entityType: string, + changes: VersionChangeDetail[], + totalUpdated: number, + targetGuid: string, + locale: string, + logLines: string[] +): void { + if (changes.length === 0) return; + + // Show all items using the logger + changes.forEach(change => { + const line = formatVersionChange(change, entityType, targetGuid, locale); + logLine(line, logLines); + }); +} + +/** + * Update all mappings after publishing + * Returns result and log lines for the logger + */ +export async function updateMappingsAfterPublish( + publishedContentIds: number[], + publishedPageIds: number[], + sourceGuid: string, + targetGuid: string, + locale: string +): Promise<{ result: MappingUpdateResult; logLines: string[] }> { + const logLines: string[] = []; + + logLine(ansiColors.cyan('\nUpdating mappings with new version IDs...'), logLines); + + const result: MappingUpdateResult = { + contentMappingsUpdated: 0, + pageMappingsUpdated: 0, + errors: [] + }; + + // Update content mappings + if (publishedContentIds.length > 0) { + const contentResult = await updateContentMappingsAfterPublish( + publishedContentIds, + sourceGuid, + targetGuid, + locale + ); + result.contentMappingsUpdated = contentResult.updated; + result.errors.push(...contentResult.errors); + + displayVersionChanges('content item', 'content', contentResult.changes, contentResult.updated, targetGuid, locale, logLines); + } + + // Update page mappings + if (publishedPageIds.length > 0) { + const pageResult = await updatePageMappingsAfterPublish( + publishedPageIds, + sourceGuid, + targetGuid, + locale + ); + result.pageMappingsUpdated = pageResult.updated; + result.errors.push(...pageResult.errors); + + displayVersionChanges('page', 'page', pageResult.changes, pageResult.updated, targetGuid, locale, logLines); + } + + // Summary line + logLine(ansiColors.green(`✓ Mappings updated: ${result.contentMappingsUpdated} content, ${result.pageMappingsUpdated} pages`), logLines); + + // Report any errors + if (result.errors.length > 0) { + logLine(ansiColors.yellow(` ⚠️ ${result.errors.length} mapping update errors (see logs)`), logLines); + } + + return { result, logLines }; +} diff --git a/src/lib/mappers/page-mapper.ts b/src/lib/mappers/page-mapper.ts index fdfd0d1..e36a612 100644 --- a/src/lib/mappers/page-mapper.ts +++ b/src/lib/mappers/page-mapper.ts @@ -128,5 +128,33 @@ export class PageMapper { return targetPage.properties.versionID > mapping.targetVersionID; } + /** + * Update only the target versionID in a mapping (used after publishing) + * Does NOT update sourceVersionID - that should only change during sync operations + * + * @returns Object with success status and old/new version IDs + */ + updateTargetVersionID(targetPageID: number, newVersionID: number): { + success: boolean; + oldVersionID?: number; + newVersionID?: number; + } { + const mapping = this.getPageMappingByPageID(targetPageID, 'target'); + if (!mapping) return { success: false }; + + const oldVersionID = mapping.targetVersionID; + + // Only update if version actually changed + if (oldVersionID !== newVersionID) { + mapping.targetVersionID = newVersionID; + this.saveMapping(); + } + + return { + success: true, + oldVersionID, + newVersionID + }; + } } \ No newline at end of file diff --git a/src/lib/publishers/index.ts b/src/lib/publishers/index.ts index db07779..507bd86 100644 --- a/src/lib/publishers/index.ts +++ b/src/lib/publishers/index.ts @@ -3,10 +3,63 @@ * * This module provides simple publisher functions that mirror the SDK patterns exactly. * These functions are lightweight wrappers around the Management SDK publishing methods. + * + * NOTE: Batch workflow operations have been consolidated into src/core/batch-workflows.ts + * The exports below are re-exported from their new locations. */ // Simple publisher functions - mirror SDK patterns export { publishContentItem } from './content-item-publisher'; export { publishPage } from './page-publisher'; export { publishContentList } from './content-list-publisher'; -export { publishBatch } from './batch-publisher'; \ No newline at end of file +export { publishBatch } from './batch-publisher'; + +// Re-export from consolidated batch-workflows service in core +export { + batchWorkflow, + type BatchItemType, + createBatches +} from '../../core/batch-workflows'; + +// Re-export workflow module +export { + workflowOrchestrator, + parseWorkflowOptions, + getOperationName +} from '../workflows'; + +// Re-export all workflow types from central types folder +export { + WorkflowOperationType, + BatchWorkflowResult, + WorkflowOrchestratorResult, + WorkflowOptions, + ContentMapping, + PageMapping, + MappingReadResult, + MappingUpdateResult, + ItemState, + SourceItemData, + PublishStatusResult +} from '../../types'; + +// Re-export mapping utilities from mappers (moved from publishers) +export { + updateMappingsAfterPublish, + updateContentMappingsAfterPublish, + updatePageMappingsAfterPublish +} from '../mappers/mapping-version-updater'; + +export { + readMappingsForGuidPair, + listAvailableMappingPairs, + getMappingSummary +} from '../mappers/mapping-reader'; + +// Re-export source publish status checker functions from shared (moved from publishers) +export { + checkSourcePublishStatus, + filterPublishedContent, + filterPublishedPages, + isPublished +} from '../shared/source-publish-status-checker'; \ No newline at end of file diff --git a/src/lib/pushers/content-pusher/util/types.ts b/src/lib/pushers/content-pusher/util/types.ts index 1e7c317..f91f234 100644 --- a/src/lib/pushers/content-pusher/util/types.ts +++ b/src/lib/pushers/content-pusher/util/types.ts @@ -26,7 +26,7 @@ export interface BatchProcessingResult { skippedCount: number; // Number of items skipped due to existing content successfulItems: BatchSuccessItem[]; failedItems: BatchFailedItem[]; - publishableIds: number[]; // Target content IDs for auto-publishing + publishableIds: number[]; // Target content IDs for workflow operations } /** diff --git a/src/lib/pushers/model-pusher.ts b/src/lib/pushers/model-pusher.ts index 384a9bd..6913f87 100644 --- a/src/lib/pushers/model-pusher.ts +++ b/src/lib/pushers/model-pusher.ts @@ -46,14 +46,17 @@ export async function pushModels(sourceData: mgmtApi.Model[], targetData: mgmtAp // TODO: we only care about the field count if the target model has NO fields and the source model has fields - // special case for the default RichTextArea model - const defaultRichTextArea = model.referenceName === 'RichTextArea' && !mapping && targetModel; - if(defaultRichTextArea){ - // force create the mapping for the default RichTextArea model + // Handle models that exist in target but have no mapping + // This ensures downstream containers can find their model mappings + const existsInTargetWithoutMapping = !mapping && targetModel; + if (existsInTargetWithoutMapping) { + // Create the mapping for existing target models (ensures containers can reference them) referenceMapper.addMapping(model, targetModel); + // Add to skip list since model already exists and is up to date + shouldSkip.push(model); + continue; // Skip remaining conditions - mapping is now created, no further action needed } - if ((!mapping && !targetModel)) { shouldCreateStub.push(model); } @@ -64,7 +67,7 @@ export async function pushModels(sourceData: mgmtApi.Model[], targetData: mgmtAp shouldUpdateFields.push(model); } // if the mapping exists, and the target has changed, we need to skip the model, not safe to update - if ((mapping && hasTargetChanged) || defaultRichTextArea) { + if (mapping && hasTargetChanged) { shouldSkip.push(model); } // if the mapping exists, and the source and target have not changed, we need to skip the model diff --git a/src/lib/pushers/orchestrate-pushers.ts b/src/lib/pushers/orchestrate-pushers.ts index a849ca4..6293073 100644 --- a/src/lib/pushers/orchestrate-pushers.ts +++ b/src/lib/pushers/orchestrate-pushers.ts @@ -300,7 +300,7 @@ export class Pushers { totalSkipped += pusherResult.skipped || 0; totalFailures += pusherResult.failed || 0; - // Collect publishable IDs for auto-publishing + // Collect publishable IDs for workflow operations if (pusherResult.publishableIds && pusherResult.publishableIds.length > 0) { if (config.elements.includes("Content")) { publishableContentIds.push(...pusherResult.publishableIds); diff --git a/src/lib/pushers/page-pusher/push-pages.ts b/src/lib/pushers/page-pusher/push-pages.ts index 468c9ce..148b3f7 100644 --- a/src/lib/pushers/page-pusher/push-pages.ts +++ b/src/lib/pushers/page-pusher/push-pages.ts @@ -36,7 +36,7 @@ export async function pushPages( let failed = 0; let skipped = 0; // No duplicates to skip since API prevents true duplicates at same hierarchy level let status: "success" | "error" = "success"; - let publishableIds: number[] = []; // Track target page IDs for auto-publishing + let publishableIds: number[] = []; // Track target page IDs for workflow operations //loop all the channels diff --git a/src/lib/pushers/template-pusher.ts b/src/lib/pushers/template-pusher.ts index 8ef39c8..1a324d2 100644 --- a/src/lib/pushers/template-pusher.ts +++ b/src/lib/pushers/template-pusher.ts @@ -49,13 +49,26 @@ export async function pushTemplates( const { sourceGuid, targetGuid } = state; const referenceMapper = new TemplateMapper(sourceGuid[0], targetGuid[0]); - const existingMapping = referenceMapper.getTemplateMapping(template, "source"); + let existingMapping = referenceMapper.getTemplateMapping(template, "source"); let targetTemplate = targetData.find(targetTemplate => targetTemplate.pageTemplateID === existingMapping?.targetPageTemplateID) || null; if (!targetTemplate) { // Try to get the template via the mapper targetTemplate = referenceMapper.getMappedEntity(existingMapping, "target"); } + // Handle templates that exist in target but have no mapping (match by name) + // This ensures downstream pages can find their template mappings + if (!existingMapping && !targetTemplate) { + targetTemplate = targetData.find(t => t.pageTemplateName === template.pageTemplateName) || null; + if (targetTemplate) { + // Create the mapping for existing target template + referenceMapper.addMapping(template, targetTemplate); + logger.template.skipped(template, "exists in target, mapping created", targetGuid[0]); + skipped++; + processedCount++; + continue; // Skip to next template - mapping is now created + } + } const isTargetSafe = existingMapping !== null && referenceMapper.hasTargetChanged(targetTemplate); const hasSourceChanges = existingMapping !== null && referenceMapper.hasSourceChanged(template); @@ -86,12 +99,12 @@ export async function pushTemplates( if (def.contentDefinitionID) { const modelMappers = new ModelMapper(sourceGuid[0], targetGuid[0]); - const modelMapping = modelMappers.getModelMappingByID(def.contentDefinitionID, 'target'); + const modelMapping = modelMappers.getModelMappingByID(def.contentDefinitionID, 'source'); if (modelMapping?.targetID) mappedDef.contentDefinitionID = modelMapping.targetID; } if (def.itemContainerID) { const containerMappers = new ContainerMapper(sourceGuid[0], targetGuid[0]); - const containerMapping = containerMappers.getContainerMappingByContentViewID(def.itemContainerID, 'target'); + const containerMapping = containerMappers.getContainerMappingByContentViewID(def.itemContainerID, 'source'); if (containerMapping?.targetContentViewID) mappedDef.itemContainerID = containerMapping.targetContentViewID; } // if (def.publishContentItemID) { diff --git a/src/lib/shared/get-fetch-api-status.ts b/src/lib/shared/get-fetch-api-status.ts new file mode 100644 index 0000000..88957a9 --- /dev/null +++ b/src/lib/shared/get-fetch-api-status.ts @@ -0,0 +1,88 @@ +/** + * Fetch API Status Checker + * + * Checks if the Fetch API CDN sync is complete for an instance. + * Used before pull operations and after publishing to ensure + * changes have propagated to the CDN. + */ + +import * as mgmtApi from '@agility/management-sdk'; +import { getApiClient } from '../../core/state'; +import ansiColors from 'ansi-colors'; + +export type FetchApiSyncMode = 'fetch' | 'preview'; + +export interface FetchApiStatus { + timestamp?: string; + completionTime?: string; + errorMessage?: string; + inProgress: boolean; + itemsAffected: number; + lastContentVersionID: number; + lastDeletedContentVersionID: number; + lastDeletedPageVersionID: number; + leaseID?: string; + maxChangeDate?: string; + maxContentModelDate?: string; + pushType: number; + startTime?: string; + websiteName?: string; +} + +/** + * Get the Fetch API sync status for an instance + * + * @param guid - The instance GUID + * @param mode - Sync mode: 'fetch' (live) or 'preview'. Defaults to 'fetch'. + * @param waitForCompletion - If true, polls until sync is complete. Defaults to false. + * @returns The sync status + */ +export async function getFetchApiStatus( + guid: string, + mode: FetchApiSyncMode = 'fetch', + waitForCompletion: boolean = false +): Promise { + const apiClient = getApiClient(); + return apiClient.instanceMethods.getFetchApiStatus(guid, mode, waitForCompletion); +} + +/** + * Wait for Fetch API sync to complete with progress messaging + * Returns log lines for capturing in logger + * + * @param guid - The instance GUID + * @param mode - Sync mode: 'fetch' (live) or 'preview'. Defaults to 'fetch'. + * @param silent - If true, suppresses console output. Defaults to false. + * @returns Object containing final status and log lines + */ +export async function waitForFetchApiSync( + guid: string, + mode: FetchApiSyncMode = 'fetch', + silent: boolean = false +): Promise<{ status: FetchApiStatus; logLines: string[] }> { + const logLines: string[] = []; + + // First check if sync is in progress + const initialStatus = await getFetchApiStatus(guid, mode, false); + + if (!initialStatus.inProgress) { + return { status: initialStatus, logLines }; + } + + // Sync is in progress, wait for completion + const waitingMsg = ansiColors.gray(`Waiting for Fetch API sync to complete...`); + logLines.push(waitingMsg); + if (!silent) { + console.log(waitingMsg); + } + + const finalStatus = await getFetchApiStatus(guid, mode, true); + + const completeMsg = ansiColors.green(`✓ Fetch API sync complete \n`); + logLines.push(completeMsg); + if (!silent) { + console.log(completeMsg); + } + + return { status: finalStatus, logLines }; +} diff --git a/src/lib/shared/index.ts b/src/lib/shared/index.ts index f547df6..30a1a8a 100644 --- a/src/lib/shared/index.ts +++ b/src/lib/shared/index.ts @@ -11,6 +11,29 @@ export function prettyException(error: any): string { return error.message || er export function logBatchError(error: any, context: string): void { console.error("Batch Error:", error); } export { pollBatchUntilComplete, extractBatchResults } from "../pushers/batch-polling"; +// Source publish status checker - checks source instance publish status +export { + checkSourcePublishStatus, + filterPublishedContent, + filterPublishedPages, + isPublished +} from './source-publish-status-checker'; + +// Fetch API status checker - checks if CDN sync is complete +export { + getFetchApiStatus, + waitForFetchApiSync, + type FetchApiStatus, + type FetchApiSyncMode +} from './get-fetch-api-status'; + +// Re-export types from central types folder +export { + ItemState, + type SourceItemData, + type PublishStatusResult +} from '../../types'; + // Version utility import * as fs from 'fs'; import * as path from 'path'; diff --git a/src/lib/shared/source-publish-status-checker.ts b/src/lib/shared/source-publish-status-checker.ts new file mode 100644 index 0000000..de91b70 --- /dev/null +++ b/src/lib/shared/source-publish-status-checker.ts @@ -0,0 +1,153 @@ +/** + * Source Publish Status Checker + * + * Reads source instance files from the agility-files folder to determine + * which items are published in the source instance. This allows workflow operations + * to only process items in the target that match the source publish state. + * Uses fileOperations for consistent filesystem access. + */ + +import { fileOperations } from '../../core'; +import { + ContentMapping, + PageMapping, + ItemState, + SourceItemData, + PublishStatusResult +} from '../../types'; + +// Re-export types for convenience +export { ItemState, SourceItemData, PublishStatusResult }; + +/** + * Check if an item is published based on its state + */ +export function isPublished(itemState: number): boolean { + return itemState === ItemState.Published; +} + +/** + * Read source item data using fileOperations + */ +function readSourceItem(fileOps: fileOperations, type: 'item' | 'page', id: number): SourceItemData | null { + try { + const data = fileOps.readJsonFile(`${type}/${id}.json`); + return data as SourceItemData | null; + } catch (error: any) { + return null; + } +} + +/** + * Filter content mappings to only include items that are published in the source + */ +export function filterPublishedContent( + contentMappings: ContentMapping[], + sourceGuid: string, + locales: string[] +): PublishStatusResult { + const result: PublishStatusResult = { + publishedContentIds: [], + unpublishedContentIds: [], + publishedPageIds: [], + unpublishedPageIds: [], + errors: [] + }; + + for (const mapping of contentMappings) { + let found = false; + let isItemPublished = false; + + // Try each locale to find the source item + for (const locale of locales) { + const fileOps = new fileOperations(sourceGuid, locale); + const sourceItem = readSourceItem(fileOps, 'item', mapping.sourceContentID); + + if (sourceItem && sourceItem.properties) { + found = true; + isItemPublished = isPublished(sourceItem.properties.state); + break; + } + } + + if (!found) { + result.errors.push(`Source content item ${mapping.sourceContentID} not found in local files`); + // Default to publishing if source not found (preserve existing behavior) + result.publishedContentIds.push(mapping.targetContentID); + } else if (isItemPublished) { + result.publishedContentIds.push(mapping.targetContentID); + } else { + result.unpublishedContentIds.push(mapping.targetContentID); + } + } + + return result; +} + +/** + * Filter page mappings to only include pages that are published in the source + */ +export function filterPublishedPages( + pageMappings: PageMapping[], + sourceGuid: string, + locales: string[] +): PublishStatusResult { + const result: PublishStatusResult = { + publishedContentIds: [], + unpublishedContentIds: [], + publishedPageIds: [], + unpublishedPageIds: [], + errors: [] + }; + + for (const mapping of pageMappings) { + let found = false; + let isItemPublished = false; + + // Try each locale to find the source page + for (const locale of locales) { + const fileOps = new fileOperations(sourceGuid, locale); + const sourceItem = readSourceItem(fileOps, 'page', mapping.sourcePageID); + + if (sourceItem && sourceItem.properties) { + found = true; + isItemPublished = isPublished(sourceItem.properties.state); + break; + } + } + + if (!found) { + result.errors.push(`Source page ${mapping.sourcePageID} not found in local files`); + // Default to publishing if source not found (preserve existing behavior) + result.publishedPageIds.push(mapping.targetPageID); + } else if (isItemPublished) { + result.publishedPageIds.push(mapping.targetPageID); + } else { + result.unpublishedPageIds.push(mapping.targetPageID); + } + } + + return result; +} + +/** + * Check publish status for all content and page mappings + * Returns filtered lists of target IDs that should be published + */ +export function checkSourcePublishStatus( + contentMappings: ContentMapping[], + pageMappings: PageMapping[], + sourceGuid: string, + locales: string[] +): PublishStatusResult { + const contentResult = filterPublishedContent(contentMappings, sourceGuid, locales); + const pageResult = filterPublishedPages(pageMappings, sourceGuid, locales); + + return { + publishedContentIds: contentResult.publishedContentIds, + unpublishedContentIds: contentResult.unpublishedContentIds, + publishedPageIds: pageResult.publishedPageIds, + unpublishedPageIds: pageResult.unpublishedPageIds, + errors: [...contentResult.errors, ...pageResult.errors] + }; +} diff --git a/src/lib/workflows/index.ts b/src/lib/workflows/index.ts new file mode 100644 index 0000000..676ef01 --- /dev/null +++ b/src/lib/workflows/index.ts @@ -0,0 +1,24 @@ +/** + * Workflows Module + * + * Central exports for all workflow-related functionality. + */ + +// Core workflow operation class +export { WorkflowOperation, WorkflowOperationResult } from './workflow-operation'; + +// Workflow orchestrator +export { workflowOrchestrator } from './workflow-orchestrator'; + +// Batch processing +export { processBatches, type BatchProcessingResult } from './process-batches'; + +// Workflow options parsing +export { parseWorkflowOptions, parseOperationType } from './workflow-options'; + +// Workflow helpers (operation names, verbs, icons) +export { getOperationName, getOperationVerb, getOperationIcon } from './workflow-helpers'; + +// Mapping utilities +export { listMappings } from './list-mappings'; +export { refreshAndUpdateMappings } from './refresh-mappings'; diff --git a/src/lib/workflows/list-mappings.ts b/src/lib/workflows/list-mappings.ts new file mode 100644 index 0000000..d701d94 --- /dev/null +++ b/src/lib/workflows/list-mappings.ts @@ -0,0 +1,38 @@ +/** + * List Mappings + * + * Display available mapping pairs for workflow operations. + */ + +import ansiColors from 'ansi-colors'; +import { listAvailableMappingPairs, getMappingSummary } from '../mappers/mapping-reader'; + +/** + * List available mapping pairs for workflow operations + */ +export function listMappings(): void { + console.log(ansiColors.cyan('\n' + '═'.repeat(50))); + console.log(ansiColors.cyan('📋 AVAILABLE MAPPINGS')); + console.log(ansiColors.cyan('═'.repeat(50))); + + const pairs = listAvailableMappingPairs(); + + if (pairs.length === 0) { + console.log(ansiColors.yellow('\nNo mappings found.')); + console.log(ansiColors.gray('Run a sync operation first to create mappings.')); + return; + } + + for (const pair of pairs) { + const summary = getMappingSummary(pair.sourceGuid, pair.targetGuid, pair.locales); + + console.log(ansiColors.white(`\n${pair.sourceGuid} → ${pair.targetGuid}`)); + console.log(ansiColors.gray(`Locales: ${pair.locales.join(', ')}`)); + console.log(ansiColors.gray(`Content items: ${summary.totalContent}`)); + console.log(ansiColors.gray(`Pages: ${summary.totalPages}`)); + } + + console.log(ansiColors.cyan('\n' + '─'.repeat(50))); + console.log(ansiColors.gray('To run a workflow operation:')); + console.log(ansiColors.white(' node dist/index.js workflows --sourceGuid --targetGuid --type publish')); +} diff --git a/src/lib/workflows/process-batches.ts b/src/lib/workflows/process-batches.ts new file mode 100644 index 0000000..b742c25 --- /dev/null +++ b/src/lib/workflows/process-batches.ts @@ -0,0 +1,237 @@ +/** + * Process Batches + * + * Processes items in batches with progress reporting and error handling. + */ + +import ansiColors from 'ansi-colors'; +import { batchWorkflow, createBatches, type BatchItemType } from '../../core/batch-workflows'; +import { getOperationName, getOperationVerb } from './workflow-helpers'; +import { WorkflowOperationType } from '../../types'; +import { state, fileOperations } from '../../core'; +import { getLogger } from '../../core/state'; +import { getContentItemsFromFileSystem } from '../getters/filesystem/get-content-items'; +import { getPagesFromFileSystem } from '../getters/filesystem/get-pages'; + +/** + * Item info for display purposes + */ +interface ItemDisplayInfo { + id: number; + name: string; + type?: string; +} + +/** + * Get content item display info from filesystem + */ +function getContentDisplayInfo(ids: number[], targetGuid: string, locale: string): Map { + const displayMap = new Map(); + + try { + const fileOps = new fileOperations(targetGuid, locale); + const contentItems = getContentItemsFromFileSystem(fileOps); + + for (const item of contentItems) { + if (ids.includes(item.contentID)) { + // Try to get a display name from fields.title, properties.referenceName, or definitionName + const displayName = item.fields?.title + || item.fields?.name + || item.properties?.referenceName + || `Item ${item.contentID}`; + const modelName = item.properties?.definitionName || ''; + + displayMap.set(item.contentID, { + id: item.contentID, + name: displayName, + type: modelName + }); + } + } + } catch (error) { + // Silently fail - we'll just show IDs without names + } + + return displayMap; +} + +/** + * Get page display info from filesystem + */ +function getPageDisplayInfo(ids: number[], targetGuid: string, locale: string): Map { + const displayMap = new Map(); + + try { + const fileOps = new fileOperations(targetGuid, locale); + const pages = getPagesFromFileSystem(fileOps); + + for (const page of pages) { + if (ids.includes(page.pageID)) { + // Use title, name, or pageID as display + const displayName = page.title || page.name || `Page ${page.pageID}`; + const pagePath = page.name ? `/${page.name}` : ''; + + displayMap.set(page.pageID, { + id: page.pageID, + name: displayName, + type: pagePath + }); + } + } + } catch (error) { + // Silently fail - we'll just show IDs without names + } + + return displayMap; +} + +/** + * Helper to log to both console (via logger) and capture lines + */ +function logLine(line: string, logLines: string[]): void { + const logger = getLogger(); + if (logger) { + logger.info(line); + } else { + console.log(line); + } + logLines.push(line); +} + +/** + * Display all items being processed (no truncation) + * Format: ● [guid][locale] content ID: {id} - Name (Type) - publishing + */ +function displayItemBreakdown( + ids: number[], + type: BatchItemType, + targetGuid: string, + locale: string, + operationName: string, + displayMap: Map, + logLines: string[] +): void { + const entityType = type === 'content' ? 'content' : 'page'; + + // Show ALL items - no truncation + for (const id of ids) { + const info = displayMap.get(id); + const guidDisplay = ansiColors.green(`[${targetGuid}]`); + const localeDisplay = ansiColors.gray(`[${locale}]`); + const symbol = ansiColors.green('●'); + + let line: string; + if (info) { + const typeDisplay = info.type ? ansiColors.gray(` (${info.type})`) : ''; + // Format: ● [guid][locale] content ID: {id} - Name (Type) - publishing + line = `${symbol} ${guidDisplay}${localeDisplay} ${ansiColors.white(entityType)} ID: ${ansiColors.cyan.underline(String(id))} - ${ansiColors.white(info.name)}${typeDisplay} - ${ansiColors.gray(operationName.toLowerCase())}`; + } else { + line = `${symbol} ${guidDisplay}${localeDisplay} ${ansiColors.white(entityType)} ID: ${ansiColors.cyan.underline(String(id))} - ${ansiColors.gray(operationName.toLowerCase())}`; + } + logLine(line, logLines); + } +} + +/** + * Batch processing result + */ +export interface BatchProcessingResult { + total: number; + processed: number; + failed: number; + batches: number; + processedIds: number[]; + logLines: string[]; +} + +/** + * Process batches for a specific item type (content or pages) + */ +export async function processBatches( + ids: number[], + type: BatchItemType, + locale: string, + operation: WorkflowOperationType, + errors: string[] +): Promise { + const logLines: string[] = []; + const results: BatchProcessingResult = { + total: ids.length, + processed: 0, + failed: 0, + batches: 0, + processedIds: [], + logLines: [] + }; + + if (ids.length === 0) return results; + + const label = type === 'content' ? 'Content' : 'Page'; + const operationName = getOperationName(operation); + const operationVerb = getOperationVerb(operation); + + logLine(ansiColors.cyan(`\n${operationName}ing ${ids.length} ${label.toLowerCase()} items...`), logLines); + + // Get item display info and show breakdown (ALL items, no truncation) + const targetGuid = state.targetGuid?.[0]; + if (targetGuid) { + const displayMap = type === 'content' + ? getContentDisplayInfo(ids, targetGuid, locale) + : getPageDisplayInfo(ids, targetGuid, locale); + + if (displayMap.size > 0) { + displayItemBreakdown(ids, type, targetGuid, locale, operationName, displayMap, logLines); + } + } + + const batches = createBatches(ids); + results.batches = batches.length; + + for (let i = 0; i < batches.length; i++) { + const batch = batches[i]; + const batchNum = i + 1; + const progress = Math.round((batchNum / batches.length) * 100); + + logLine(ansiColors.gray(`[${progress}%] ${label} batch ${batchNum}/${batches.length}: ${operationName}ing ${batch.length} items...`), logLines); + + try { + const batchResult = await batchWorkflow(batch, locale, operation, type); + + if (batchResult.success) { + results.processed += batchResult.processedIds.length; + results.processedIds.push(...batchResult.processedIds); + } else { + results.failed += batch.length; + errors.push(`${label} batch ${batchNum}: ${batchResult.error}`); + } + } catch (error: any) { + results.failed += batch.length; + errors.push(`${label} batch ${batchNum}: ${error.message}`); + } + + // Small delay between batches to prevent throttling + if (i < batches.length - 1) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + } + + // Display count - clarify when API processes more items than requested (nested content) + let summaryLine: string; + if (results.processed > results.total) { + // API processed additional nested items + summaryLine = ansiColors.green(`✓ ${label} ${operationVerb}: ${results.processed} items (${results.total} requested + ${results.processed - results.total} nested)`); + } else if (results.processed === results.total) { + summaryLine = ansiColors.green(`✓ ${label} ${operationVerb}: ${results.processed} items`); + } else { + // Some items failed or were skipped + summaryLine = ansiColors.green(`✓ ${label} ${operationVerb}: ${results.processed}/${results.total} items`); + } + logLine(summaryLine, logLines); + + if (results.failed > 0) { + logLine(ansiColors.red(`✗ ${results.failed} ${label.toLowerCase()} items failed`), logLines); + } + + results.logLines = logLines; + return results; +} diff --git a/src/lib/workflows/refresh-mappings.ts b/src/lib/workflows/refresh-mappings.ts new file mode 100644 index 0000000..3256908 --- /dev/null +++ b/src/lib/workflows/refresh-mappings.ts @@ -0,0 +1,173 @@ +/** + * Refresh Mappings + * + * Refresh target instance data and update mappings after publishing. + */ + +import ansiColors from 'ansi-colors'; +import * as fs from 'fs'; +import * as path from 'path'; +import { Pull } from '../../core/pull'; +import { getAllApiKeys, getState } from '../../core/state'; +import { updateMappingsAfterPublish } from '../mappers/mapping-version-updater'; +import { waitForFetchApiSync } from '../shared/get-fetch-api-status'; +import { generateLogHeader } from '../shared'; + +/** + * Check if we have valid API keys for the target GUID + */ +function hasValidTargetKeys(targetGuid: string): boolean { + const apiKeys = getAllApiKeys(); + return apiKeys.some(key => key.guid === targetGuid); +} + +/** + * Write log lines to a file + */ +function writeLogFile(logLines: string[], targetGuid: string, locale: string): string | null { + try { + const state = getState(); + const logDir = path.join(process.cwd(), state.rootPath, targetGuid, 'logs'); + + // Create logs directory if it doesn't exist + if (!fs.existsSync(logDir)) { + fs.mkdirSync(logDir, { recursive: true }); + } + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const logFileName = `publish-${locale}-${timestamp}.log`; + const logFilePath = path.join(logDir, logFileName); + + // Add header + const header = generateLogHeader('Publish', { + 'Target GUID': targetGuid, + 'Locale': locale + }); + + // Strip ANSI colors for file output + const stripAnsi = (str: string) => str.replace(/\x1B\[[0-9;]*[mK]/g, ''); + const cleanLines = logLines.map(line => stripAnsi(line)); + + const content = header + cleanLines.join('\n') + '\n'; + fs.writeFileSync(logFilePath, content, 'utf8'); + + return logFilePath; + } catch (error) { + return null; + } +} + +/** + * Refresh target instance data and update mappings with new versionIDs after publishing + * + * @param publishedContentIds - Content IDs that were published + * @param publishedPageIds - Page IDs that were published + * @param sourceGuid - Source instance GUID + * @param targetGuid - Target instance GUID + * @param locale - Locale code + * @param publishLogLines - Log lines from the publish operation to include in log file + */ +export async function refreshAndUpdateMappings( + publishedContentIds: number[], + publishedPageIds: number[], + sourceGuid: string, + targetGuid: string, + locale: string, + publishLogLines: string[] = [] +): Promise { + // Start with publish log lines if provided + const logLines: string[] = [...publishLogLines]; + + const headerLine = ansiColors.cyan('\nRefreshing target instance data...'); + logLines.push(headerLine); + console.log(headerLine); + + // Check if we have API keys for the target - if not, key fetch failed earlier + if (!hasValidTargetKeys(targetGuid)) { + const warnLine = ansiColors.yellow(` ⚠️ No API keys available for target ${targetGuid} - skipping refresh and mapping updates`); + const infoLine1 = ansiColors.gray(' This typically indicates an API connection issue (503, timeout, etc.)'); + const infoLine2 = ansiColors.gray(' Mappings will be updated on next successful sync'); + logLines.push(warnLine, infoLine1, infoLine2); + console.log(warnLine); + console.log(infoLine1); + console.log(infoLine2); + + // Still write log file even if we can't refresh + const logFilePath = writeLogFile(logLines, targetGuid, locale); + if (logFilePath) { + console.log(ansiColors.gray(`\n📄 Log file: ${logFilePath}`)); + } + return; + } + + try { + // Wait for Fetch API sync to complete before refreshing + // This ensures we're pulling the latest published data from the CDN + try { + const syncResult = await waitForFetchApiSync(targetGuid, 'fetch', false); + logLines.push(...syncResult.logLines); + } catch (error: any) { + const warnLine = ansiColors.yellow(` ⚠️ Could not check Fetch API status: ${error.message}`); + logLines.push(warnLine); + console.log(warnLine); + // Continue with refresh anyway - the status check is best-effort + } + + const pull = new Pull(); + + // Run an incremental pull on the target instance + const pullResult = await pull.pullInstances(true); + + // Check if the pull was successful before updating mappings + if (!pullResult.success) { + const warnLine = ansiColors.yellow(' ⚠️ Target refresh failed - skipping mapping version updates'); + const infoLine = ansiColors.gray(' Run a manual pull to refresh data and update mappings'); + logLines.push(warnLine, infoLine); + console.log(warnLine); + console.log(infoLine); + + // Still write log file on failure + const logFilePath = writeLogFile(logLines, targetGuid, locale); + if (logFilePath) { + console.log(ansiColors.gray(`\n📄 Log file: ${logFilePath}`)); + } + return; + } + + const successLine = ansiColors.green('✓ Target instance data refreshed'); + logLines.push(successLine); + console.log(successLine); + + // Update the mappings with the new versionIDs + const mappingResult = await updateMappingsAfterPublish( + publishedContentIds, + publishedPageIds, + sourceGuid, + targetGuid, + locale + ); + + // Add mapping update log lines + logLines.push(...mappingResult.logLines); + + // Write log file + const logFilePath = writeLogFile(logLines, targetGuid, locale); + if (logFilePath) { + const logPathLine = ansiColors.gray(`\n📄 Log file: ${logFilePath}`); + console.log(logPathLine); + } + + } catch (error: any) { + const errorLine = ansiColors.yellow(` ⚠️ Warning: Could not refresh/update mappings after publish: ${error.message}`); + const infoLine = ansiColors.gray(' Mappings may be stale until next sync'); + logLines.push(errorLine, infoLine); + console.error(errorLine); + console.log(infoLine); + + // Still write log file on error + const logFilePath = writeLogFile(logLines, targetGuid, locale); + if (logFilePath) { + console.log(ansiColors.gray(`\n📄 Log file: ${logFilePath}`)); + } + } +} diff --git a/src/lib/workflows/workflow-helpers.ts b/src/lib/workflows/workflow-helpers.ts new file mode 100644 index 0000000..fd8000f --- /dev/null +++ b/src/lib/workflows/workflow-helpers.ts @@ -0,0 +1,67 @@ +/** + * Workflow Helper Functions + * + * Utility functions for workflow operations - operation names, verbs, icons. + */ + +import { WorkflowOperationType } from '../../types'; + +/** + * Get human-readable operation name + */ +export function getOperationName(operation: WorkflowOperationType): string { + switch (operation) { + case WorkflowOperationType.Publish: + return 'publish'; + case WorkflowOperationType.Unpublish: + return 'unpublish'; + case WorkflowOperationType.Approve: + return 'approve'; + case WorkflowOperationType.Decline: + return 'decline'; + case WorkflowOperationType.RequestApproval: + return 'request approval'; + default: + return 'process'; + } +} + +/** + * Get operation verb for logging (past tense) + */ +export function getOperationVerb(operation: WorkflowOperationType): string { + switch (operation) { + case WorkflowOperationType.Publish: + return 'published'; + case WorkflowOperationType.Unpublish: + return 'unpublished'; + case WorkflowOperationType.Approve: + return 'approved'; + case WorkflowOperationType.Decline: + return 'declined'; + case WorkflowOperationType.RequestApproval: + return 'submitted for approval'; + default: + return 'processed'; + } +} + +/** + * Get operation icon for logging + */ +export function getOperationIcon(operation: WorkflowOperationType): string { + switch (operation) { + case WorkflowOperationType.Publish: + return '📤'; + case WorkflowOperationType.Unpublish: + return '📥'; + case WorkflowOperationType.Approve: + return '✅'; + case WorkflowOperationType.Decline: + return '❌'; + case WorkflowOperationType.RequestApproval: + return '📝'; + default: + return '⚙️'; + } +} diff --git a/src/lib/workflows/workflow-operation.ts b/src/lib/workflows/workflow-operation.ts new file mode 100644 index 0000000..d942288 --- /dev/null +++ b/src/lib/workflows/workflow-operation.ts @@ -0,0 +1,282 @@ +/** + * Workflow Operation Core Module + * + * Standalone module that reads mappings from the filesystem and performs + * workflow operations (publish, unpublish, approve, decline, requestApproval) + * on content and pages in the target instance. + */ + +import ansiColors from 'ansi-colors'; +import { state, initializeLogger, finalizeLogger, getLogger } from '../../core/state'; +import { readMappingsForGuidPair, getMappingSummary } from '../mappers/mapping-reader'; +import { parseWorkflowOptions, parseOperationType } from './workflow-options'; +import { getOperationName } from './workflow-helpers'; +import { workflowOrchestrator } from './workflow-orchestrator'; +import { listMappings } from './list-mappings'; +import { refreshAndUpdateMappings } from './refresh-mappings'; +import { checkSourcePublishStatus } from '../shared/source-publish-status-checker'; +import { WorkflowOperationResult, WorkflowOperationType } from '../../types'; + +// Re-export type for convenience +export { WorkflowOperationResult }; + +export class WorkflowOperation { + /** + * Execute workflow operation from mapping files + */ + async executeFromMappings(): Promise { + const startTime = Date.now(); + + // Initialize logger + initializeLogger('push'); + const logger = getLogger(); + + // Get operation type from state + const operationType = parseOperationType(state.operationType); + const operationName = getOperationName(operationType); + + const result: WorkflowOperationResult = { + success: true, + contentProcessed: 0, + contentFailed: 0, + pagesProcessed: 0, + pagesFailed: 0, + elapsedTime: 0, + errors: [], + operation: operationName + }; + + try { + const { sourceGuid, targetGuid, locale: locales } = state; + + // Validate required parameters + if (!sourceGuid || sourceGuid.length === 0) { + throw new Error('Source GUID is required. Use --sourceGuid flag.'); + } + if (!targetGuid || targetGuid.length === 0) { + throw new Error('Target GUID is required. Use --targetGuid flag.'); + } + if (!locales || locales.length === 0) { + throw new Error('At least one locale is required. Use --locale flag.'); + } + + const source = sourceGuid[0]; + const target = targetGuid[0]; + const primaryLocale = locales[0]; + + console.log(ansiColors.cyan('\n' + '═'.repeat(50))); + console.log(ansiColors.cyan(`📦 WORKFLOW OPERATION: ${operationName.toUpperCase()}`)); + console.log(ansiColors.cyan('═'.repeat(50))); + console.log(ansiColors.gray(`Source: ${source}`)); + console.log(ansiColors.gray(`Target: ${target}`)); + console.log(ansiColors.gray(`Locales: ${locales.join(', ')}`)); + console.log(ansiColors.gray(`Operation: ${operationName}`)); + + // Get mapping summary + const summary = getMappingSummary(source, target, locales); + // Check if explicit IDs are provided (bypasses mappings lookup) + const hasExplicitContentIDs = state.explicitContentIDs && state.explicitContentIDs.length > 0; + const hasExplicitPageIDs = state.explicitPageIDs && state.explicitPageIDs.length > 0; + const useExplicitIDs = hasExplicitContentIDs || hasExplicitPageIDs; + + // Parse workflow options - process both content and pages by default + const options = parseWorkflowOptions(true, primaryLocale); + if (!options) { + throw new Error('Failed to parse workflow options'); + } + + // Override with the actual operation type from state + options.operation = operationType; + + let contentIds: number[]; + let pageIds: number[]; + + if (useExplicitIDs) { + // Explicit IDs mode - bypass mappings lookup + console.log(ansiColors.cyan('\n🔧 Using explicit IDs (bypassing mappings lookup)')); + + contentIds = hasExplicitContentIDs ? state.explicitContentIDs : []; + pageIds = hasExplicitPageIDs ? state.explicitPageIDs : []; + + console.log(ansiColors.gray(` Explicit content IDs: ${contentIds.length > 0 ? contentIds.join(', ') : '(none)'}`)); + console.log(ansiColors.gray(` Explicit page IDs: ${pageIds.length > 0 ? pageIds.join(', ') : '(none)'}`)); + + if (contentIds.length === 0 && pageIds.length === 0) { + console.log(ansiColors.yellow('\n⚠️ No valid IDs provided.')); + result.elapsedTime = Date.now() - startTime; + return result; + } + } else { + // Standard mode - use mappings files + console.log(ansiColors.gray(`\nMapping Summary:`)); + console.log(ansiColors.gray(`Content items: ${summary.totalContent}`)); + console.log(ansiColors.gray(`Pages: ${summary.totalPages}`)); + console.log(ansiColors.gray(`Locales with data: ${summary.localesFound.join(', ') || 'none'}`)); + + if (summary.totalContent === 0 && summary.totalPages === 0) { + console.log(ansiColors.yellow('\n⚠️ No mappings found to process.')); + console.log(ansiColors.gray(' Run a sync operation first to create mappings, or use --contentIDs/--pageIDs to specify IDs directly.')); + result.elapsedTime = Date.now() - startTime; + return result; + } + + // Read mappings + const mappingResult = readMappingsForGuidPair(source, target, locales); + + if (mappingResult.errors.length > 0) { + console.log(ansiColors.yellow('\nWarnings during mapping read:')); + mappingResult.errors.forEach(err => console.log(ansiColors.yellow(` - ${err}`))); + } + + // For publish operations, check source publish status to filter only published items + contentIds = mappingResult.contentIds; + pageIds = mappingResult.pageIds; + + if (operationType === WorkflowOperationType.Publish) { + console.log(ansiColors.cyan('\nChecking source instance publish status...')); + const publishStatus = checkSourcePublishStatus( + mappingResult.contentMappings, + mappingResult.pageMappings, + source, + locales + ); + + // Report status check warnings + if (publishStatus.errors.length > 0) { + console.log(ansiColors.yellow(`${publishStatus.errors.length} items not found in source files (will be included)`)); + } + + // Report filtering results + const totalContentMapped = mappingResult.contentIds.length; + const totalPagesMapped = mappingResult.pageIds.length; + const contentPublishedInSource = publishStatus.publishedContentIds.length; + const pagesPublishedInSource = publishStatus.publishedPageIds.length; + const contentSkipped = publishStatus.unpublishedContentIds.length; + const pagesSkipped = publishStatus.unpublishedPageIds.length; + + console.log(ansiColors.gray(`Content: ${contentPublishedInSource}/${totalContentMapped} published in source (${contentSkipped} staging/unpublished skipped)`)); + console.log(ansiColors.gray(`Pages: ${pagesPublishedInSource}/${totalPagesMapped} published in source (${pagesSkipped} staging/unpublished skipped)`)); + + // Filter IDs based on publish mode AND source publish status + contentIds = options.processContent ? publishStatus.publishedContentIds : []; + pageIds = options.processPages ? publishStatus.publishedPageIds : []; + } else { + // For non-publish operations, use all mapped IDs + contentIds = options.processContent ? mappingResult.contentIds : []; + pageIds = options.processPages ? mappingResult.pageIds : []; + } + } + + const modeDescription = options.processContent && options.processPages + ? 'content and pages' + : options.processContent + ? 'content only' + : 'pages only'; + + console.log(ansiColors.cyan(`\n${operationName.charAt(0).toUpperCase() + operationName.slice(1)}ing ${modeDescription}...`)); + console.log(ansiColors.gray(`Content items to ${operationName}: ${contentIds.length}`)); + console.log(ansiColors.gray(`Pages to ${operationName}: ${pageIds.length}`)); + + // DRY RUN: Show preview and exit without executing + if (state.dryRun) { + console.log(ansiColors.yellow('\n' + '═'.repeat(50))); + console.log(ansiColors.yellow(`🔍 DRY RUN PREVIEW - ${operationName.toUpperCase()}`)); + console.log(ansiColors.yellow('═'.repeat(50))); + console.log(ansiColors.gray('\nThe following items would be processed:')); + + if (contentIds.length > 0) { + console.log(ansiColors.cyan(`\n📄 Content Items (${contentIds.length}):`)); + const displayContentIds = contentIds.slice(0, 20); + displayContentIds.forEach(id => console.log(ansiColors.white(` • ID: ${id}`))); + if (contentIds.length > 20) { + console.log(ansiColors.gray(` ... and ${contentIds.length - 20} more content items`)); + } + } + + if (pageIds.length > 0) { + console.log(ansiColors.cyan(`\n📑 Pages (${pageIds.length}):`)); + const displayPageIds = pageIds.slice(0, 20); + displayPageIds.forEach(id => console.log(ansiColors.white(` • ID: ${id}`))); + if (pageIds.length > 20) { + console.log(ansiColors.gray(` ... and ${pageIds.length - 20} more pages`)); + } + } + + console.log(ansiColors.yellow('\n' + '─'.repeat(50))); + console.log(ansiColors.yellow('⚠️ DRY RUN COMPLETE - No changes were made')); + console.log(ansiColors.gray(`Remove --dryRun flag to execute the ${operationName} operation`)); + console.log(ansiColors.yellow('─'.repeat(50))); + + result.contentProcessed = contentIds.length; + result.pagesProcessed = pageIds.length; + result.elapsedTime = Date.now() - startTime; + finalizeLogger(); + return result; + } + + // Execute workflow operation + console.log(ansiColors.cyan('\n' + '─'.repeat(50))); + console.log(ansiColors.cyan(`🚀 ${operationName.toUpperCase()} PHASE (${modeDescription})`)); + console.log(ansiColors.cyan('─'.repeat(50))); + + const workflowResult = await workflowOrchestrator(contentIds, pageIds, options); + + // Update results + result.contentProcessed = workflowResult.contentResults.processed; + result.contentFailed = workflowResult.contentResults.failed; + result.pagesProcessed = workflowResult.pageResults.processed; + result.pagesFailed = workflowResult.pageResults.failed; + result.success = workflowResult.success; + result.errors = workflowResult.errors; + + // If items were published, refresh target instance data and update mappings + if (operationType === WorkflowOperationType.Publish && + (workflowResult.contentResults.processed > 0 || workflowResult.pageResults.processed > 0)) { + await refreshAndUpdateMappings( + workflowResult.contentResults.processedIds, + workflowResult.pageResults.processedIds, + source, + target, + primaryLocale, + workflowResult.logLines // Pass publish log lines to include in log file + ); + } + + // Final summary + result.elapsedTime = Date.now() - startTime; + const totalProcessed = result.contentProcessed + result.pagesProcessed; + const totalFailed = result.contentFailed + result.pagesFailed; + const totalSeconds = Math.floor(result.elapsedTime / 1000); + + console.log(ansiColors.cyan('\n' + '═'.repeat(50))); + console.log(ansiColors.cyan(`📊 ${operationName.toUpperCase()} COMPLETE`)); + console.log(ansiColors.cyan('═'.repeat(50))); + console.log(ansiColors.green(`✓ Processed: ${totalProcessed} items`)); + if (totalFailed > 0) { + console.log(ansiColors.red(`✗ Failed: ${totalFailed} items`)); + } + console.log(ansiColors.gray(`Total time: ${Math.floor(totalSeconds / 60)}m ${totalSeconds % 60}s`)); + + if (result.errors.length > 0) { + console.log(ansiColors.yellow('\nErrors encountered:')); + result.errors.forEach(err => console.log(ansiColors.red(` - ${err}`))); + } + + } catch (error: any) { + result.success = false; + result.errors.push(error.message); + console.error(ansiColors.red(`\n❌ Workflow operation failed: ${error.message}`)); + } + + finalizeLogger(); + result.elapsedTime = Date.now() - startTime; + return result; + } + + /** + * List available mapping pairs for workflow operations + */ + listMappings(): void { + listMappings(); + } +} diff --git a/src/lib/workflows/workflow-options.ts b/src/lib/workflows/workflow-options.ts new file mode 100644 index 0000000..e8903c3 --- /dev/null +++ b/src/lib/workflows/workflow-options.ts @@ -0,0 +1,89 @@ +/** + * Workflow Options Parsing + * + * Functions for parsing and converting workflow operation options. + */ + +import { WorkflowOperationType, WorkflowOptions } from '../../types'; + +/** + * Convert string operation type to WorkflowOperationType enum + */ +export function parseOperationType(operationType: string | undefined): WorkflowOperationType { + if (!operationType) return WorkflowOperationType.Publish; + + switch (operationType.toLowerCase()) { + case 'publish': + return WorkflowOperationType.Publish; + case 'unpublish': + return WorkflowOperationType.Unpublish; + case 'approve': + return WorkflowOperationType.Approve; + case 'decline': + return WorkflowOperationType.Decline; + case 'requestapproval': + case 'request-approval': + case 'request_approval': + return WorkflowOperationType.RequestApproval; + default: + return WorkflowOperationType.Publish; + } +} + +/** + * Parse workflow options from state/command args + */ +export function parseWorkflowOptions( + operationType: string | boolean | WorkflowOperationType, + locale: string +): WorkflowOptions | null { + if (!operationType) return null; + + // Default operation is Publish + let operation = WorkflowOperationType.Publish; + let processContent = true; + let processPages = true; + + // Handle string operation types + if (typeof operationType === 'string') { + const value = operationType.toLowerCase(); + + // Check for operation type + switch (value) { + case 'publish': + operation = WorkflowOperationType.Publish; + break; + case 'unpublish': + operation = WorkflowOperationType.Unpublish; + break; + case 'approve': + operation = WorkflowOperationType.Approve; + break; + case 'decline': + operation = WorkflowOperationType.Decline; + break; + case 'requestapproval': + case 'request-approval': + case 'request_approval': + operation = WorkflowOperationType.RequestApproval; + break; + case 'content': + processPages = false; + break; + case 'pages': + processContent = false; + break; + case 'true': + // Default behavior - process both + break; + default: + // Unrecognized value - default to publish both + break; + } + } else if (typeof operationType === 'number') { + // Direct WorkflowOperationType enum value + operation = operationType; + } + + return { processContent, processPages, locale, operation }; +} diff --git a/src/lib/workflows/workflow-orchestrator.ts b/src/lib/workflows/workflow-orchestrator.ts new file mode 100644 index 0000000..19c3ed0 --- /dev/null +++ b/src/lib/workflows/workflow-orchestrator.ts @@ -0,0 +1,76 @@ +/** + * Workflow Orchestrator + * + * Orchestrates batch workflow operations for content items and pages. + */ + +import ansiColors from 'ansi-colors'; +import { processBatches } from './process-batches'; +import { getOperationName, getOperationVerb } from './workflow-helpers'; +import { WorkflowOrchestratorResult, WorkflowOptions } from '../../types'; + +/** + * Helper to log to both console and capture lines + */ +function logLine(line: string, logLines: string[]): void { + console.log(line); + logLines.push(line); +} + +/** + * Orchestrate workflow operations for content items and pages + * Processes items in batches and reports progress + */ +export async function workflowOrchestrator( + contentIds: number[], + pageIds: number[], + options: WorkflowOptions +): Promise { + const errors: string[] = []; + const logLines: string[] = []; + const { locale, processContent, processPages, operation } = options; + const operationName = getOperationName(operation); + const operationVerb = getOperationVerb(operation); + + // Process content and pages + const contentResults = processContent + ? await processBatches(contentIds, 'content', locale, operation, errors) + : { total: 0, processed: 0, failed: 0, batches: 0, processedIds: [], logLines: [] }; + + // Collect content log lines + logLines.push(...contentResults.logLines); + + const pageResults = processPages + ? await processBatches(pageIds, 'pages', locale, operation, errors) + : { total: 0, processed: 0, failed: 0, batches: 0, processedIds: [], logLines: [] }; + + // Collect page log lines + logLines.push(...pageResults.logLines); + + // Summary + const totalProcessed = contentResults.processed + pageResults.processed; + const totalFailed = contentResults.failed + pageResults.failed; + const totalRequested = contentResults.total + pageResults.total; + const totalNested = totalProcessed > totalRequested ? totalProcessed - totalRequested : 0; + + if (totalRequested > 0) { + if (totalNested > 0) { + logLine(ansiColors.cyan(`\nWorkflow summary: ${totalProcessed} items ${operationVerb} (${totalRequested} requested + ${totalNested} nested)`), logLines); + } else { + logLine(ansiColors.cyan(`\nWorkflow summary: ${totalProcessed}/${totalRequested} items ${operationVerb} successfully`), logLines); + } + if (totalFailed > 0) { + logLine(ansiColors.yellow(` ${totalFailed} items failed`), logLines); + } + } else { + logLine(ansiColors.gray(`\nNo items to ${operationName}`), logLines); + } + + return { + success: errors.length === 0, + contentResults, + pageResults, + errors, + logLines + }; +} diff --git a/src/tests/setup.ts b/src/tests/setup.ts new file mode 100644 index 0000000..a1749eb --- /dev/null +++ b/src/tests/setup.ts @@ -0,0 +1,20 @@ +/** + * Jest setup file - loads environment variables for testing + * + * Test env file location: src/tests/.env.test + * Copy .env.test.example to .env.test and fill in your test credentials + */ +import dotenv from 'dotenv'; +import path from 'path'; + +// Load .env.test from the tests folder (not project root) +const envPath = path.resolve(__dirname, '.env'); +const result = dotenv.config({ path: envPath }); + +if (result.error) { + console.warn(` +⚠️ Test environment file not found: ${envPath} + Copy src/tests/.env.test.example to src/tests/.env + and fill in your test credentials. +`); +} diff --git a/src/tests/shared/fetch-api-status.integration.test.ts b/src/tests/shared/fetch-api-status.integration.test.ts new file mode 100644 index 0000000..51a66f6 --- /dev/null +++ b/src/tests/shared/fetch-api-status.integration.test.ts @@ -0,0 +1,120 @@ +/** + * Integration tests for Fetch API Status helper + * + * Tests the getFetchApiStatus and waitForFetchApiSync functions + * against a real Agility CMS instance. + * + * Setup: + * 1. Copy src/tests/env.test.example to src/tests/.env + * 2. Fill in your test credentials + * + * Required env vars in src/tests/.env: + * - AGILITY_TOKEN - Valid authentication token + * - AGILITY_GUID or AGILITY_TARGET_GUID - Instance GUID to check + * + * Run with: npm run test:integration + */ + +// Disable SSL certificate verification for local development +process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; + +import { getFetchApiStatus, waitForFetchApiSync, FetchApiStatus } from '../../lib/shared/get-fetch-api-status'; +import { state } from '../../core/state'; +import * as mgmtApi from '@agility/management-sdk'; +import { primeFromEnv } from '../../core/state'; + +describe('Fetch API Status - Integration Tests', () => { + let testGuid: string; + + beforeAll(async () => { + // Prime state from .env + primeFromEnv(); + + // Get required environment variables + const token = process.env.AGILITY_TOKEN2 || process.env.AGILITY_TOKEN; + testGuid = process.env.AGILITY_GUID || process.env.AGILITY_TARGET_GUID || ''; + const baseUrl = process.env.AGILITY_BASE_URL || process.env.BASE_URL; + + if (!token) { + throw new Error('AGILITY_TOKEN is required in .env for integration tests'); + } + if (!testGuid) { + throw new Error('AGILITY_GUID or AGILITY_TARGET_GUID is required in .env for integration tests'); + } + + // Initialize API client with real credentials + const options: mgmtApi.Options = { + token: token, + baseUrl: baseUrl, + refresh_token: null, + duration: 3000, + retryCount: 500, + }; + + const apiClient = new mgmtApi.ApiClient(options); + + // Set state for the helper functions + state.mgmtApiOptions = options; + state.cachedApiClient = apiClient; + }); + + describe('getFetchApiStatus', () => { + it('should return sync status for fetch mode', async () => { + const status = await getFetchApiStatus(testGuid, 'fetch', false); + + expect(status).toBeDefined(); + expect(typeof status.inProgress).toBe('boolean'); + expect(typeof status.lastContentVersionID).toBe('number'); + expect(typeof status.pushType).toBe('number'); + + // pushType should be 2 for fetch mode + expect(status.pushType).toBe(2); + + }, 30000); + + it('should return sync status for preview mode', async () => { + const status = await getFetchApiStatus(testGuid, 'preview', false); + + expect(status).toBeDefined(); + expect(typeof status.inProgress).toBe('boolean'); + expect(typeof status.lastContentVersionID).toBe('number'); + expect(typeof status.pushType).toBe('number'); + + // pushType should be 1 for preview mode + expect(status.pushType).toBe(1); + + }, 30000); + }); + + describe('waitForFetchApiSync', () => { + it('should wait for sync to complete (or return immediately if not syncing)', async () => { + const startTime = Date.now(); + const result = await waitForFetchApiSync(testGuid, 'fetch', true); + const elapsed = Date.now() - startTime; + + expect(result).toBeDefined(); + expect(result.status).toBeDefined(); + expect(result.status.inProgress).toBe(false); + expect(Array.isArray(result.logLines)).toBe(true); + + }, 120000); // 2 minute timeout for waiting + }); + + describe('error handling', () => { + it('should handle invalid GUID gracefully', async () => { + // Temporarily override the state with invalid credentials + const originalClient = state.cachedApiClient; + + try { + // This should throw or return an error + await getFetchApiStatus('invalid-guid-xxx', 'fetch', false); + // If it doesn't throw, that's also acceptable (API might return a default) + } catch (error: any) { + expect(error).toBeDefined(); + } + + // Restore original client + state.cachedApiClient = originalClient; + }, 30000); + }); +}); diff --git a/src/tests/workflows/batch-workflows.integration.test.ts b/src/tests/workflows/batch-workflows.integration.test.ts new file mode 100644 index 0000000..de4ad04 --- /dev/null +++ b/src/tests/workflows/batch-workflows.integration.test.ts @@ -0,0 +1,155 @@ +/** + * Integration tests for batch workflow operations + * Tests workflow operations (publish, unpublish, etc.) on content and pages using a REAL API client + * + * Setup: + * 1. Copy src/tests/env.test.example to src/tests/.env.test + * 2. Fill in your test credentials + * + * Required env vars in src/tests/.env.test: + * - AGILITY_TOKEN - Valid authentication token + * - AGILITY_TARGET_GUID or AGILITY_GUID - Target instance GUID + * - AGILITY_LOCALE or AGILITY_LOCALES - Locale(s) for testing + * - CONTENTIDS_TO_BATCH_PUBLISH - Comma-separated content IDs + * - PAGES_TO_BATCH_PUBLISH - Comma-separated page IDs + * + * Run with: npm test -- --testPathPattern="integration" + */ + +// Disable SSL certificate verification for local development +process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; + +import { batchWorkflow } from '../../core/batch-workflows'; +import { WorkflowOperationType } from '@agility/management-sdk'; +import { state } from '../../core/state'; +import * as mgmtApi from '@agility/management-sdk'; +import { primeFromEnv } from '../../core/state'; + +// Helper function to parse comma-separated IDs from environment variable +function parseIDs(envVar: string | undefined, fallback: number[]): number[] { + if (!envVar) return fallback; + return envVar + .split(',') + .map(id => parseInt(id.trim(), 10)) + .filter(id => !isNaN(id)); +} + +// Get test data from environment variables +const TEST_CONTENT_IDS = parseIDs(process.env.CONTENTIDS_TO_BATCH_PUBLISH, []); +const TEST_PAGE_IDS = parseIDs(process.env.PAGES_TO_BATCH_PUBLISH, []); +const TEST_LOCALE = process.env.AGILITY_LOCALE || process.env.AGILITY_LOCALES?.split(',')[0] || 'en-us'; +const BASE_URL = process.env.AGILITY_BASE_URL || process.env.BASE_URL || 'https://api.agilitycms.com'; + +describe('Batch Workflow Operations - Integration Tests', () => { + let apiClient: mgmtApi.ApiClient; + + beforeAll(async () => { + // Prime state from .env + primeFromEnv(); + + // Get required environment variables + const token = process.env.AGILITY_TOKEN2 || process.env.AGILITY_TOKEN; + const targetGuid = process.env.AGILITY_TARGET_GUID || process.env.AGILITY_GUID; + + if (!token) { + throw new Error('AGILITY_TOKEN is required in .env for integration tests'); + } + if (!targetGuid) { + throw new Error('AGILITY_TARGET_GUID or AGILITY_GUID is required in .env for integration tests'); + } + + // Initialize API client with real credentials + const options: mgmtApi.Options = { + token: token, + baseUrl: BASE_URL, + refresh_token: null, + duration: 3000, + retryCount: 500, + }; + + apiClient = new mgmtApi.ApiClient(options); + + // Set state for the workflow functions + state.targetGuid = [targetGuid]; + state.mgmtApiOptions = options; + state.cachedApiClient = apiClient; + }); + + // ============================================================================ + // Content Workflow Operations + // ============================================================================ + + describe('Content Workflow Operations', () => { + beforeAll(() => { + if (TEST_CONTENT_IDS.length === 0) { + console.warn('CONTENTIDS_TO_BATCH_PUBLISH not set - content tests will be skipped'); + } + }); + + it('should run publish workflow operation on content items', async () => { + if (TEST_CONTENT_IDS.length === 0) return; + + const result = await batchWorkflow(TEST_CONTENT_IDS, TEST_LOCALE, WorkflowOperationType.Publish, 'content'); + expect(result.success).toBe(true); + expect(result.processedIds).toBeDefined(); + expect(result.processedIds.length).toBeGreaterThan(0); + expect(result.error).toBeUndefined(); + }, 30000); + + it('should run unpublish workflow operation on content items', async () => { + if (TEST_CONTENT_IDS.length === 0) return; + + const result = await batchWorkflow(TEST_CONTENT_IDS, TEST_LOCALE, WorkflowOperationType.Unpublish, 'content'); + expect(result.success).toBe(true); + expect(result.processedIds).toBeDefined(); + expect(result.processedIds.length).toBeGreaterThan(0); + expect(result.error).toBeUndefined(); + }, 30000); + + it('should handle workflow operation on invalid content IDs gracefully', async () => { + const invalidContentIDs = [999999, 999998]; + const result = await batchWorkflow(invalidContentIDs, TEST_LOCALE, WorkflowOperationType.Publish, 'content'); + expect(result).toBeDefined(); + expect(result.success !== undefined).toBe(true); + }, 30000); + }); + + // ============================================================================ + // Page Workflow Operations + // ============================================================================ + + describe('Page Workflow Operations', () => { + beforeAll(() => { + if (TEST_PAGE_IDS.length === 0) { + console.warn('PAGES_TO_BATCH_PUBLISH not set - page tests will be skipped'); + } + }); + + it('should run publish workflow operation on pages', async () => { + if (TEST_PAGE_IDS.length === 0) return; + + const result = await batchWorkflow(TEST_PAGE_IDS, TEST_LOCALE, WorkflowOperationType.Publish, 'pages'); + expect(result.success).toBe(true); + expect(result.processedIds).toBeDefined(); + expect(result.processedIds.length).toBeGreaterThan(0); + expect(result.error).toBeUndefined(); + }, 30000); + + it('should run unpublish workflow operation on pages', async () => { + if (TEST_PAGE_IDS.length === 0) return; + + const result = await batchWorkflow(TEST_PAGE_IDS, TEST_LOCALE, WorkflowOperationType.Unpublish, 'pages'); + expect(result.success).toBe(true); + expect(result.processedIds).toBeDefined(); + expect(result.processedIds.length).toBeGreaterThan(0); + expect(result.error).toBeUndefined(); + }, 30000); + + it('should handle workflow operation on invalid page IDs gracefully', async () => { + const invalidPageIDs = [999999, 999998]; + const result = await batchWorkflow(invalidPageIDs, TEST_LOCALE, WorkflowOperationType.Publish, 'pages'); + expect(result).toBeDefined(); + expect(result.success !== undefined).toBe(true); + }, 30000); + }); +}); diff --git a/src/types/index.ts b/src/types/index.ts index 64e2680..19dc096 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -12,4 +12,7 @@ export * from './cliToken'; // Note: comparisonResult.ts doesn't export anything, skipping // ReferenceMapperV2 types -export * from './referenceMapperV2'; \ No newline at end of file +export * from './referenceMapperV2'; + +// Workflow types (batch workflows, mappings, publish status) +export * from './workflows'; \ No newline at end of file diff --git a/src/types/sourceData.ts b/src/types/sourceData.ts index cf5ccf0..229a379 100644 --- a/src/types/sourceData.ts +++ b/src/types/sourceData.ts @@ -35,7 +35,7 @@ export interface PusherResult { failed: number; skipped: number; status: 'success' | 'error'; - publishableIds?: number[]; // Optional: target instance IDs for auto-publishing (content items and pages only) + publishableIds?: number[]; // Optional: target instance IDs for workflow operations (content items and pages only) } /** diff --git a/src/types/workflows.ts b/src/types/workflows.ts new file mode 100644 index 0000000..cc6785b --- /dev/null +++ b/src/types/workflows.ts @@ -0,0 +1,164 @@ +/** + * Workflow Types + * + * Type definitions for batch workflow operations, mappings, and publish status checking. + */ + +import { WorkflowOperationType } from '@agility/management-sdk'; + +// Re-export WorkflowOperationType for convenience +export { WorkflowOperationType }; + +// ============================================================================ +// Batch Workflow Types +// ============================================================================ + +/** + * Result from a batch workflow operation + */ +export interface BatchWorkflowResult { + success: boolean; + processedIds: number[]; + failedCount: number; + error?: string; +} + +/** + * Combined result from workflow orchestration + */ +export interface WorkflowOrchestratorResult { + success: boolean; + contentResults: { + total: number; + processed: number; + failed: number; + batches: number; + processedIds: number[]; + }; + pageResults: { + total: number; + processed: number; + failed: number; + batches: number; + processedIds: number[]; + }; + errors: string[]; + logLines: string[]; +} + +/** + * Options for workflow operations + */ +export interface WorkflowOptions { + processContent: boolean; + processPages: boolean; + locale: string; + operation: WorkflowOperationType; +} + +/** + * Result from a workflow operation command + */ +export interface WorkflowOperationResult { + success: boolean; + contentProcessed: number; + contentFailed: number; + pagesProcessed: number; + pagesFailed: number; + elapsedTime: number; + errors: string[]; + operation: string; +} + +// ============================================================================ +// Mapping Types +// ============================================================================ + +/** + * Content item mapping between source and target instances + */ +export interface ContentMapping { + sourceGuid: string; + targetGuid: string; + sourceContentID: number; + targetContentID: number; + sourceVersionID: number; + targetVersionID: number; +} + +/** + * Page mapping between source and target instances + */ +export interface PageMapping { + sourceGuid: string; + targetGuid: string; + sourcePageID: number; + targetPageID: number; + sourceVersionID: number; + targetVersionID: number; + sourcePageTemplateName: string | null; + targetPageTemplateName: string | null; +} + +/** + * Result from reading mappings + */ +export interface MappingReadResult { + contentIds: number[]; + pageIds: number[]; + contentMappings: ContentMapping[]; + pageMappings: PageMapping[]; + errors: string[]; +} + +/** + * Result from updating mappings after publishing + */ +export interface MappingUpdateResult { + contentMappingsUpdated: number; + pageMappingsUpdated: number; + errors: string[]; +} + +// ============================================================================ +// Publish Status Types +// ============================================================================ + +/** + * Item state values from the Agility CMS ItemState enum + */ +export enum ItemState { + New = -1, + None = 0, + Staging = 1, + Published = 2, + Deleted = 3, + Approved = 4, + AwaitingApproval = 5, + Declined = 6, + Unpublished = 7 +} + +/** + * Source item data structure for publish status checking + */ +export interface SourceItemData { + contentID?: number; + pageID?: number; + properties: { + state: number; + modified: string; + versionID: number; + }; +} + +/** + * Result from checking publish status of source items + */ +export interface PublishStatusResult { + publishedContentIds: number[]; + unpublishedContentIds: number[]; + publishedPageIds: number[]; + unpublishedPageIds: number[]; + errors: string[]; +} diff --git a/yarn.lock b/yarn.lock index c2bffc9..2a70273 100644 --- a/yarn.lock +++ b/yarn.lock @@ -24,10 +24,10 @@ dotenv "^8.2.0" proper-lockfile "^4.1.2" -"@agility/management-sdk@^0.1.33": - version "0.1.33" - resolved "https://registry.npmjs.org/@agility/management-sdk/-/management-sdk-0.1.33.tgz" - integrity sha512-+lxk49pi4nPJO+jZLECYGEp1U30HApwtoRGZSNGkboOAsWkCIzP4URcZPLgOExRt5iiXjKSCLxMcryR44q5h2g== +"@agility/management-sdk@^0.1.38": + version "0.1.38" + resolved "https://registry.npmjs.org/@agility/management-sdk/-/management-sdk-0.1.38.tgz" + integrity sha512-g6/hNgCjf+uzcJbkPaxeWqwaAid50tMnRW0KYQ4J/fGuJqtcZG0et0X3m76Ev3yUqnkpIO/+C5zMIQMghN3/oQ== dependencies: axios "^0.27.2" @@ -1699,10 +1699,10 @@ follow-redirects@^1.14.0, follow-redirects@^1.14.9: resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz" integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== -form-data@^4.0.0: - version "4.0.3" - resolved "https://registry.npmjs.org/form-data/-/form-data-4.0.3.tgz" - integrity sha512-qsITQPfmvMOSAdeyZ+12I1c+CKSstAFAwu+97zrnWAbIr5u8wfsExUzCesVLC8NgHuRUqNN4Zy6UPWUTRGslcA== +form-data@^4.0.0, form-data@^4.0.5: + version "4.0.5" + resolved "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz" + integrity sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w== dependencies: asynckit "^0.4.0" combined-stream "^1.0.8" @@ -1720,11 +1720,6 @@ fs.realpath@^1.0.0: resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -fsevents@^2.3.2: - version "2.3.3" - resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz" - integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== - function-bind@^1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz" From 5432680186e0c01e542265dd759f3507a53ccb14 Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 18 Feb 2026 10:23:43 -0500 Subject: [PATCH 08/19] Latest --- package-lock.json | 9 + package.json | 2 +- src/core/auth.ts | 142 +++++++- src/core/batch-workflows.ts | 298 ++++++++++++++++- src/core/fileOperations.ts | 68 +++- src/core/logs.ts | 177 ++++++++-- src/core/push.ts | 312 +++++++++++++++--- src/core/state.ts | 186 +++++++++-- src/lib/downloaders/download-sync-sdk.ts | 11 +- .../downloaders/orchestrate-downloaders.ts | 23 +- .../downloaders/store-interface-filesystem.ts | 7 +- .../models/model-dependency-tree-builder.ts | 6 +- src/lib/pushers/asset-pusher.ts | 176 ++++++++-- src/lib/pushers/batch-polling.ts | 289 ++++++++++++---- .../content-pusher/content-batch-processor.ts | 22 +- .../pushers/content-pusher/content-pusher.ts | 52 ++- .../content-pusher/util/change-detection.ts | 30 +- .../filter-content-items-for-processing.ts | 24 +- .../util/find-content-in-target-instance.ts | 9 +- src/lib/pushers/gallery-pusher.ts | 111 +++++-- src/lib/pushers/orchestrate-pushers.ts | 104 ++++-- src/lib/pushers/page-pusher/process-page.ts | 118 ++++--- .../pushers/page-pusher/process-sitemap.ts | 51 ++- src/lib/pushers/page-pusher/push-pages.ts | 22 +- .../shared/source-publish-status-checker.ts | 12 +- src/lib/workflows/process-batches.ts | 38 ++- src/lib/workflows/workflow-operation.ts | 64 +++- src/types/sourceData.ts | 14 + src/types/workflows.ts | 7 + yarn.lock | 5 + 30 files changed, 2028 insertions(+), 361 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7a9be0a..e4e8f37 100644 --- a/package-lock.json +++ b/package-lock.json @@ -83,6 +83,7 @@ "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", "license": "MIT", + "peer": true, "dependencies": { "follow-redirects": "^1.14.0" } @@ -154,6 +155,7 @@ "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.27.1", @@ -1462,6 +1464,7 @@ "integrity": "sha512-hcxGs9TfQGghOM8atpRT+bBMUX7V8WosdYt98bQ59wUToJck55eCOlemJ+0FpOZOQw5ff7LSi9+IO56KvYEFyQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~5.26.4" } @@ -1950,6 +1953,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001726", "electron-to-chromium": "^1.5.173", @@ -3394,6 +3398,7 @@ "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.6.tgz", "integrity": "sha512-M1WuAmb7pn9zdFRtQYk26ZBoY043Sse0wVDdk4Bppr+JOXyQYybdtvK+l9wUibhtjdjvtoiNy8tk+EgsYIUqKg==", "license": "MIT", + "peer": true, "dependencies": { "ansi-escapes": "^4.2.1", "chalk": "^4.1.1", @@ -4461,6 +4466,7 @@ "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -6098,6 +6104,7 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -7590,6 +7597,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -7697,6 +7705,7 @@ "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/package.json b/package.json index 2d10907..c6bc1ba 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agility/cli", - "version": "1.0.0-beta.13", + "version": "1.0.0-beta.13.1", "description": "Agility CLI for working with your content. (Public Beta)", "repository": { "type": "git", diff --git a/src/core/auth.ts b/src/core/auth.ts index 9d60d92..4efc1a4 100644 --- a/src/core/auth.ts +++ b/src/core/auth.ts @@ -199,6 +199,73 @@ export class Auth { return "https://mgmt.aglty.io"; } + /** + * Determine the Content Fetch API URL based on GUID suffix. + * NOTE: This is separate from the Management API URL. + * The Fetch API is always cloud-based, even when running --local for management. + * --local only affects the Management API, not the Content Fetch/Sync API. + */ + determineFetchUrl(guid?: string): string { + let baseGUID = guid; + if (!baseGUID) { + baseGUID = state.sourceGuid[0]; + } + + // Content Fetch API URLs are determined by GUID suffix only + // --local, --dev, --preprod flags do NOT affect the fetch URL + if (baseGUID) { + switch (true) { + case baseGUID.endsWith("d"): + return "https://api-dev.aglty.io"; + case baseGUID.endsWith("u"): + return "https://api.aglty.io"; + case baseGUID.endsWith("c"): + return "https://api-ca.aglty.io"; + case baseGUID.endsWith("e"): + return "https://api-eu.aglty.io"; + case baseGUID.endsWith("a"): + return "https://api-aus.aglty.io"; + case baseGUID.endsWith("us2"): + return "https://api-usa2.aglty.io"; + } + } + // Default to US + return "https://api.aglty.io"; + } + + /** + * Determine the cloud Management API URL based on GUID suffix. + * This IGNORES --local flag and always returns the cloud URL. + * Used for operations that must hit the cloud (e.g., getting API keys). + */ + determineCloudMgmtUrl(guid?: string): string { + let baseGUID = guid; + if (!baseGUID) { + baseGUID = state.sourceGuid[0]; + } + + // Cloud Management API URLs are determined by GUID suffix only + // --local flag is ignored - these endpoints must hit the cloud + if (baseGUID) { + switch (true) { + case baseGUID.endsWith("d"): + return "https://mgmt-dev.aglty.io"; + case baseGUID.endsWith("u"): + return "https://mgmt.aglty.io"; + case baseGUID.endsWith("c"): + return "https://mgmt-ca.aglty.io"; + case baseGUID.endsWith("e"): + return "https://mgmt-eu.aglty.io"; + case baseGUID.endsWith("a"): + return "https://mgmt-aus.aglty.io"; + case baseGUID.endsWith("us2"): + return "https://mgmt-usa2.aglty.io"; + } + } + // Default to US + return "https://mgmt.aglty.io"; + } + getBaseUrl(guid: string, userBaseUrl: string = null): string { let baseUrl = this.determineBaseUrl(guid); return `${baseUrl}/oauth`; @@ -293,6 +360,15 @@ export class Auth { redirectUri )}&state=cli-code%2e${code}`; + // Debug logging for local development + if (state.local || state.dev) { + console.log("\n🔍 OAuth Debug Info:"); + console.log(` Base URL: ${baseUrl}`); + console.log(` Redirect URI: ${redirectUri}`); + console.log(` Full Auth URL: ${authUrl}`); + console.log(""); + } + await open(authUrl); return code; } @@ -356,12 +432,16 @@ export class Auth { const mgmtApiOptions = new (await import("@agility/management-sdk")).Options(); mgmtApiOptions.token = await this.getToken(); - // // Store basic mgmt API options in state + // CRITICAL: Set baseUrl for local/dev/preprod modes BEFORE creating the client + // This ensures getLocales() and other SDK calls use the correct endpoint + const baseUrl = this.determineBaseUrl(); + mgmtApiOptions.baseUrl = baseUrl; + + // Store basic mgmt API options in state state.mgmtApiOptions = mgmtApiOptions; + state.baseUrl = baseUrl; - // Clear cached API client to ensure fresh connection with new auth state - // const { clearApiClient } = await import('./state'); - // clearApiClient(); + // Create the cached API client with proper configuration state.cachedApiClient = new mgmtApi.ApiClient(state.mgmtApiOptions); // Load user data for interactive prompts and general use @@ -770,9 +850,28 @@ export class Auth { async getPreviewKey(guid: string, userBaseUrl: string = null) { try { - const result = await this.executeGet("/GetPreviewKey?guid=" + guid, guid, userBaseUrl); - // The API returns a raw string, not a JSON object with a previewKey property - return result; + // Use determineBaseUrl to respect --local flag (localhost:5050) + const baseUrl = this.determineBaseUrl(guid); + const url = `${baseUrl}/oauth/GetPreviewKey?guid=${guid}`; + const token = await this.getToken(); + + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${token}`, + "Cache-Control": "no-cache", + "User-Agent": "agility-cli-fetch/1.0", + }, + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText} for ${url}`); + } + + const textResponse = await response.text(); + return textResponse.startsWith('"') && textResponse.endsWith('"') + ? textResponse.slice(1, -1) + : textResponse; } catch (err) { throw err; } @@ -780,9 +879,28 @@ export class Auth { async getFetchKey(guid: string, userBaseUrl: string = null) { try { - const result = await this.executeGet("/GetFetchKey?guid=" + guid, guid, userBaseUrl); - // The API returns a raw string, not a JSON object with a fetchKey property - return result; + // Use determineBaseUrl to respect --local flag (localhost:5050) + const baseUrl = this.determineBaseUrl(guid); + const url = `${baseUrl}/oauth/GetFetchKey?guid=${guid}`; + const token = await this.getToken(); + + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${token}`, + "Cache-Control": "no-cache", + "User-Agent": "agility-cli-fetch/1.0", + }, + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText} for ${url}`); + } + + const textResponse = await response.text(); + return textResponse.startsWith('"') && textResponse.endsWith('"') + ? textResponse.slice(1, -1) + : textResponse; } catch (err) { throw err; } @@ -904,10 +1022,12 @@ export class Auth { if (!state.channel) missingFields.push("channel (use --channel or AGILITY_WEBSITE in .env)"); break; + case "push": case "sync": + // Both push and sync require source and target GUIDs if (!state.sourceGuid || state.sourceGuid.length === 0) missingFields.push("sourceGuid (use --sourceGuid or AGILITY_GUID in .env)"); - if (!state.targetGuid || state.targetGuid.length === 0) missingFields.push("targetGuid (use --targetGuid)"); + if (!state.targetGuid || state.targetGuid.length === 0) missingFields.push("targetGuid (use --targetGuid or AGILITY_TARGET_GUID in .env)"); // Check for locales: either user-specified OR auto-detected per-GUID mappings const hasSyncUserLocales = state.locale && state.locale.length > 0; diff --git a/src/core/batch-workflows.ts b/src/core/batch-workflows.ts index 69dde20..bfbcd67 100644 --- a/src/core/batch-workflows.ts +++ b/src/core/batch-workflows.ts @@ -20,10 +20,60 @@ export { getOperationName, getOperationVerb, getOperationIcon } from '../lib/wor export { parseWorkflowOptions, parseOperationType } from '../lib/workflows/workflow-options'; /** - * Batch size for processing - prevents API throttling + * Batch size for processing - smaller batches for workflow operations + * to prevent API timeouts during publish/unpublish */ const BATCH_SIZE = 250; +/** + * Progress indicator interval (log a dot every 2 seconds while waiting) + */ +const PROGRESS_INTERVAL_MS = 2000; + +/** + * Batch polling configuration - increased for workflow operations + * 600 retries × 3s = 30 minutes max wait time + */ +const BATCH_POLL_MAX_RETRIES = 600; +const BATCH_POLL_INTERVAL_MS = 3000; + +/** + * Run a promise with progress indicator logging + */ +async function withProgressIndicator( + promise: Promise, + label: string +): Promise { + let dotCount = 0; + const startTime = Date.now(); + + // Log progress dots while waiting + const progressInterval = setInterval(() => { + dotCount++; + const elapsed = Math.round((Date.now() - startTime) / 1000); + process.stdout.write(`.`); + // Every 10 dots (20 seconds), log elapsed time + if (dotCount % 10 === 0) { + process.stdout.write(` (${elapsed}s)`); + } + }, PROGRESS_INTERVAL_MS); + + try { + const result = await promise; + clearInterval(progressInterval); + if (dotCount > 0) { + console.log(''); // New line after dots + } + return result; + } catch (error) { + clearInterval(progressInterval); + if (dotCount > 0) { + console.log(''); // New line after dots + } + throw error; + } +} + /** * Extract detailed error message from various error formats */ @@ -61,6 +111,160 @@ function extractErrorDetails(error: any): string { return String(error) || 'Unknown workflow error'; } +/** + * Parse partial success from error data JSON + * Returns {successCount, failureCount, totalItems} if parseable, null otherwise + */ +function parsePartialSuccessFromError(errorMessage: string): { + successCount: number; + failureCount: number; + totalItems: number; + failedItems?: any[]; +} | null { + try { + // Try to extract JSON from error message + const jsonMatch = errorMessage.match(/\{[\s\S]*\}/); + if (jsonMatch) { + const errorData = JSON.parse(jsonMatch[0]); + if (typeof errorData.successCount === 'number' && typeof errorData.failureCount === 'number') { + return { + successCount: errorData.successCount, + failureCount: errorData.failureCount, + totalItems: errorData.totalItems || errorData.successCount + errorData.failureCount, + failedItems: errorData.failedItems + }; + } + } + } catch { + // Not parseable as JSON with partial success info + } + return null; +} + +/** + * Custom batch polling with batch ID tracking and partial success handling + * Polls until batch is complete or timeout, includes batch ID in all error messages + */ +/** + * Create a simple progress bar string + */ +function createProgressBar(percent: number, width: number = 20): string { + const filled = Math.round((percent / 100) * width); + const empty = width - filled; + return `[${'█'.repeat(filled)}${'░'.repeat(empty)}]`; +} + +async function pollBatchWorkflow( + batchID: number, + targetGuid: string, + type: 'content' | 'pages', + totalItems: number, // Total items in this batch for progress display + maxRetries: number = BATCH_POLL_MAX_RETRIES, + intervalMs: number = BATCH_POLL_INTERVAL_MS +): Promise<{ + success: boolean; + processedIds: number[]; + partialSuccess?: { successCount: number; failureCount: number; failedItems?: any[] }; + error?: string; +}> { + const apiClient = getApiClient(); + let retryCount = 0; + let lastBatchState = -1; + const startTime = Date.now(); + + while (retryCount < maxRetries) { + try { + const batch = await apiClient.batchMethods.getBatch(batchID, targetGuid); + const batchType = type === 'content' ? 'Content' : 'Page'; + // BatchState.Processed = 3 + if (batch.batchState === 3) { + // Clear the progress line and show completion + const elapsed = Math.round((Date.now() - startTime) / 1000); + process.stdout.write(ansiColors.yellow.dim(`\r${batchType} batch ${batchID}: ${createProgressBar(100)} ${totalItems}/${totalItems} (${elapsed}s)\n`)); + + // Batch completed - check for errors in errorData + if (batch.errorData && batch.errorData.length > 0) { + // Try to parse partial success from error data + const partialSuccess = parsePartialSuccessFromError(batch.errorData); + + if (partialSuccess && partialSuccess.successCount > 0) { + // Partial success - some items succeeded + const processedIds: number[] = []; + if (batch.items && Array.isArray(batch.items)) { + batch.items.forEach((item: any) => { + if (item.itemID > 0 && !item.errorMessage) { + processedIds.push(item.itemID); + } + }); + } + + return { + success: true, // Treat as success since some items were processed + processedIds, + partialSuccess: { + successCount: partialSuccess.successCount, + failureCount: partialSuccess.failureCount, + failedItems: partialSuccess.failedItems + } + }; + } + + // Full failure + return { + success: false, + processedIds: [], + error: `Batch ${batchID} completed with errors: ${batch.errorData}` + }; + } + + // Success - extract processed IDs + const processedIds: number[] = []; + if (batch.items && Array.isArray(batch.items)) { + batch.items.forEach((item: any) => { + if (item.itemID > 0) { + processedIds.push(item.itemID); + } + }); + } + + return { success: true, processedIds }; + } + + // Still processing - show progress with numItemsProcessed + const numProcessed = typeof batch.numItemsProcessed === 'number' ? batch.numItemsProcessed : 0; + const percentComplete = totalItems > 0 ? Math.round((numProcessed / totalItems) * 100) : 0; + const elapsed = Math.round((Date.now() - startTime) / 1000); + process.stdout.write(ansiColors.yellow.dim(`\r${batchType} batch ${batchID}: ${createProgressBar(percentComplete)} ${numProcessed}/${totalItems} (${elapsed}s) `)); + + if (batch.batchState !== lastBatchState) { + lastBatchState = batch.batchState; + } + + retryCount++; + await new Promise(resolve => setTimeout(resolve, intervalMs)); + + } catch (pollError: any) { + // Network error during polling - retry + retryCount++; + if (retryCount >= maxRetries) { + return { + success: false, + processedIds: [], + error: `Batch ${batchID} polling failed after ${maxRetries} retries: ${pollError.message}` + }; + } + await new Promise(resolve => setTimeout(resolve, intervalMs)); + } + } + + // Timeout + return { + success: false, + processedIds: [], + error: `Batch ${batchID} timed out after ${Math.round(maxRetries * intervalMs / 60000)} minutes. Please check the Batches page in the Agility Content Manager app.` + }; +} + /** * Item type for batch workflow operations */ @@ -68,6 +272,7 @@ export type BatchItemType = 'content' | 'pages'; /** * Unified batch workflow operation for content items or pages + * Uses custom polling with batch ID tracking for better error messages * * @param ids - Array of IDs to process * @param locale - Target locale @@ -100,29 +305,86 @@ export async function batchWorkflow( throw new Error(`${label} IDs array is empty`); } - // const operationName = getOperationName(operation); + // Call appropriate SDK method with returnBatchId=true for custom polling + // Get batch ID immediately using returnBatchId=true + const batchIdResult = type === 'content' + ? await apiClient.contentMethods.batchWorkflowContent(ids, targetGuid[0], locale, operation, true) + : await apiClient.pageMethods.batchWorkflowPages(ids, targetGuid[0], locale, operation, true); - // Log the attempt for debugging - // if (state.verbose) { - // console.log(ansiColors.gray(`${operationName}ing ${ids.length} ${label} to ${targetGuid[0]} (${locale})...`)); - // } - - // Call appropriate SDK method based on type - const processedIds = type === 'content' - ? await apiClient.contentMethods.batchWorkflowContent(ids, targetGuid[0], locale, operation, false) - : await apiClient.pageMethods.batchWorkflowPages(ids, targetGuid[0], locale, operation, false); - - return { - success: true, - processedIds, - failedCount: 0 - }; + const batchID = Array.isArray(batchIdResult) ? batchIdResult[0] : batchIdResult; + + if (!batchID || batchID <= 0) { + throw new Error(`Failed to create batch for ${label}`); + } + + // Custom polling with batch ID tracking and progress display + const pollResult = await pollBatchWorkflow(batchID, targetGuid[0], type, ids.length); + + if (pollResult.success) { + // Handle partial success + if (pollResult.partialSuccess) { + const { successCount, failureCount, failedItems } = pollResult.partialSuccess; + console.log(ansiColors.yellow(`\n ⚠️ Batch ${batchID} completed with errors: ${successCount}/${successCount + failureCount} items succeeded`)); + + // Log failed items details if available + if (failedItems && failedItems.length > 0 && state.verbose) { + failedItems.forEach((item: any) => { + console.log(ansiColors.red(` - Item ${item.itemId}: ${item.errorMessage || 'Unknown error'}`)); + }); + } + + return { + success: true, // Partial success is still success + processedIds: pollResult.processedIds, + failedCount: failureCount, + batchId: batchID, + partialSuccess: { + successCount, + failureCount, + batchId: batchID + } + }; + } + + return { + success: true, + processedIds: pollResult.processedIds, + failedCount: 0, + batchId: batchID + }; + } else { + // Full failure + console.error(ansiColors.red(`\n ❌ ${pollResult.error}`)); + return { + success: false, + processedIds: [], + failedCount: ids.length, + batchId: batchID, + error: pollResult.error + }; + } } catch (error: any) { + // Log the full error for debugging + const errorDetails = extractErrorDetails(error); + console.error(ansiColors.red(`\n ❌ Batch ${type} workflow failed: ${errorDetails}`)); + + // Log additional error context if available + if (error.response) { + console.error(ansiColors.gray(` Status: ${error.response.status}`)); + if (error.response.data) { + console.error(ansiColors.gray(` Response: ${JSON.stringify(error.response.data, null, 2).substring(0, 500)}`)); + } + } + if (error.stack && state.verbose) { + console.error(ansiColors.gray(` Stack: ${error.stack}`)); + } + return { success: false, processedIds: [], failedCount: ids.length, - error: extractErrorDetails(error) + error: errorDetails + // No batchId here since we may have failed before getting one }; } } diff --git a/src/core/fileOperations.ts b/src/core/fileOperations.ts index 3c974ce..906a3d5 100644 --- a/src/core/fileOperations.ts +++ b/src/core/fileOperations.ts @@ -89,6 +89,68 @@ export class fileOperations { return cleaned; } + /** + * Sanitize an object by removing non-serializable properties (like HTTPS Agents) + * This prevents "Converting circular structure to JSON" errors when saving SDK responses + */ + private sanitizeForJson(obj: any): any { + if (obj === null || obj === undefined) { + return obj; + } + + if (typeof obj !== 'object') { + return obj; + } + + // Handle arrays + if (Array.isArray(obj)) { + return obj.map(item => this.sanitizeForJson(item)); + } + + // Skip known non-serializable types + const constructorName = obj.constructor?.name; + if (constructorName === 'Agent' || + constructorName === 'ClientRequest' || + constructorName === 'IncomingMessage' || + constructorName === 'Socket' || + constructorName === 'TLSSocket') { + return undefined; + } + + // Create a clean copy of the object + const cleanObj: any = {}; + for (const key of Object.keys(obj)) { + // Skip properties that are likely to contain circular references + if (key === 'agent' || + key === '_httpMessage' || + key === 'socket' || + key === 'connection' || + key === 'request' || + key === 'response' || + key === '_events' || + key === '_eventsCount' || + key === 'httpsAgent' || + key === 'httpAgent') { + continue; + } + + const value = obj[key]; + + // Skip functions + if (typeof value === 'function') { + continue; + } + + // Recursively sanitize nested objects + const sanitizedValue = this.sanitizeForJson(value); + if (sanitizedValue !== undefined) { + cleanObj[key] = sanitizedValue; + } + } + + return cleanObj; + } + exportFiles(folder: string, fileIdentifier: any, extractedObject: any, baseFolder?: string) { let effectiveBase: string; if (baseFolder) { @@ -116,7 +178,11 @@ export class fileOperations { } const fileName = path.join(directoryForFile, `${fileIdentifier}.json`); - fs.writeFileSync(fileName, JSON.stringify(extractedObject)); + + // Sanitize the object to remove non-serializable properties (like HTTPS Agents) + // This prevents "Converting circular structure to JSON" errors with --local mode + const sanitizedObject = this.sanitizeForJson(extractedObject); + fs.writeFileSync(fileName, JSON.stringify(sanitizedObject)); } appendFiles(folder: string, fileIdentifier: any, extractedObject: any) { diff --git a/src/core/logs.ts b/src/core/logs.ts index 51511ed..b83e36c 100644 --- a/src/core/logs.ts +++ b/src/core/logs.ts @@ -4,6 +4,67 @@ import * as fs from "fs"; import * as path from "path"; import { generateLogHeader } from "../lib/shared"; +/** + * Safe JSON stringify that handles circular references and non-serializable objects + * Prevents "Converting circular structure to JSON" errors when logging SDK responses + */ +function safeStringify(obj: any, indent?: number): string { + const seen = new WeakSet(); + + const replacer = (key: string, value: any) => { + // Skip known non-serializable properties + if (key === 'agent' || + key === '_httpMessage' || + key === 'socket' || + key === 'connection' || + key === 'request' || + key === 'response' || + key === '_events' || + key === '_eventsCount' || + key === 'httpsAgent' || + key === 'httpAgent' || + key === 'sockets' || + key === 'freeSockets' || + key === '_currentRequest') { + return undefined; + } + + // Skip functions + if (typeof value === 'function') { + return undefined; + } + + // Handle circular references + if (typeof value === 'object' && value !== null) { + // Skip known non-serializable types + const constructorName = value.constructor?.name; + if (constructorName === 'Agent' || + constructorName === 'ClientRequest' || + constructorName === 'IncomingMessage' || + constructorName === 'Socket' || + constructorName === 'TLSSocket' || + constructorName === 'Writable' || + constructorName === 'ReadableState' || + constructorName === 'WritableState') { + return `[${constructorName}]`; + } + + if (seen.has(value)) { + return '[Circular]'; + } + seen.add(value); + } + + return value; + }; + + try { + return JSON.stringify(obj, replacer, indent); + } catch (e) { + return `[Unable to stringify: ${e.message}]`; + } +} + export type OperationType = "pull" | "push" | "sync"; export type EntityType = @@ -604,12 +665,56 @@ export class Logs { error: (payload: any, apiError: any, targetGuid?: string) => { const itemName = payload?.fileName || payload?.name || `Asset ${payload?.mediaID || "Unknown"}`; - const errorDetails = apiError?.message || apiError || "Unknown error"; - // we need a better error logger for data elements + + // Extract the actual API error message from nested SDK exception structure + let errorDetails = "Unknown error"; + let apiUrl = ""; + let httpStatus = ""; + let responseData = ""; + + if (typeof apiError === 'string') { + errorDetails = apiError; + } else { + // Try multiple paths to find the actual error + const innerErr = apiError?.innerError; + + // Get URL from config + if (innerErr?.config) { + apiUrl = `${innerErr.config.baseURL || ''}${innerErr.config.url || ''}`; + } + + // Get status code + if (innerErr?.response?.status) { + httpStatus = String(innerErr.response.status); + } else if (innerErr?.code) { + httpStatus = innerErr.code; // e.g., ECONNREFUSED, ENOTFOUND + } + + // Get response data (the actual server error message) + if (innerErr?.response?.data) { + const data = innerErr.response.data; + responseData = typeof data === 'string' ? data : JSON.stringify(data); + errorDetails = typeof data === 'string' ? data : (data.message || data.error || data.Message || data.title || JSON.stringify(data)); + } else if (innerErr?.message) { + errorDetails = innerErr.message; + } else if (apiError?.message) { + errorDetails = apiError.message; + } + } + this.logDataElement("asset", "failed", "failed", itemName, targetGuid || this.guid, errorDetails); - - const asset = payload?.asset || payload; - console.log("error", asset); + + // Log comprehensive error details for debugging + console.log(ansiColors.red(` API Error: ${errorDetails}`)); + if (apiUrl) { + console.log(ansiColors.gray(` URL: ${apiUrl}`)); + } + if (httpStatus) { + console.log(ansiColors.gray(` Status: ${httpStatus}`)); + } + if (responseData && responseData !== errorDetails) { + console.log(ansiColors.gray(` Response: ${responseData.substring(0, 500)}`)); + } }, }; @@ -686,52 +791,57 @@ export class Logs { // Content Item logging methods content = { downloaded: (entity: any, details?: string, locale?: string) => { - const itemName = entity?.properties?.referenceName || `${entity?.contentID || "Unknown"}`; + const referenceName = entity?.properties?.referenceName; + const contentID = entity?.contentID; + // Show both referenceName and contentID for debugging: "referenceName (contentID: 123)" + const itemName = referenceName + ? (contentID ? `${referenceName} (contentID: ${contentID})` : referenceName) + : `Content ${contentID || "Unknown"}`; this.logDataElement("content", "downloaded", "success", itemName, this.guid, details, locale); }, uploaded: (entity: any, details?: string, locale?: string, targetGuid?: string) => { - const itemName = - entity?.properties?.referenceName || - entity?.fields?.title || - entity?.fields?.name || - `Content ${entity?.contentID || "Unknown"}`; + const referenceName = entity?.properties?.referenceName || entity?.fields?.title || entity?.fields?.name; + const contentID = entity?.contentID; + const itemName = referenceName + ? (contentID ? `${referenceName} (contentID: ${contentID})` : referenceName) + : `Content ${contentID || "Unknown"}`; this.logDataElement("content", "uploaded", "success", itemName, targetGuid || this.guid, details, locale); }, created: (entity: any, details?: string, locale?: string, targetGuid?: string) => { - const itemName = - entity?.properties?.referenceName || - entity?.fields?.title || - entity?.fields?.name || - `Content ${entity?.contentID || "Unknown"}`; + const referenceName = entity?.properties?.referenceName || entity?.fields?.title || entity?.fields?.name; + const contentID = entity?.contentID; + const itemName = referenceName + ? (contentID ? `${referenceName} (contentID: ${contentID})` : referenceName) + : `Content ${contentID || "Unknown"}`; this.logDataElement("content", "created", "success", itemName, targetGuid || this.guid, details, locale); }, updated: (entity: any, details?: string, locale?: string, targetGuid?: string) => { - const itemName = - entity?.properties?.referenceName || - entity?.fields?.title || - entity?.fields?.name || - `Content ${entity?.contentID || "Unknown"}`; + const referenceName = entity?.properties?.referenceName || entity?.fields?.title || entity?.fields?.name; + const contentID = entity?.contentID; + const itemName = referenceName + ? (contentID ? `${referenceName} (contentID: ${contentID})` : referenceName) + : `Content ${contentID || "Unknown"}`; this.logDataElement("content", "updated", "success", itemName, targetGuid || this.guid, details, locale); }, skipped: (entity: any, details?: string, locale?: string, targetGuid?: string) => { - const itemName = - entity?.properties?.referenceName || - entity?.fields?.title || - entity?.fields?.name || - `Content ${entity?.contentID || "Unknown"}`; + const referenceName = entity?.properties?.referenceName || entity?.fields?.title || entity?.fields?.name; + const contentID = entity?.contentID; + const itemName = referenceName + ? (contentID ? `${referenceName} (contentID: ${contentID})` : referenceName) + : `Content ${contentID || "Unknown"}`; this.logDataElement("content", "skipped", "skipped", itemName, targetGuid || this.guid, details, locale); }, error: (payload: any, apiError: any, locale?: string, targetGuid?: string) => { - const itemName = - payload?.properties?.referenceName || - payload?.fields?.title || - payload?.fields?.name || - `Content ${payload?.contentID || "Unknown"}`; + const referenceName = payload?.properties?.referenceName || payload?.fields?.title || payload?.fields?.name; + const contentID = payload?.contentID; + const itemName = referenceName + ? (contentID ? `${referenceName} (contentID: ${contentID})` : referenceName) + : `Content ${contentID || "Unknown"}`; const errorDetails = apiError?.message || apiError || "Unknown error"; // we need a better error logger for data elements this.logDataElement("content", "error", "failed", itemName, targetGuid || this.guid, errorDetails, locale); @@ -840,8 +950,9 @@ export class Logs { this.logDataElement("gallery", "failed", "failed", itemName, targetGuid || this.guid, errorDetails); console.log(gallery.mediaGroupingID, gallery.name); - console.log(ansiColors.red(JSON.stringify(apiError, null, 2))); - console.log(ansiColors.red(JSON.stringify(payload, null, 2))); + // Use safeStringify to handle circular references in SDK responses (e.g., HTTPS Agents) + console.log(ansiColors.red(safeStringify(apiError, 2))); + console.log(ansiColors.red(safeStringify(payload, 2))); }, }; diff --git a/src/core/push.ts b/src/core/push.ts index 6e12a30..a357b20 100644 --- a/src/core/push.ts +++ b/src/core/push.ts @@ -1,6 +1,6 @@ import * as path from "path"; import * as fs from "fs"; -import { getState, initializeLogger, finalizeLogger, getLogger, state, setState } from "./state"; +import { getState, initializeLogger, finalizeLogger, getLogger, state, setState, clearFailedContentRegistry, getPageCmsLink, getContentCmsLink } from "./state"; import ansiColors from "ansi-colors"; import { markPushStart, clearTimestamps } from "../lib/incremental"; @@ -18,6 +18,9 @@ export class Push { async pushInstances(fromSync: boolean = false): Promise<{ success: boolean; results: any[]; elapsedTime: number }> { const { isSync, sourceGuid, targetGuid, models, modelsWithDeps, autoPublish } = state; + // Clear failed content registry from any previous sync + clearFailedContentRegistry(); + // Initialize logger for push operation // Determine if this is a sync operation by checking if both source and target GUIDs exist initializeLogger(isSync ? "sync" : "push"); @@ -92,25 +95,114 @@ export class Push { } }); - const success = totalFailed === 0; + // Collect log file paths for display at the very end + const logFilePaths = results + .map(res => res.logFilePath) + .filter(path => path) as string[]; + + // Collect sync failure details from results for error summary + let totalSyncFailures = 0; + const syncErrors: Array<{ locale?: string; type: string; error: string }> = []; + const syncFailureDetails: Array<{ name: string; error: string; type?: 'content' | 'page'; pageID?: number; contentID?: number; guid?: string; locale?: string }> = []; + + results.forEach((result: PushResults) => { + // Track item-level failures from totalFailures + if (result.totalFailures > 0) { + totalSyncFailures += result.totalFailures; + } + // Collect individual failure details + if (result.failureDetails && result.failureDetails.length > 0) { + syncFailureDetails.push(...result.failureDetails); + } + // Track operation-level failures + if (result.failed && result.failed.length > 0) { + result.failed.forEach(f => { + syncErrors.push({ type: 'sync', error: `${f.operation}: ${f.error}` }); + }); + } + }); + + // Calculate overall success - check both operation failures and item failures + const success = totalFailed === 0 && totalSyncFailures === 0; - // Use the orchestrator summary function to handle all completion logic + // Use the orchestrator summary function to handle completion logic + // But DON'T show log files yet - we'll show them at the very end const logger = getLogger(); - if (logger) { - - const logFilePaths = results - .map(res => res.logFilePath) - .filter(path => path); - - logger.orchestratorSummary(results, totalElapsedTime, success, logFilePaths); + logger.orchestratorSummary(results, totalElapsedTime, success, []); // Empty array = no log files shown yet } finalizeLogger(); // Finalize global logger if it exists - // Auto-publish if enabled and sync was successful - if (isSync && autoPublish && success) { - await this.executeAutoPublish(results, autoPublish); + // Track all errors for final summary + let autoPublishErrors: Array<{ locale: string; type: string; error: string }> = []; + + // Auto-publish if enabled (failures are expected and shouldn't block publish) + if (isSync && autoPublish) { + autoPublishErrors = await this.executeAutoPublish(results, autoPublish); + } + + // Final error summary - show if there were ANY failures (sync or auto-publish) + const hasFailures = totalSyncFailures > 0 || syncErrors.length > 0 || autoPublishErrors.length > 0; + + if (hasFailures) { + console.log(ansiColors.red('\n' + '═'.repeat(50))); + console.log(ansiColors.red('⚠️ ERROR SUMMARY')); + console.log(ansiColors.red('═'.repeat(50))); + + // Show sync failure details line by line with links + if (syncFailureDetails.length > 0) { + console.log(ansiColors.red(`\n Sync Failures (${syncFailureDetails.length}):`)); + syncFailureDetails.forEach(({ name, error, type, pageID, contentID, guid, locale }) => { + // Format: [guid][locale] • name: error + const prefix = guid && locale ? `[${guid}][${locale}]` : guid ? `[${guid}]` : ''; + console.log(ansiColors.red(` ${prefix} • ${name}: ${error}`)); + // Add link for page failures + if (type === 'page' && pageID && guid && locale) { + const pageLink = getPageCmsLink(guid, locale, pageID); + console.log(ansiColors.gray(` ${pageLink}`)); + // Also show content link if page failed due to missing content mapping + if (contentID) { + const contentLink = getContentCmsLink(guid, locale, contentID); + console.log(ansiColors.gray(` ${contentLink}`)); + } + } + // Add link for content failures + if (type === 'content' && contentID && guid && locale) { + const link = getContentCmsLink(guid, locale, contentID); + console.log(ansiColors.gray(` ${link}`)); + } + }); + } else if (totalSyncFailures > 0) { + // Fallback if no detailed failure info available + console.log(ansiColors.red(` Sync: ${totalSyncFailures} items failed (see details above)`)); + } + + // Show detailed sync errors (operation-level) + if (syncErrors.length > 0) { + console.log(ansiColors.red(`\n Operation Errors:`)); + syncErrors.forEach(({ locale, type, error }) => { + const localeDisplay = locale ? `[${locale}]` : ''; + console.log(ansiColors.red(` • ${localeDisplay} ${type}: ${error}`)); + }); + } + + // Show auto-publish errors + if (autoPublishErrors.length > 0) { + console.log(ansiColors.red(`\n Auto-Publish Errors:`)); + autoPublishErrors.forEach(({ locale, type, error }) => { + const localeDisplay = locale ? `[${locale}]` : ''; + console.log(ansiColors.red(` • ${localeDisplay} ${type}: ${error}`)); + }); + } + } + + // Show log file paths at the very end + if (logFilePaths.length > 0) { + console.log(ansiColors.cyan('\n📄 Log Files:')); + logFilePaths.forEach((path) => { + console.log(` ${path}`); + }); } // Only exit if not called from another operation @@ -134,18 +226,27 @@ export class Push { /** * Execute auto-publish after sync completes + * IMPORTANT: Publishes items per-locale since the batch workflow API requires a locale parameter + * Returns array of errors for display in final summary */ - private async executeAutoPublish(results: PushResults[], autoPublishMode: string): Promise { - // Collect all publishable IDs from sync results - const allContentIds: number[] = []; - const allPageIds: number[] = []; + private async executeAutoPublish(results: PushResults[], autoPublishMode: string): Promise> { + // Collect per-locale publishable IDs from sync results + const contentIdsByLocale = new Map(); + const pageIdsByLocale = new Map(); for (const result of results) { - if (result.publishableContentIds && result.publishableContentIds.length > 0) { - allContentIds.push(...result.publishableContentIds); + // Use per-locale tracking if available + if (result.publishableContentIdsByLocale) { + result.publishableContentIdsByLocale.forEach((ids, locale) => { + const existing = contentIdsByLocale.get(locale) || []; + contentIdsByLocale.set(locale, [...existing, ...ids]); + }); } - if (result.publishablePageIds && result.publishablePageIds.length > 0) { - allPageIds.push(...result.publishablePageIds); + if (result.publishablePageIdsByLocale) { + result.publishablePageIdsByLocale.forEach((ids, locale) => { + const existing = pageIdsByLocale.get(locale) || []; + pageIdsByLocale.set(locale, [...existing, ...ids]); + }); } } @@ -153,38 +254,171 @@ export class Push { const publishContent = autoPublishMode === 'content' || autoPublishMode === 'both'; const publishPages = autoPublishMode === 'pages' || autoPublishMode === 'both'; - const contentIdsToPublish = publishContent ? allContentIds : []; - const pageIdsToPublish = publishPages ? allPageIds : []; + // Get all locales that have items to publish + const allLocales = new Set(); + if (publishContent) { + Array.from(contentIdsByLocale.keys()).forEach(locale => allLocales.add(locale)); + } + if (publishPages) { + Array.from(pageIdsByLocale.keys()).forEach(locale => allLocales.add(locale)); + } - // Check if there's anything to publish - if (contentIdsToPublish.length === 0 && pageIdsToPublish.length === 0) { - console.log(ansiColors.yellow('\n⚠️ Auto-publish: No items to publish from sync operation')); - return; + if (allLocales.size === 0) { + console.log(ansiColors.yellow('\n⚠️ Auto-publish: No items to publish from sync operation')); + return []; } + // Calculate totals for summary + let totalContent = 0; + let totalPages = 0; + contentIdsByLocale.forEach((ids) => { if (publishContent) totalContent += new Set(ids).size; }); + pageIdsByLocale.forEach((ids) => { if (publishPages) totalPages += new Set(ids).size; }); + console.log(ansiColors.cyan('\n' + '═'.repeat(50))); console.log(ansiColors.cyan('🚀 AUTO-PUBLISH')); console.log(ansiColors.cyan('═'.repeat(50))); console.log(ansiColors.gray(`Mode: ${autoPublishMode}`)); - console.log(ansiColors.gray(`Content items to publish: ${contentIdsToPublish.length}`)); - console.log(ansiColors.gray(`Pages to publish: ${pageIdsToPublish.length}`)); + console.log(ansiColors.gray(`Locales to publish: ${Array.from(allLocales).join(', ')}`)); + console.log(ansiColors.gray(`Total content items: ${totalContent}`)); + console.log(ansiColors.gray(`Total pages: ${totalPages}`)); + + // Import workflow dependencies + const { processBatches } = await import('../lib/workflows/process-batches'); + const { WorkflowOperationType } = await import('../types'); + const { updateMappingsAfterPublish } = await import('../lib/mappers/mapping-version-updater'); + const { waitForFetchApiSync } = await import('../lib/shared/get-fetch-api-status'); + + // Track all errors for summary + const allErrors: { locale: string; type: string; error: string }[] = []; + // Track all published IDs for mappings refresh + const publishedContentIdsByLocale = new Map(); + const publishedPageIdsByLocale = new Map(); try { - // Set explicit IDs in state for the workflow operation - setState({ - explicitContentIDs: contentIdsToPublish, - explicitPageIDs: pageIdsToPublish, - operationType: 'publish' - }); + // Process each locale separately - this is CRITICAL because the batch workflow API + // requires a locale parameter to publish the correct locale-specific content + for (const locale of Array.from(allLocales)) { + const contentIds = publishContent ? Array.from(new Set(contentIdsByLocale.get(locale) || [])) : []; + const pageIds = publishPages ? Array.from(new Set(pageIdsByLocale.get(locale) || [])) : []; + + if (contentIds.length === 0 && pageIds.length === 0) { + continue; + } + + console.log(ansiColors.cyan(`\n─── Publishing ${locale} ───`)); + console.log(ansiColors.gray(` Content: ${contentIds.length} items, Pages: ${pageIds.length} items`)); + + const errors: string[] = []; + const publishedContentIds: number[] = []; + const publishedPageIds: number[] = []; + + // Publish content for this locale + if (contentIds.length > 0) { + const contentResult = await processBatches(contentIds, 'content', locale, WorkflowOperationType.Publish, errors); + publishedContentIds.push(...contentResult.processedIds); + if (contentResult.failed > 0) { + console.log(ansiColors.yellow(` ⚠️ ${contentResult.failed} content items failed`)); + } + } + + // Publish pages for this locale + if (pageIds.length > 0) { + const pageResult = await processBatches(pageIds, 'pages', locale, WorkflowOperationType.Publish, errors); + publishedPageIds.push(...pageResult.processedIds); + if (pageResult.failed > 0) { + console.log(ansiColors.yellow(` ⚠️ ${pageResult.failed} pages failed`)); + } + } + + // Track published IDs for mappings refresh + if (publishedContentIds.length > 0) { + publishedContentIdsByLocale.set(locale, publishedContentIds); + } + if (publishedPageIds.length > 0) { + publishedPageIdsByLocale.set(locale, publishedPageIds); + } + + // Collect errors for summary + errors.forEach(err => { + allErrors.push({ locale, type: 'publish', error: err }); + }); + } - // Import and execute workflow operation - const { WorkflowOperation } = await import('../lib/workflows'); - const workflowOp = new WorkflowOperation(); - await workflowOp.executeFromMappings(); + console.log(ansiColors.green('\n✓ Auto-publish complete')); + + // Refresh target instance data and update mappings after publishing + // This ensures the mappings are up-to-date with the newly published content + const targetGuid = state.targetGuid?.[0]; + const sourceGuid = state.sourceGuid?.[0]; + + if (targetGuid && sourceGuid) { + const hasPublishedItems = publishedContentIdsByLocale.size > 0 || publishedPageIdsByLocale.size > 0; + + if (hasPublishedItems) { + // Step 1: Wait for Fetch API sync to complete (ONCE for all locales) + console.log(ansiColors.cyan('\nRefreshing target instance data...')); + try { + await waitForFetchApiSync(targetGuid, 'fetch', false); + } catch (error: any) { + console.log(ansiColors.yellow(` ⚠️ Could not check Fetch API status: ${error.message}`)); + // Continue anyway - status check is best-effort + } + + // Step 2: Do ONE pull to refresh target data (ONCE for all locales) + const pull = new Pull(); + const pullResult = await pull.pullInstances(true); + + if (!pullResult.success) { + console.log(ansiColors.yellow(' ⚠️ Target refresh failed - skipping mapping version updates')); + console.log(ansiColors.gray(' Run a manual pull to refresh data and update mappings')); + } else { + console.log(ansiColors.green('✓ Target instance data refreshed')); + + // Step 3: Update mappings for each locale that had published items + for (const locale of Array.from(allLocales)) { + const contentIds = publishedContentIdsByLocale.get(locale) || []; + const pageIds = publishedPageIdsByLocale.get(locale) || []; + + if (contentIds.length > 0 || pageIds.length > 0) { + try { + const mappingResult = await updateMappingsAfterPublish( + contentIds, + pageIds, + sourceGuid, + targetGuid, + locale + ); + + // Log any mapping errors + if (mappingResult.result.errors.length > 0) { + mappingResult.result.errors.forEach(err => { + allErrors.push({ + locale, + type: 'mapping', + error: err + }); + }); + } + } catch (refreshError: any) { + allErrors.push({ + locale, + type: 'refresh', + error: `Mappings update failed: ${refreshError.message}` + }); + } + } + } + } + } + } } catch (error: any) { console.error(ansiColors.red(`\n❌ Auto-publish failed: ${error.message}`)); + allErrors.push({ locale: 'all', type: 'fatal', error: error.message }); } + + // Return errors for display in final summary + return allErrors; } private async handleResetFlag(guid: string): Promise { diff --git a/src/core/state.ts b/src/core/state.ts index 8adb934..ce2786a 100644 --- a/src/core/state.ts +++ b/src/core/state.ts @@ -94,6 +94,10 @@ export interface State { isPush: boolean; isPull: boolean; isSync: boolean; + + // Failed content registry - tracks content items that failed during sync + // Used by page pusher to provide better error messages when content mappings are missing + failedContentRegistry: Map; } // Global state - populated from argv and referenced throughout the app @@ -163,6 +167,9 @@ export const state: State = { isPush: false, isPull: false, isSync: false, + + // Failed content registry - tracks content items that failed during sync + failedContentRegistry: new Map(), }; /** @@ -243,6 +250,7 @@ export function setState(argv: any) { // Workflow operation control if (argv.operationType !== undefined) state.operationType = argv.operationType; if (argv.dryRun !== undefined) state.dryRun = argv.dryRun; + if (argv.autoPublish !== undefined) state.autoPublish = argv.autoPublish; // Explicit ID overrides - parse comma-separated strings into number arrays if (argv.contentIDs !== undefined && argv.contentIDs !== "") { @@ -257,6 +265,14 @@ export function setState(argv: any) { .map((id: string) => parseInt(id.trim(), 10)) .filter((id: number) => !isNaN(id) && id > 0); } + + // Direct array assignment for programmatic use (e.g., auto-publish) + if (argv.explicitContentIDs !== undefined && Array.isArray(argv.explicitContentIDs)) { + state.explicitContentIDs = argv.explicitContentIDs; + } + if (argv.explicitPageIDs !== undefined && Array.isArray(argv.explicitPageIDs)) { + state.explicitPageIDs = argv.explicitPageIDs; + } // Model-specific if (argv.models !== undefined) state.models = argv.models; @@ -287,34 +303,37 @@ export function primeFromEnv(): { hasEnvFile: boolean; primedValues: string[] } const envFiles = ['.env', '.env.local', '.env.development', '.env.production']; const primedValues: string[] = []; + // Match KEY=value only on uncommented lines (line start or after newline, optional whitespace, no #) + const uncommentedLine = (key: string) => new RegExp(`(?:^|\\n)\\s*${key}=([^\\n]+)`, 'm'); + for (const envFile of envFiles) { const envPath = path.join(process.cwd(), envFile); if (fs.existsSync(envPath)) { const envContent = fs.readFileSync(envPath, 'utf8'); - // Parse all relevant environment variables + // Parse all relevant environment variables (uncommented lines only) const envVars = { - AGILITY_GUID: envContent.match(/AGILITY_GUID=([^\n]+)/), - AGILITY_TARGET_GUID: envContent.match(/AGILITY_TARGET_GUID=([^\n]+)/), - AGILITY_WEBSITE: envContent.match(/AGILITY_WEBSITE=([^\n]+)/), - AGILITY_LOCALES: envContent.match(/AGILITY_LOCALES=([^\n]+)/), - AGILITY_TEST: envContent.match(/AGILITY_TEST=([^\n]+)/), - AGILITY_OVERWRITE: envContent.match(/AGILITY_OVERWRITE=([^\n]+)/), - - AGILITY_PREVIEW: envContent.match(/AGILITY_PREVIEW=([^\n]+)/), - AGILITY_VERBOSE: envContent.match(/AGILITY_VERBOSE=([^\n]+)/), - AGILITY_HEADLESS: envContent.match(/AGILITY_HEADLESS=([^\n]+)/), - AGILITY_ELEMENTS: envContent.match(/AGILITY_ELEMENTS=([^\n]+)/), - AGILITY_ROOT_PATH: envContent.match(/AGILITY_ROOT_PATH=([^\n]+)/), - AGILITY_BASE_URL: envContent.match(/AGILITY_BASE_URL=([^\n]+)/), - AGILITY_DEV: envContent.match(/AGILITY_DEV=([^\n]+)/), - AGILITY_LOCAL: envContent.match(/AGILITY_LOCAL=([^\n]+)/), - AGILITY_PREPROD: envContent.match(/AGILITY_PREPROD=([^\n]+)/), - AGILITY_LEGACY_FOLDERS: envContent.match(/AGILITY_LEGACY_FOLDERS=([^\n]+)/), - AGILITY_INSECURE: envContent.match(/AGILITY_INSECURE=([^\n]+)/), - - AGILITY_MODELS: envContent.match(/AGILITY_MODELS=([^\n]+)/), - AGILITY_TOKEN: envContent.match(/AGILITY_TOKEN=([^\n]+)/), + AGILITY_GUID: envContent.match(uncommentedLine('AGILITY_GUID')), + AGILITY_TARGET_GUID: envContent.match(uncommentedLine('AGILITY_TARGET_GUID')), + AGILITY_WEBSITE: envContent.match(uncommentedLine('AGILITY_WEBSITE')), + AGILITY_LOCALES: envContent.match(uncommentedLine('AGILITY_LOCALES')), + AGILITY_TEST: envContent.match(uncommentedLine('AGILITY_TEST')), + AGILITY_OVERWRITE: envContent.match(uncommentedLine('AGILITY_OVERWRITE')), + + AGILITY_PREVIEW: envContent.match(uncommentedLine('AGILITY_PREVIEW')), + AGILITY_VERBOSE: envContent.match(uncommentedLine('AGILITY_VERBOSE')), + AGILITY_HEADLESS: envContent.match(uncommentedLine('AGILITY_HEADLESS')), + AGILITY_ELEMENTS: envContent.match(uncommentedLine('AGILITY_ELEMENTS')), + AGILITY_ROOT_PATH: envContent.match(uncommentedLine('AGILITY_ROOT_PATH')), + AGILITY_BASE_URL: envContent.match(uncommentedLine('AGILITY_BASE_URL')), + AGILITY_DEV: envContent.match(uncommentedLine('AGILITY_DEV')), + AGILITY_LOCAL: envContent.match(uncommentedLine('AGILITY_LOCAL')), + AGILITY_PREPROD: envContent.match(uncommentedLine('AGILITY_PREPROD')), + AGILITY_LEGACY_FOLDERS: envContent.match(uncommentedLine('AGILITY_LEGACY_FOLDERS')), + AGILITY_INSECURE: envContent.match(uncommentedLine('AGILITY_INSECURE')), + + AGILITY_MODELS: envContent.match(uncommentedLine('AGILITY_MODELS')), + AGILITY_TOKEN: envContent.match(uncommentedLine('AGILITY_TOKEN')), }; // Only prime state values that aren't already set from command line @@ -413,15 +432,16 @@ export function primeFromEnv(): { hasEnvFile: boolean; primedValues: string[] } if (envVars.AGILITY_TOKEN && envVars.AGILITY_TOKEN[1] && !state.token) { // Strip quotes from token value if present let tokenValue = envVars.AGILITY_TOKEN[1].trim(); - if ((tokenValue.startsWith('"') && tokenValue.endsWith('"')) || + if ((tokenValue.startsWith('"') && tokenValue.endsWith('"')) || (tokenValue.startsWith("'") && tokenValue.endsWith("'"))) { - tokenValue = tokenValue.slice(1, -1); + tokenValue = tokenValue.slice(1, -1).trim(); + } + // Only prime token when we actually have a non-empty value + if (tokenValue.length > 0) { + state.token = tokenValue; + process.env.AGILITY_TOKEN = tokenValue; + primedValues.push('token'); } - - state.token = tokenValue; - // Also set in process.env so getUserProvidedToken() can find it - process.env.AGILITY_TOKEN = tokenValue; - primedValues.push('token'); } if (primedValues.length > 0) { @@ -535,6 +555,10 @@ export function getApiClient(): mgmtApi.ApiClient { } else if (!state.mgmtApiOptions && state.token) { state.mgmtApiOptions = new Options(); state.mgmtApiOptions.token = state.token; + // Ensure baseUrl is set for local/dev/preprod modes + if (state.baseUrl) { + state.mgmtApiOptions.baseUrl = state.baseUrl; + } } // Create and cache the client state.cachedApiClient = new mgmtApi.ApiClient(state.mgmtApiOptions); @@ -747,3 +771,107 @@ export function endTimer(): void { export function clearLogger(): void { state.logger = undefined; } + +// ============================================================================ +// Failed Content Registry +// Tracks content items that failed during sync so page pusher can provide +// better error messages when content mappings are missing +// ============================================================================ + +/** + * Register a failed content item + * @param contentID - The source content ID that failed + * @param referenceName - The reference name of the content item + * @param error - The error message + * @param locale - The locale being processed + */ +export function registerFailedContent( + contentID: number, + referenceName: string, + error: string, + locale: string +): void { + state.failedContentRegistry.set(contentID, { referenceName, error, locale }); +} + +/** + * Look up a failed content item by its source content ID + * @param contentID - The source content ID to look up + * @returns The failure info if found, or undefined + */ +export function getFailedContent(contentID: number): { referenceName: string; error: string; locale: string } | undefined { + return state.failedContentRegistry.get(contentID); +} + +/** + * Clear the failed content registry (should be called at start of each sync) + */ +export function clearFailedContentRegistry(): void { + state.failedContentRegistry.clear(); +} + +/** + * Get the CMS app URL based on environment + * - dev/local/preprod: app-qa.publishwithagility.com + * - prod: app.agilitycms.com + */ +export function getCmsAppUrl(): string { + if (state.dev || state.local || state.preprod) { + return 'https://app-qa.publishwithagility.com'; + } + return 'https://app.agilitycms.com'; +} + +/** + * Generate a link to a page in the CMS app + */ +export function getPageCmsLink(guid: string, locale: string, pageID: number): string { + return `${getCmsAppUrl()}/instance/${guid}/${locale}/pages/page-${pageID}`; +} + +/** + * Generate a link to a content item in the CMS app + * URL format: /content/item-{containerID}/listitem-{contentID} + * The containerID can be arbitrary (using 0) - only the listitem ID matters for navigation + */ +export function getContentCmsLink(guid: string, locale: string, contentID: number): string { + return `${getCmsAppUrl()}/instance/${guid}/${locale}/content/item-0/listitem-${contentID}`; +} + +/** + * Check if a content item file exists in source data + */ +export function contentExistsInSourceData(guid: string, locale: string, contentID: number): boolean { + const fs = require('fs'); + const path = require('path'); + const contentPath = path.join(process.cwd(), state.rootPath, guid, locale, 'item', `${contentID}.json`); + return fs.existsSync(contentPath); +} + +/** + * Check if a content item exists in another locale but not the current one + * Returns the locale where it exists, or null if not found anywhere + */ +export function contentExistsInOtherLocale(guid: string, currentLocale: string, contentID: number): string | null { + const fs = require('fs'); + const path = require('path'); + + // Get the guid folder and find all locale folders + const guidPath = path.join(process.cwd(), state.rootPath, guid); + if (!fs.existsSync(guidPath)) return null; + + const entries = fs.readdirSync(guidPath, { withFileTypes: true }); + const localeFolders = entries + .filter((entry: any) => entry.isDirectory() && entry.name !== 'models' && entry.name !== currentLocale) + .map((entry: any) => entry.name); + + // Check each locale for the content item + for (const locale of localeFolders) { + const contentPath = path.join(guidPath, locale, 'item', `${contentID}.json`); + if (fs.existsSync(contentPath)) { + return locale; + } + } + + return null; +} diff --git a/src/lib/downloaders/download-sync-sdk.ts b/src/lib/downloaders/download-sync-sdk.ts index 4ee977f..a0cf3f4 100644 --- a/src/lib/downloaders/download-sync-sdk.ts +++ b/src/lib/downloaders/download-sync-sdk.ts @@ -5,7 +5,6 @@ import { state, getApiKeysForGuid, getLoggerForGuid } from "core/state"; import { fileOperations } from "core/fileOperations"; import { handleSyncToken } from "./sync-token-handler"; import { getAllChannels } from "lib/shared/get-all-channels"; -import ansiColors from "ansi-colors"; import { Auth } from "core/auth"; const storeInterfaceFileSystem = require("./store-interface-filesystem"); @@ -48,15 +47,21 @@ export async function downloadSyncSDKByLocaleAndChannel( const logger = getLoggerForGuid(guid); // Configure the Agility Sync client + // NOTE: Use determineFetchUrl (not determineBaseUrl) because: + // - Management API uses localhost:5050 when --local is set + // - Content Fetch/Sync API is always cloud-based (based on GUID suffix) + // The baseUrl must include the GUID path segment for the SDK to construct correct URLs: + // e.g., https://api-dev.aglty.io/{guid} → https://api-dev.aglty.io/{guid}/preview/{locale}/sync/items const auth = new Auth(); - const baseUrl = auth.determineBaseUrl(guid); + const fetchUrl = auth.determineFetchUrl(guid); + const baseUrlWithGuid = `${fetchUrl}/${guid}`; const agilityConfig = { guid: guid, apiKey: apiKey, isPreview: true, languages: [locale], channels: [channel], - baseUrl: baseUrl.replace('mgmt','api'), + baseUrl: baseUrlWithGuid, store: { interface: storeInterfaceFileSystem, options: { diff --git a/src/lib/downloaders/orchestrate-downloaders.ts b/src/lib/downloaders/orchestrate-downloaders.ts index 57126d7..82cec26 100644 --- a/src/lib/downloaders/orchestrate-downloaders.ts +++ b/src/lib/downloaders/orchestrate-downloaders.ts @@ -88,7 +88,8 @@ export class Downloader { } /** - * Orchestrate multiple GUIDs concurrently (DEFAULT METHOD) + * Orchestrate multiple GUIDs (DEFAULT METHOD) + * Uses sequential mode when --local flag is set to prevent overwhelming local API */ async instanceOrchestrator(fromPush: boolean): Promise { const state = getState(); @@ -97,8 +98,26 @@ export class Downloader { if (allGuids.length === 0) { throw new Error('No GUIDs available for download operation'); } + + // Use sequential mode when running against local API to prevent crashes + // Local debugging sessions can't handle as many concurrent requests + if (state.local) { + console.log(ansiColors.gray('Using sequential download mode for local API...')); + const successfulResults: DownloadResults[] = []; + + for (const guid of allGuids) { + try { + const result = await this.guidDownloader(guid, fromPush); + successfulResults.push(result); + } catch (error: any) { + console.error(`Failed download: ${guid} - ${error?.message || 'Unknown error'}`); + } + } + + return successfulResults; + } - // Start ALL downloads simultaneously (true parallel execution) + // Start ALL downloads simultaneously (true parallel execution) for cloud APIs const downloadTasks = allGuids.map(guid => this.guidDownloader(guid, fromPush)); const results = await Promise.allSettled(downloadTasks); diff --git a/src/lib/downloaders/store-interface-filesystem.ts b/src/lib/downloaders/store-interface-filesystem.ts index 9b7e7c4..0242887 100644 --- a/src/lib/downloaders/store-interface-filesystem.ts +++ b/src/lib/downloaders/store-interface-filesystem.ts @@ -247,11 +247,8 @@ const saveItem = async ({ options, item, itemType, languageCode, itemID }) => { } let json = JSON.stringify(item); - // Add specific debug logs around file write - // console.log(`[Debug saveItem] About to write: ${itemType} (ID: ${itemID}) to ${absoluteFilePath}`); - fs.writeFileSync(absoluteFilePath, json); - // console.log(`[Debug saveItem] Write successful for: ${absoluteFilePath}`); - + fs.writeFileSync(absoluteFilePath, json); + // Use structured logging instead of basic console.log if (logger) { diff --git a/src/lib/models/model-dependency-tree-builder.ts b/src/lib/models/model-dependency-tree-builder.ts index eed598b..4c27b58 100644 --- a/src/lib/models/model-dependency-tree-builder.ts +++ b/src/lib/models/model-dependency-tree-builder.ts @@ -479,11 +479,7 @@ export class ModelDependencyTreeBuilder { } }); - // // Debug output - // if (contentItemsScanned > 0) { - // console.log(ansiColors.gray(` 🔍 [DEBUG] Scanned ${contentItemsScanned} content item(s), found ${totalUrlsFound} asset URL(s), ${tree.assets.size} unique asset(s) in tree`)); - // } - + // Also check pages for asset references if (this.sourceData.pages) { this.sourceData.pages.forEach(page => { diff --git a/src/lib/pushers/asset-pusher.ts b/src/lib/pushers/asset-pusher.ts index 0ed213c..800664a 100644 --- a/src/lib/pushers/asset-pusher.ts +++ b/src/lib/pushers/asset-pusher.ts @@ -9,6 +9,35 @@ import { fileOperations } from "../../core/fileOperations"; import path from "path"; import { GalleryMapper } from "lib/mappers/gallery-mapper"; +/** + * Extract meaningful error message from API errors + */ +function extractErrorMessage(error: any): string { + // Check for direct axios response data first (our direct axios calls) + if (error?.response?.data) { + const data = error.response.data; + if (typeof data === 'string') return data; + if (data.exceptionMessage) return data.exceptionMessage; // Agility API format + if (data.message) return data.message; + if (data.error) return data.error; + if (data.Message) return data.Message; + if (data.Error) return data.Error; + if (data.title) return data.title; + } + // Check for SDK-wrapped axios response data + if (error?.innerError?.response?.data) { + const data = error.innerError.response.data; + if (typeof data === 'string') return data; + if (data.exceptionMessage) return data.exceptionMessage; + if (data.message) return data.message; + if (data.error) return data.error; + if (data.Message) return data.Message; + if (data.Error) return data.Error; + } + // Fall back to error message + return error?.message || error?.innerError?.message || String(error); +} + export async function pushAssets( sourceData: mgmtApi.Media[], // TODO: Type these targetData: mgmtApi.Media[], // TODO: Type these @@ -65,10 +94,27 @@ export async function pushAssets( let folderPath = containerFolderPath === "." ? "/" : containerFolderPath; - // TODO: this is where we need to check if the asset is a gallery asset and if so, we need to check if the gallery is up to date // Use simplified change detection pattern const existingMapping = referenceMapper.getAssetMapping(media, "source"); - const shouldCreate = existingMapping === null; + + // Also check if asset already exists in target by originKey (path+filename) + const targetAssetByOriginKey = targetData.find(t => t.originKey === media.originKey); + + // Debug logging for asset matching (verbose mode) + if (state.verbose && !existingMapping && !targetAssetByOriginKey) { + // Show first few target originKeys for comparison + const sampleTargetKeys = targetData.slice(0, 3).map(t => t.originKey); + } + + // If no mapping but asset exists by originKey in target, create mapping and skip + if (!existingMapping && targetAssetByOriginKey) { + referenceMapper.addMapping(media, targetAssetByOriginKey); + logger.asset.skipped(media, "already exists in target by path", targetGuid[0]); + skipped++; + continue; + } + + const shouldCreate = existingMapping === null && !targetAssetByOriginKey; // get the target asset, check if the source and targets need updates const targetAsset: mgmtApi.Media = targetData.find(targetAsset => targetAsset.mediaID === existingMapping?.targetMediaID) || null; @@ -112,7 +158,10 @@ export async function pushAssets( skipped++; } } catch (error: any) { - logger.asset.error(media, error, targetGuid[0]); + const errorMsg = extractErrorMessage(error); + logger.asset.error(media, errorMsg, targetGuid[0]); + + failed++; currentStatus = "error"; overallStatus = "error"; @@ -133,6 +182,7 @@ export async function pushAssets( /** * Create a new asset in the target instance + * Note: We make direct axios call instead of SDK because SDK doesn't include form-data headers */ async function createAsset( media: mgmtApi.Media, @@ -147,16 +197,51 @@ async function createAsset( // Handle gallery if present let targetMediaGroupingID = await resolveGalleryMapping(media, apiClient, sourceGuid, targetGuid); - const fileOps = new fileOperations(targetGuid); - // Upload the asset - const form = new FormData(); - if (!fileOps.checkFileExists(absoluteLocalFilePath)) { - throw new Error(`Local asset file not found: ${absoluteLocalFilePath}`); + const fs = require('fs'); + const pathModule = require('path'); + + // Resolve to absolute path from workspace root + const resolvedPath = pathModule.resolve(process.cwd(), absoluteLocalFilePath); + + // Check file exists and has content + if (!fs.existsSync(resolvedPath)) { + throw new Error(`Local asset file not found: ${resolvedPath}`); + } + + const fileStats = fs.statSync(resolvedPath); + if (fileStats.size === 0) { + throw new Error(`Local asset file is empty (0 bytes): ${resolvedPath}`); } - const fileBuffer = fileOps.createReadStream(absoluteLocalFilePath); - form.append("files", fileBuffer, media.fileName); - const uploadedMediaArray = await apiClient.assetMethods.upload(form, folderPath, targetGuid, targetMediaGroupingID); + + // Build form data with file stream using resolved absolute path + const form = new FormData(); + const fileStream = fs.createReadStream(resolvedPath); + form.append("files", fileStream, media.fileName); + + // Make direct axios call with form-data headers (SDK bug workaround) + // The SDK's executePost doesn't include form.getHeaders() which is required for multipart uploads + const axios = require('axios'); + + // Get the base URL from the API client's options + const baseUrl = (apiClient as any)._options?.baseUrl || determineBaseUrl(targetGuid); + const token = (apiClient as any)._options?.token; + + const apiPath = `asset/upload?folderPath=${encodeURIComponent(folderPath)}&groupingID=${targetMediaGroupingID}`; + const url = `${baseUrl}/api/v1/instance/${targetGuid}/${apiPath}`; + + const response = await axios.post(url, form, { + headers: { + ...form.getHeaders(), // Critical: include multipart boundary + 'Authorization': `Bearer ${token}`, + 'Cache-Control': 'no-cache' + }, + maxContentLength: Infinity, + maxBodyLength: Infinity, + httpsAgent: state.local ? new (require('https').Agent)({ rejectUnauthorized: false }) : undefined + }); + + const uploadedMediaArray = response.data as mgmtApi.Media[]; if (!uploadedMediaArray || uploadedMediaArray.length === 0) { throw new Error(`API did not return uploaded media details for ${media.fileName}`); @@ -169,8 +254,23 @@ async function createAsset( return uploadedMedia; } +/** + * Determine base URL for a GUID (fallback if not available from SDK) + */ +function determineBaseUrl(guid: string): string { + const separator = guid.split('-'); + if (separator[1] === 'd') return "https://mgmt-dev.aglty.io"; + if (separator[1] === 'u') return "https://mgmt.aglty.io"; + if (separator[1] === 'us2') return "https://mgmt-usa2.aglty.io"; + if (separator[1] === 'c') return "https://mgmt-ca.aglty.io"; + if (separator[1] === 'e') return "https://mgmt-eu.aglty.io"; + if (separator[1] === 'a') return "https://mgmt-aus.aglty.io"; + return "https://mgmt.aglty.io"; +} + /** * Update an existing asset in the target instance + * Note: We make direct axios call instead of SDK because SDK doesn't include form-data headers */ async function updateAsset( media: mgmtApi.Media, @@ -183,27 +283,59 @@ async function updateAsset( logger: Logs ): Promise { // Handle gallery if present - let targetMediaGroupingID = await resolveGalleryMapping(media, apiClient, sourceGuid, targetGuid); - const fileOps = new fileOperations(targetGuid); - // Upload the asset (this will replace the existing one) - const form = new FormData(); - if (!fileOps.checkFileExists(absoluteLocalFilePath)) { - throw new Error(`Local asset file not found: ${absoluteLocalFilePath}`); + + const fs = require('fs'); + const pathModule = require('path'); + + // Resolve to absolute path from workspace root + const resolvedPath = pathModule.resolve(process.cwd(), absoluteLocalFilePath); + + // Check file exists and has content + if (!fs.existsSync(resolvedPath)) { + throw new Error(`Local asset file not found: ${resolvedPath}`); } - const fileBuffer = fileOps.createReadStream(absoluteLocalFilePath); - form.append("files", fileBuffer, media.fileName); - - const uploadedMediaArray = await apiClient.assetMethods.upload(form, folderPath, targetGuid, targetMediaGroupingID); + + const fileStats = fs.statSync(resolvedPath); + if (fileStats.size === 0) { + throw new Error(`Local asset file is empty (0 bytes): ${resolvedPath}`); + } + + // Build form data with file stream using resolved absolute path + const form = new FormData(); + const fileStream = fs.createReadStream(resolvedPath); + form.append("files", fileStream, media.fileName); + + // Make direct axios call with form-data headers (SDK bug workaround) + const axios = require('axios'); + + const baseUrl = (apiClient as any)._options?.baseUrl || determineBaseUrl(targetGuid); + const token = (apiClient as any)._options?.token; + + const apiPath = `asset/upload?folderPath=${encodeURIComponent(folderPath)}&groupingID=${targetMediaGroupingID}`; + const url = `${baseUrl}/api/v1/instance/${targetGuid}/${apiPath}`; + + const response = await axios.post(url, form, { + headers: { + ...form.getHeaders(), + 'Authorization': `Bearer ${token}`, + 'Cache-Control': 'no-cache' + }, + maxContentLength: Infinity, + maxBodyLength: Infinity, + httpsAgent: state.local ? new (require('https').Agent)({ rejectUnauthorized: false }) : undefined + }); + + const uploadedMediaArray = response.data as mgmtApi.Media[]; if (!uploadedMediaArray || uploadedMediaArray.length === 0) { throw new Error(`API did not return uploaded media details for ${media.fileName}`); } + const uploadedMedia = uploadedMediaArray[0]; logger.asset.uploaded(media, "uploaded", targetGuid); - return uploadedMedia; } diff --git a/src/lib/pushers/batch-polling.ts b/src/lib/pushers/batch-polling.ts index 9cc71e9..89b4fd4 100644 --- a/src/lib/pushers/batch-polling.ts +++ b/src/lib/pushers/batch-polling.ts @@ -1,6 +1,122 @@ import * as mgmtApi from '@agility/management-sdk'; import ansiColors from 'ansi-colors'; +/** + * Extract the error message from a JSON error response or plain text + * Handles both JSON format {"message":"..."} and plain text errors + */ +function extractErrorMessage(errorText: string): string { + if (!errorText) return 'Unknown error'; + + try { + // Try to parse as JSON + const parsed = JSON.parse(errorText); + // Return just the message field if it exists + if (parsed.message) { + return parsed.message; + } + // Fallback to exceptionType if no message + if (parsed.exceptionType) { + return `${parsed.exceptionType}: ${parsed.message || 'No details'}`; + } + return errorText; + } catch { + // Not JSON, return as-is but truncate if too long + // Also extract message from exception format: "ExceptionType: Message" + const exceptionMatch = errorText.match(/^[\w.]+Exception:\s*(.+?)(?:\r?\n|$)/); + if (exceptionMatch) { + return exceptionMatch[1].trim(); + } + return errorText.length > 200 ? errorText.substring(0, 200) + '...' : errorText; + } +} + +/** + * Create a simple progress bar string + */ +function createProgressBar(percent: number, width: number = 20): string { + const filled = Math.round((percent / 100) * width); + const empty = width - filled; + return `[${'█'.repeat(filled)}${'░'.repeat(empty)}]`; +} + +/** + * Log batch errors using structured failedItems array (preferred) or legacy items array + */ +function logBatchErrors(batchStatus: any, originalPayloads?: any[]): void { + // Prefer structured failedItems array from new API + if (Array.isArray(batchStatus.failedItems) && batchStatus.failedItems.length > 0) { + batchStatus.failedItems.forEach((failed: any) => { + const batchItemId = failed.batchItemId ?? failed.batchItemID ?? '?'; + const errorType = failed.errorType || 'Error'; + const errorMessage = failed.errorMessage || 'Unknown error'; + const itemType = failed.itemType || 'Item'; + + // Try to find the original payload by batchItemId to get referenceName + let referenceName = 'unknown'; + if (originalPayloads) { + // batchItemId typically corresponds to index in the batch + const payload = originalPayloads.find((p, idx) => p?.batchItemId === batchItemId) + || originalPayloads[batchStatus.failedItems.indexOf(failed)]; + referenceName = payload?.properties?.referenceName || payload?.referenceName || 'unknown'; + } + + console.error(ansiColors.red(` ✗ ${itemType} ${batchItemId} (${referenceName}): ${errorMessage}`)); + }); + return; + } + + // Fallback to legacy items array with errorMessage field + if (Array.isArray(batchStatus.items)) { + batchStatus.items.forEach((item: any, index: number) => { + if (item.errorMessage) { + const errorMessage = extractErrorMessage(item.errorMessage); + const batchItemId = item.batchItemID || item.batchItemId || `idx:${index}`; + const referenceName = originalPayloads?.[index]?.properties?.referenceName || 'unknown'; + + console.error(ansiColors.red(` ✗ Batch item ${batchItemId} (${referenceName}): ${errorMessage}`)); + } + }); + } +} + +/** BatchState.Processed = 3 */ +const BATCH_STATE_PROCESSED = 3; + +/** + * Normalize batch response so we handle both camelCase (SDK) and PascalCase (.NET API). + */ +function normalizeBatchStatus(batchStatus: any): any { + if (!batchStatus) return batchStatus; + return { + ...batchStatus, + batchState: batchStatus.batchState ?? batchStatus.BatchState, + numItemsProcessed: batchStatus.numItemsProcessed ?? batchStatus.NumItemsProcessed, + percentComplete: batchStatus.percentComplete ?? batchStatus.PercentComplete, + items: batchStatus.items ?? batchStatus.Items ?? [], + errorData: batchStatus.errorData ?? batchStatus.ErrorData, + }; +} + +/** + * Derive number of processed items from batch status. + * Uses numItemsProcessed when set by the API; otherwise counts from items array + * (items with itemID > 0 or processedItemVersionID set are considered processed). + */ +function getNumProcessed(batchStatus: any): number { + const num = batchStatus?.numItemsProcessed ?? batchStatus?.NumItemsProcessed; + if (typeof num === 'number' && num >= 0) { + return num; + } + const items = batchStatus?.items ?? batchStatus?.Items; + if (!Array.isArray(items)) return 0; + return items.filter((item: any) => { + const id = item?.itemID ?? item?.itemId; + const versionId = item?.processedItemVersionID ?? item?.processedItemVersionId; + return (typeof id === 'number' && id > 0) || (versionId != null && versionId !== ''); + }).length; +} + /** * Simple batch polling function - polls until batch status is 3 (complete) */ @@ -11,71 +127,68 @@ export async function pollBatchUntilComplete( originalPayloads?: any[], // Original payloads for error matching maxAttempts: number = 300, // 10 minutes at 2s intervals - increased from 120 intervalMs: number = 2000, // 2 seconds - batchType?: string // Type of batch for better logging + batchType?: string, // Type of batch for better logging + totalItems?: number // Total items in batch for progress display ): Promise { let attempts = 0; let consecutiveErrors = 0; + const startTime = Date.now(); + // Default totalItems from originalPayloads if not provided + const itemCount = totalItems ?? originalPayloads?.length ?? 0; - // console.log(`🔄 Polling batch ${batchID} until complete (max ${maxAttempts} attempts, ~${Math.round(maxAttempts * intervalMs / 60000)} minutes)...`); + // Show progress starting at 0 immediately so the bar doesn't jump on first poll + const batchTypeStr = batchType ? `${batchType} batch` : 'Batch'; + const initialBar = createProgressBar(0); + const initialLine = + itemCount > 0 + ? `${batchTypeStr} ${batchID}: ${initialBar} 0/${itemCount} (0s)` + : `${batchTypeStr} ${batchID}: ${initialBar} 0% (0s)`; + process.stdout.write(ansiColors.yellow.dim(initialLine) + ' '); while (attempts < maxAttempts) { try { - // Use getBatch from management SDK - const batchStatus = await apiClient.batchMethods.getBatch(batchID, targetGuid); - + // Poll: get current batch status from API (expandItems=true by default in SDK) + const raw = await apiClient.batchMethods.getBatch(batchID, targetGuid); + const batchStatus = normalizeBatchStatus(raw); + // Reset consecutive errors on successful API call consecutiveErrors = 0; - + if (!batchStatus) { - // console.warn(`⚠️ No batch status returned for batch ${batchID} (attempt ${attempts + 1}/${maxAttempts})`); attempts++; await new Promise(resolve => setTimeout(resolve, intervalMs)); continue; } - - if (batchStatus.batchState === 3) { - // console.log(`✅ Batch ${batchID} completed successfully after ${attempts + 1} attempts`); - // check for batch item errors - if (Array.isArray(batchStatus.items)) { - batchStatus.items.forEach((item: any, index: number) => { - if(item.errorMessage) { - // show the error and the item separately - const itemClean = { ...item} - delete itemClean.errorMessage; - console.error(ansiColors.red(`⚠️ Item ${item.itemID} (index ${index}) failed with error: ${item.errorMessage}`)); - console.log(ansiColors.gray.italic('📋 Batch Item Details:')); - console.log(ansiColors.gray.italic(JSON.stringify(itemClean, null, 2))); - - // FIFO matching: Show the original payload that caused this error - if (originalPayloads && originalPayloads[index]) { - console.log(ansiColors.yellow.italic('🔍 Original Payload that Failed:')); - console.log(ansiColors.yellow.italic(JSON.stringify(originalPayloads[index], null, 2))); - } else if (originalPayloads) { - console.warn(ansiColors.yellow(`⚠️ Could not match payload at index ${index} (total payloads: ${originalPayloads.length})`)); - } - - if (batchStatus.errorData) { - console.log(ansiColors.red.italic('🔍 Additional Error Data:')); - console.log(batchStatus.errorData + "\n"); - } - } - }); - } + const state = batchStatus.batchState; + if (state === BATCH_STATE_PROCESSED) { + // Clear the in-place progress line before logging completion/errors + process.stdout.write('\r' + ' '.repeat(80) + '\r'); + // Batch complete - log errors using structured failedItems if available + logBatchErrors(batchStatus, originalPayloads); return batchStatus; - } else { + } - // Create a cycling dot pattern that resets every 3 attempts - let dots = '.'.repeat((attempts % 3) + 1); - - // Include batch type in logging if provided - const batchTypeStr = batchType ? `${batchType} batch` : 'Batch'; - console.log(ansiColors.yellow.dim(`${batchTypeStr} ${batchID} in progress ${dots}`)); - if (batchStatus.errorData) { - console.log(`Error: ${batchStatus.errorData}`); - } + // Still in progress: show progress from this poll's response + const batchTypeStr = batchType ? `${batchType} batch` : 'Batch'; + const numProcessed = getNumProcessed(batchStatus); + const elapsed = Math.round((Date.now() - startTime) / 1000); + const pct = batchStatus.percentComplete; + let line: string; + if (itemCount > 0) { + const percentComplete = Math.round((numProcessed / itemCount) * 100); + const progressBar = createProgressBar(percentComplete); + line = `${batchTypeStr} ${batchID}: ${progressBar} ${numProcessed}/${itemCount} (${elapsed}s)`; + } else { + const percentComplete = typeof pct === 'number' && pct >= 0 ? pct : 0; + const progressBar = createProgressBar(percentComplete); + line = `${batchTypeStr} ${batchID}: ${progressBar} ${percentComplete}% (${elapsed}s)`; } - + process.stdout.write('\r' + ansiColors.yellow.dim(line) + ' '); + if (batchStatus.errorData) { + console.log(`Error: ${batchStatus.errorData}`); + } + attempts++; await new Promise(resolve => setTimeout(resolve, intervalMs)); @@ -89,8 +202,9 @@ export async function pollBatchUntilComplete( // Try one more time with extended timeout before giving up try { - const finalCheck = await apiClient.batchMethods.getBatch(batchID, targetGuid); - if (finalCheck?.batchState === 3) { + const finalRaw = await apiClient.batchMethods.getBatch(batchID, targetGuid); + const finalCheck = normalizeBatchStatus(finalRaw); + if (finalCheck?.batchState === BATCH_STATE_PROCESSED) { console.log(`✅ Batch ${batchID} was actually successful! Polling errors were transient.`); return finalCheck; } @@ -101,6 +215,7 @@ export async function pollBatchUntilComplete( attempts++; if (attempts >= maxAttempts) { + process.stdout.write('\r' + ' '.repeat(80) + '\r\n'); throw new Error(`Failed to poll batch ${batchID} after ${maxAttempts} attempts (${consecutiveErrors} consecutive errors): ${error.message}`); } @@ -110,34 +225,88 @@ export async function pollBatchUntilComplete( } } + process.stdout.write('\r' + ' '.repeat(80) + '\r\n'); throw new Error(`Batch ${batchID} polling timed out after ${maxAttempts} attempts (~${Math.round(maxAttempts * intervalMs / 60000)} minutes)`); } /** * Extract results from completed batch + * Uses structured failedItems from new API if available, falls back to legacy items array */ -export function extractBatchResults(batch: any, originalItems: any[]): { successfulItems: any[], failedItems: any[] } { +export function extractBatchResults(batch: any, originalItems: any[]): { + successfulItems: any[], + failedItems: any[], + summary?: { totalItems: number, successCount: number, failureCount: number, durationMs: number } +} { const successfulItems: any[] = []; const failedItems: any[] = []; + + // Extract summary info if available from new API + const summary = batch?.totalItems !== undefined ? { + totalItems: batch.totalItems, + successCount: batch.successCount ?? 0, + failureCount: batch.failureCount ?? 0, + durationMs: batch.durationMs ?? 0 + } : undefined; + + // Use structured failedItems from new API if available + if (Array.isArray(batch?.failedItems) && batch.failedItems.length > 0) { + // Track which indices failed + const failedBatchItemIds = new Set(batch.failedItems.map((f: any) => f.batchItemId ?? f.batchItemID)); + + // Process failed items from structured array + batch.failedItems.forEach((failed: any, idx: number) => { + const batchItemId = failed.batchItemId ?? failed.batchItemID; + // Try to match to original item - batchItemId might be 1-indexed or match some property + const originalItem = originalItems[idx] || originalItems.find((item, i) => i === batchItemId - 1); + + failedItems.push({ + originalItem: originalItem || null, + newItem: null, + error: failed.errorMessage || 'Unknown error', + errorType: failed.errorType, + itemType: failed.itemType, + batchItemId: batchItemId, + index: idx + }); + }); + + // Remaining items are successful (from items array or inferred) + if (Array.isArray(batch?.items)) { + batch.items.forEach((item: any, index: number) => { + const batchItemId = item.batchItemID || item.batchItemId; + if (!failedBatchItemIds.has(batchItemId) && item.itemID > 0) { + successfulItems.push({ + originalItem: originalItems[index], + newId: item.itemID, + newItem: item, + index + }); + } + }); + } + + return { successfulItems, failedItems, summary }; + } + // Fallback to legacy items array processing if (!batch?.items || !Array.isArray(batch.items)) { - // All items failed if no items array return { successfulItems: [], failedItems: originalItems.map((item, index) => ({ originalItem: item, error: 'No batch items returned', index - })) + })), + summary }; } - // Process each batch item + // Process each batch item (legacy) batch.items.forEach((item: any, index: number) => { const originalItem = originalItems[index]; if (item.itemID > 0 && !item.itemNull) { - // Successful item successfulItems.push({ originalItem, newId: item.itemID, @@ -145,17 +314,23 @@ export function extractBatchResults(batch: any, originalItems: any[]): { success index }); } else { - // Failed item + let errorMsg = 'Failed to create item'; + if (item.errorMessage) { + errorMsg = extractErrorMessage(item.errorMessage); + } else if (!item.itemNull) { + errorMsg = `Invalid ID: ${item.itemID}`; + } + failedItems.push({ originalItem, newItem: null, - error: item.itemNull ? 'Item creation returned null' : `Invalid ID: ${item.itemID}`, + error: errorMsg, index }); } }); - return { successfulItems, failedItems }; + return { successfulItems, failedItems, summary }; } diff --git a/src/lib/pushers/content-pusher/content-batch-processor.ts b/src/lib/pushers/content-pusher/content-batch-processor.ts index 5f30090..13fd820 100644 --- a/src/lib/pushers/content-pusher/content-batch-processor.ts +++ b/src/lib/pushers/content-pusher/content-batch-processor.ts @@ -109,6 +109,18 @@ export class ContentBatchProcessor { const { successfulItems, failedItems } = extractBatchResults(completedBatch, contentBatch); // Convert to expected format + // Filter publishableIds to only include items that are Published (state === 2) in source + const publishableSuccessItems = successfulItems.filter((item) => { + const sourceState = item.originalItem?.properties?.state; + return sourceState === 2; // Only published items + }); + + // Log staging items being skipped from auto-publish + const stagingItems = successfulItems.filter((item) => item.originalItem?.properties?.state !== 2); + if (stagingItems.length > 0) { + console.log(ansiColors.gray(` 📋 Skipping auto-publish for ${stagingItems.length} content item(s) (not published in source)`)); + } + const batchResult = { successCount: successfulItems.length, failureCount: failedItems.length, @@ -122,7 +134,7 @@ export class ContentBatchProcessor { originalContent: item.originalItem, error: item.error, })), - publishableIds: successfulItems.map((item) => item.newId), + publishableIds: publishableSuccessItems.map((item) => item.newId), }; totalSuccessCount += batchResult.successCount; @@ -204,13 +216,19 @@ export class ContentBatchProcessor { // console.log(`🎯 Content batch processing complete: ${totalSuccessCount} success, ${totalFailureCount} failed`); + // Filter final publishableIds to only include items Published (state === 2) in source + const publishableItems = allSuccessfulItems.filter((item) => { + const sourceState = item.originalContent?.properties?.state; + return sourceState === 2; // Only published items + }); + return { successCount: totalSuccessCount, failureCount: totalFailureCount, skippedCount: totalSkippedCount, successfulItems: allSuccessfulItems, failedItems: allFailedItems, - publishableIds: allSuccessfulItems.map((item) => item.newContentId), + publishableIds: publishableItems.map((item) => item.newContentId), }; } diff --git a/src/lib/pushers/content-pusher/content-pusher.ts b/src/lib/pushers/content-pusher/content-pusher.ts index 68898f8..7b78a0c 100644 --- a/src/lib/pushers/content-pusher/content-pusher.ts +++ b/src/lib/pushers/content-pusher/content-pusher.ts @@ -2,7 +2,7 @@ // Removed finder imports - using mapper directly import ansiColors from "ansi-colors"; // Removed ContentBatchProcessor import - individual pusher only handles individual processing -import { getLoggerForGuid, state } from 'core/state'; +import { getLoggerForGuid, state, registerFailedContent } from 'core/state'; import { ContentItemMapper } from "lib/mappers/content-item-mapper"; import { filterContentItemsForProcessing } from './util/filter-content-items-for-processing'; import { getContentItemTypes } from './util/get-content-item-types'; @@ -35,7 +35,7 @@ export async function pushContent( const contentItems = sourceData || []; if (contentItems.length === 0) { - return { status: "success" as const, successful: 0, failed: 0, skipped: 0, publishableIds: [] }; + return { status: "success" as const, successful: 0, failed: 0, skipped: 0, publishableIds: [], failureDetails: [] }; } // Deterministically classify content items based on list references (fulllist=true) @@ -52,6 +52,7 @@ export async function pushContent( let totalFailed = 0; let totalSkipped = 0; const allPublishableIds: number[] = []; + const allFailureDetails: Array<{ name: string; error: string; type?: 'content' | 'page'; contentID?: number; guid?: string; locale?: string }> = []; try { // Import getApiClient for both batch configurations @@ -101,6 +102,25 @@ export async function pushContent( totalSkipped += filteredLinkedContentItems.skippedCount; totalSkipped += linkedResult.skippedCount; allPublishableIds.push(...linkedResult.publishableIds); + // Collect failure details for error summary and register in failed content registry + if (linkedResult.failedItems && linkedResult.failedItems.length > 0) { + linkedResult.failedItems.forEach(item => { + const name = item.originalContent?.properties?.referenceName || 'Unknown'; + const contentID = item.originalContent?.contentID; + allFailureDetails.push({ + name, + error: item.error, + type: 'content', + contentID, + guid: sourceGuidStr, + locale + }); + // Register in global registry so page pusher can provide better error messages + if (contentID) { + registerFailedContent(contentID, name, item.error, locale); + } + }); + } } // Process normal content items first (no dependencies) @@ -139,6 +159,25 @@ export async function pushContent( totalSkipped += filteredNormalContentItems.skippedCount; totalSkipped += normalResult.skippedCount; allPublishableIds.push(...normalResult.publishableIds); + // Collect failure details for error summary and register in failed content registry + if (normalResult.failedItems && normalResult.failedItems.length > 0) { + normalResult.failedItems.forEach(item => { + const name = item.originalContent?.properties?.referenceName || 'Unknown'; + const contentID = item.originalContent?.contentID; + allFailureDetails.push({ + name, + error: item.error, + type: 'content', + contentID, + guid: sourceGuidStr, + locale + }); + // Register in global registry so page pusher can provide better error messages + if (contentID) { + registerFailedContent(contentID, name, item.error, locale); + } + }); + } } // Convert batch result to expected PusherResult format @@ -148,9 +187,18 @@ export async function pushContent( failed: totalFailed, skipped: totalSkipped, publishableIds: allPublishableIds, + failureDetails: allFailureDetails, }; } catch (batchError: any) { console.error(ansiColors.red(`❌ Batch processing failed: ${batchError.message}`)); + return { + status: "error" as const, + successful: totalSuccessful, + failed: totalFailed + 1, + skipped: totalSkipped, + publishableIds: allPublishableIds, + failureDetails: [...allFailureDetails, { name: 'Batch processing', error: batchError.message }], + }; } } diff --git a/src/lib/pushers/content-pusher/util/change-detection.ts b/src/lib/pushers/content-pusher/util/change-detection.ts index ad1a711..c3615b9 100644 --- a/src/lib/pushers/content-pusher/util/change-detection.ts +++ b/src/lib/pushers/content-pusher/util/change-detection.ts @@ -22,7 +22,7 @@ export function changeDetection( ): ChangeDetection { // Validate source entity structure if (!sourceEntity || !sourceEntity.properties) { - console.error(`[ChangeDetection] Invalid source entity structure:`, sourceEntity); + // console.error(`[ChangeDetection] Invalid source entity structure:`, sourceEntity); return { entity: null, shouldUpdate: false, @@ -33,8 +33,13 @@ export function changeDetection( }; } + const itemName = sourceEntity.properties?.referenceName || `ID:${sourceEntity.contentID}`; + if (!mapping && !targetEntity) { //if we have no target content and no mapping + // if (state.verbose) { + // console.log(`[ChangeDetection] ${itemName}: No mapping and no target entity → CREATE`); + // } return { entity: null, shouldUpdate: false, @@ -52,9 +57,13 @@ export function changeDetection( const mappedSourceVersion = (mapping?.sourceVersionID || 0) as number; const mappedTargetVersion = (mapping?.targetVersionID || 0) as number; - if (sourceVersion > 0 && targetVersion > 0) - //both the source and the target exist + // Verbose logging for version comparison debugging + // if (state.verbose) { + // console.log(`[ChangeDetection] ${itemName}: sourceV=${sourceVersion} (mapped=${mappedSourceVersion}), targetV=${targetVersion} (mapped=${mappedTargetVersion})`); + // } + if (sourceVersion > 0 && targetVersion > 0) { + //both the source and the target exist if (sourceVersion > mappedSourceVersion && targetVersion > mappedTargetVersion) { //CONFLICT DETECTION @@ -67,6 +76,10 @@ export function changeDetection( const sourceUrl = `https://app.agilitycms.com/instance/${state.sourceGuid[0]}/${locale}/content/listitem-${sourceEntity.contentID}`; const targetUrl = `https://app.agilitycms.com/instance/${state.targetGuid[0]}/${locale}/content/listitem-${targetEntity.contentID}`; + // if (state.verbose) { + // console.log(`[ChangeDetection] ${itemName}: CONFLICT - both versions changed`); + // } + return { entity: targetEntity, shouldUpdate: false, @@ -75,13 +88,16 @@ export function changeDetection( isConflict: true, reason: `Both source and target versions have been updated. Please resolve manually.\n - source: ${sourceUrl} \n - target: ${targetUrl}.` }; - } + } if (sourceVersion > mappedSourceVersion && targetVersion <= mappedTargetVersion) { //SOURCE UPDATE ONLY // Source version is newer the mapped source version // and target version is NOT newer than mapped target version + // if (state.verbose) { + // console.log(`[ChangeDetection] ${itemName}: UPDATE - source version newer (${sourceVersion} > ${mappedSourceVersion})`); + // } return { entity: targetEntity, shouldUpdate: true, @@ -94,6 +110,9 @@ export function changeDetection( const { overwrite } = state; if (overwrite) { + // if (state.verbose) { + // console.log(`[ChangeDetection] ${itemName}: UPDATE - overwrite mode enabled`); + // } return { entity: targetEntity, shouldUpdate: true, @@ -104,6 +123,9 @@ export function changeDetection( }; } + // if (state.verbose) { + // console.log(`[ChangeDetection] ${itemName}: SKIP - up to date`); + // } return { entity: targetEntity, shouldUpdate: false, diff --git a/src/lib/pushers/content-pusher/util/filter-content-items-for-processing.ts b/src/lib/pushers/content-pusher/util/filter-content-items-for-processing.ts index 4e8bb6e..14355ee 100644 --- a/src/lib/pushers/content-pusher/util/filter-content-items-for-processing.ts +++ b/src/lib/pushers/content-pusher/util/filter-content-items-for-processing.ts @@ -37,6 +37,12 @@ export async function filterContentItemsForProcessing({ const itemsToProcess: any[] = []; const itemsToSkip: any[] = []; + // Track decision stats for summary logging + let createCount = 0; + let updateCount = 0; + let skipCount = 0; + let conflictCount = 0; + for (const contentItem of contentItems) { const itemName = contentItem.properties.referenceName || "Unknown"; @@ -48,20 +54,29 @@ export async function filterContentItemsForProcessing({ const { content, shouldUpdate, shouldCreate, shouldSkip, isConflict, reason } = findResult; if (isConflict) { - ///CONFLICT DETECTED - logger.content.error(contentItem, `!! Conflict detected for content ${itemName}: ${reason}`, locale, targetGuid); + // CONFLICT DETECTED - log warning and skip + console.warn( + `⚠️ Conflict detected content ${ansiColors.underline(itemName)} ${ansiColors.bold.grey("changes detected in both source and target")}. Please resolve manually.` + ); + if (reason) { + console.warn(` ${reason}`); + } itemsToSkip.push(contentItem); + conflictCount++; continue; } else if (shouldCreate) { // Content doesn't exist - include it for creation itemsToProcess.push(contentItem); + createCount++; } else if (shouldUpdate) { // Content exists but needs updating itemsToProcess.push(contentItem); + updateCount++; } else if (shouldSkip) { // Content exists and is up to date - skip logger.content.skipped(contentItem, "up to date, skipping", locale, targetGuid); itemsToSkip.push(contentItem); + skipCount++; } } catch (error: any) { // If we can't check, err on the side of processing it @@ -70,6 +85,11 @@ export async function filterContentItemsForProcessing({ } } + // Log decision summary if verbose + if (state.verbose && contentItems.length > 0) { + console.log(ansiColors.gray(`[FilterContent] Decision summary: ${createCount} create, ${updateCount} update, ${skipCount} skip, ${conflictCount} conflict`)); + } + return { itemsToProcess, itemsToSkip, diff --git a/src/lib/pushers/content-pusher/util/find-content-in-target-instance.ts b/src/lib/pushers/content-pusher/util/find-content-in-target-instance.ts index d4d9b51..a994110 100644 --- a/src/lib/pushers/content-pusher/util/find-content-in-target-instance.ts +++ b/src/lib/pushers/content-pusher/util/find-content-in-target-instance.ts @@ -28,6 +28,7 @@ export function findContentInTargetInstance({ referenceMapper }: Props): FindResult { const state = getState(); + const itemName = sourceContent.properties?.referenceName || `ID:${sourceContent.contentID}`; // STEP 1: Find existing mapping @@ -37,9 +38,15 @@ export function findContentInTargetInstance({ let targetContent: mgmtApi.ContentItem | null = null; if (mapping) { - // STEP 2: Find target content item using mapping targetContent = referenceMapper.getMappedEntity(mapping, "target"); + + // Diagnostic: mapping exists but target entity file is missing + if (!targetContent && state.verbose) { + // console.log(`[FindContent] ${itemName}: Mapping exists (target ID: ${mapping.targetContentID}) but target entity file not found`); + } + } else if (state.verbose) { + // console.log(`[FindContent] ${itemName}: No mapping found for source content ID ${sourceContent.contentID}`); } // STEP 3: Use change detection for conflict resolution diff --git a/src/lib/pushers/gallery-pusher.ts b/src/lib/pushers/gallery-pusher.ts index e3d97ea..05bd6d4 100644 --- a/src/lib/pushers/gallery-pusher.ts +++ b/src/lib/pushers/gallery-pusher.ts @@ -4,6 +4,30 @@ import { Logs } from "core/logs"; import { state, getState, getApiClient, getLoggerForGuid } from "core/state"; import { GalleryMapper } from "lib/mappers/gallery-mapper"; +/** + * Extract meaningful error message from API errors + */ +function extractErrorMessage(error: any): string { + // Check for axios response data (actual API error message) + if (error?.innerError?.response?.data) { + const data = error.innerError.response.data; + if (typeof data === 'string') return data; + if (data.message) return data.message; + if (data.error) return data.error; + if (data.Message) return data.Message; + if (data.Error) return data.Error; + } + // Check for direct response data + if (error?.response?.data) { + const data = error.response.data; + if (typeof data === 'string') return data; + if (data.message) return data.message; + if (data.error) return data.error; + } + // Fall back to error message + return error?.message || error?.innerError?.message || String(error); +} + /** * Enhanced gallery finder with proper target safety and conflict resolution * Logic Flow: Target Safety FIRST → Change Delta SECOND → Conflict Resolution @@ -44,21 +68,33 @@ export async function pushGalleries( for (const sourceGallery of galleries) { let currentStatus: "success" | "error" = "success"; try { - const existingMapping = referenceMapper.getGalleryMapping(sourceGallery, "source"); - const targetGallery = targetData.find(targetGallery => { return targetGallery.mediaGroupingID === sourceGallery.mediaGroupingID}); + const existingMapping = referenceMapper.getGalleryMapping(sourceGallery, "source"); + + // Check both: mapping file AND if gallery exists by name in target data + const targetGalleryByName = targetData.find(t => t.name === sourceGallery.name); + const targetGalleryById = targetData.find(t => t.mediaGroupingID === existingMapping?.targetMediaGroupingID); + + // If no mapping but gallery exists by name in target, create/update the mapping + if (!existingMapping && targetGalleryByName) { + // Gallery exists in target by name but no mapping - add mapping and skip + referenceMapper.addMapping(sourceGallery, targetGalleryByName); + logger.gallery.skipped(sourceGallery, "already exists in target by name", targetGuid[0]); + skipped++; + continue; + } - const shouldCreate = existingMapping === null; + const shouldCreate = existingMapping === null && !targetGalleryByName; if (shouldCreate) { // Gallery needs to be created (doesn't exist in target) await createGallery(sourceGallery, apiClient, targetGuid[0], referenceMapper, logger); successful++; - } else { - - const isTargetSafe = existingMapping !== null && referenceMapper.hasTargetChanged(targetGallery); - const hasSourceChanges = existingMapping !== null && referenceMapper.hasSourceChanged(sourceGallery); - let shouldUpdate = existingMapping !== null && isTargetSafe && hasSourceChanges; - let shouldSkip = existingMapping !== null && !isTargetSafe && !hasSourceChanges; + } else if (existingMapping) { + const targetGallery = targetGalleryById || targetGalleryByName; + const isTargetSafe = referenceMapper.hasTargetChanged(targetGallery); + const hasSourceChanges = referenceMapper.hasSourceChanged(sourceGallery); + let shouldUpdate = isTargetSafe && hasSourceChanges; + let shouldSkip = !isTargetSafe && !hasSourceChanges; if (overwrite) { shouldUpdate = true; @@ -72,20 +108,17 @@ export async function pushGalleries( } else if (shouldSkip) { // Gallery exists and is up to date - skip logger.gallery.skipped(sourceGallery, "up to date, skipping", targetGuid[0]); - // console.log(`✓ Gallery ${ansiColors.underline(sourceGallery.name)} ${ansiColors.bold.gray('up to date, skipping')}`); skipped++; } } } catch (error: any) { - logger.gallery.error(sourceGallery, error, targetGuid[0]) + const errorMsg = extractErrorMessage(error); + logger.gallery.error(sourceGallery, errorMsg, targetGuid[0]); failed++; currentStatus = "error"; overallStatus = "error"; } finally { processedCount++; - // if (onProgress) { - // onProgress(processedCount, totalGroupings, currentStatus); - // } } } @@ -99,6 +132,7 @@ export async function pushGalleries( /** * Create a new gallery in the target instance + * Only sends essential fields - lets the API set modifiedBy/modifiedOn */ async function createGallery( mediaGrouping: mgmtApi.assetMediaGrouping, @@ -107,19 +141,32 @@ async function createGallery( referenceMapper: GalleryMapper, logger: Logs ): Promise { - const payload = { ...mediaGrouping, mediaGroupingID: 0 }; - try { - const savedGallery = await apiClient.assetMethods.saveGallery(targetGuid, payload); - referenceMapper.addMapping(mediaGrouping, savedGallery); - logger.gallery.created(mediaGrouping, "created", targetGuid); - } catch (error) { - - logger.gallery.error(mediaGrouping, error, payload, targetGuid); - } + // Build payload with essential fields; API will set modifiedBy/modifiedOn automatically + // Include all required type fields with null for optional server-set values + const payload: mgmtApi.assetMediaGrouping = { + mediaGroupingID: 0, // 0 = create new + groupingType: null, // Let API set this + groupingTypeID: mediaGrouping.groupingTypeID ?? 1, // 1 = gallery + name: mediaGrouping.name, + description: mediaGrouping.description ?? null, + modifiedBy: null, // Let API set this + modifiedByName: null, // Let API set this + modifiedOn: null, // Let API set this + isDeleted: false, + isFolder: mediaGrouping.isFolder ?? false, + metaData: mediaGrouping.metaData && Object.keys(mediaGrouping.metaData).length > 0 + ? mediaGrouping.metaData + : {} + }; + // Let errors propagate to caller for proper failure tracking + const savedGallery = await apiClient.assetMethods.saveGallery(targetGuid, payload); + referenceMapper.addMapping(mediaGrouping, savedGallery); + logger.gallery.created(mediaGrouping, "created", targetGuid); } /** * Update an existing gallery in the target instance + * Only sends essential fields - lets the API set modifiedBy/modifiedOn */ async function updateGallery( sourceGallery: mgmtApi.assetMediaGrouping, @@ -129,7 +176,23 @@ async function updateGallery( referenceMapper: GalleryMapper, logger: Logs ): Promise { - const payload = { ...sourceGallery, mediaGroupingID: targetID }; + // Build payload with essential fields; API will set modifiedBy/modifiedOn automatically + // Include all required type fields with null for optional server-set values + const payload: mgmtApi.assetMediaGrouping = { + mediaGroupingID: targetID, // Use target's ID for update + groupingType: null, // Let API set this + groupingTypeID: sourceGallery.groupingTypeID ?? 1, + name: sourceGallery.name, + description: sourceGallery.description ?? null, + modifiedBy: null, // Let API set this + modifiedByName: null, // Let API set this + modifiedOn: null, // Let API set this + isDeleted: sourceGallery.isDeleted ?? false, + isFolder: sourceGallery.isFolder ?? false, + metaData: sourceGallery.metaData && Object.keys(sourceGallery.metaData).length > 0 + ? sourceGallery.metaData + : {} + }; const savedGallery = await apiClient.assetMethods.saveGallery(targetGuid, payload); referenceMapper.addMapping(sourceGallery, savedGallery); logger.gallery.updated(sourceGallery, "updated", targetGuid); diff --git a/src/lib/pushers/orchestrate-pushers.ts b/src/lib/pushers/orchestrate-pushers.ts index 6293073..dc7c131 100644 --- a/src/lib/pushers/orchestrate-pushers.ts +++ b/src/lib/pushers/orchestrate-pushers.ts @@ -2,7 +2,7 @@ import { getState, initializeGuidLogger, finalizeGuidLogger } from "../../core/s import { fileOperations } from "../../core/fileOperations"; import ansiColors from "ansi-colors"; import { GuidDataLoader, GuidEntities, ModelFilterOptions } from "./guid-data-loader"; -import { PusherResult, SourceData } from "../../types/sourceData"; +import { PusherResult, SourceData, FailureDetail } from "../../types/sourceData"; import { state } from "../../core/state"; import { PUSH_OPERATIONS, PushOperationsRegistry, PushOperationConfig } from "./push-operations-config"; @@ -19,6 +19,11 @@ export interface PushResults { totalSkipped: number; publishableContentIds: number[]; publishablePageIds: number[]; + // Per-locale tracking for proper batch workflow locale handling + publishableContentIdsByLocale: Map; + publishablePageIdsByLocale: Map; + // Individual failure details for error summary + failureDetails: FailureDetail[]; } export interface PusherConfig { @@ -29,11 +34,15 @@ export interface PusherConfig { export class Pushers { private config: PusherConfig; private startTime: Date = new Date(); - private fileOps: fileOperations; + private fileOps: fileOperations | null = null; constructor(config: PusherConfig = {}) { this.config = config; - this.fileOps = new fileOperations(state.sourceGuid[0], null); + // Defer fileOps creation until we have a valid sourceGuid + // This allows validation to provide a helpful error message first + if (state.sourceGuid && state.sourceGuid.length > 0 && state.sourceGuid[0]) { + this.fileOps = new fileOperations(state.sourceGuid[0], null); + } } /** @@ -54,6 +63,9 @@ export class Pushers { totalSkipped: 0, publishableContentIds: [], publishablePageIds: [], + publishableContentIdsByLocale: new Map(), + publishablePageIdsByLocale: new Map(), + failureDetails: [], }; try { @@ -69,6 +81,9 @@ export class Pushers { results.totalSkipped = pushResults.totalSkipped; results.publishableContentIds = pushResults.publishableContentIds; results.publishablePageIds = pushResults.publishablePageIds; + results.publishableContentIdsByLocale = pushResults.publishableContentIdsByLocale; + results.publishablePageIdsByLocale = pushResults.publishablePageIdsByLocale; + results.failureDetails = pushResults.failureDetails; // Calculate final duration results.totalDuration = Date.now() - startTime; @@ -142,6 +157,9 @@ export class Pushers { totalSkipped: number; publishableContentIds: number[]; publishablePageIds: number[]; + publishableContentIdsByLocale: Map; + publishablePageIdsByLocale: Map; + failureDetails: FailureDetail[]; }> { const { locale: locales, elements: stateElements } = state; const elements = stateElements.split(","); @@ -152,6 +170,11 @@ export class Pushers { let totalSkipped = 0; const publishableContentIds: number[] = []; const publishablePageIds: number[] = []; + // Per-locale tracking for proper batch workflow locale handling + const publishableContentIdsByLocale = new Map(); + const publishablePageIdsByLocale = new Map(); + // Collect individual failure details + const failureDetails: FailureDetail[] = []; // DEPENDENCY-OPTIMIZED ORDER: Galleries → Assets → Models → Containers → Content → Templates → Pages const pusherConfig = [ @@ -195,18 +218,22 @@ export class Pushers { for (const config of pusherConfig) { if (config === PUSH_OPERATIONS.pages || config === PUSH_OPERATIONS.content) continue; // Execute guid level op - await this.executePushOperation({ + const result = await this.executePushOperation({ config, sourceData, targetData, locale: locales[0], - totalSuccess, - totalFailures, - totalSkipped, publishableContentIds, publishablePageIds, elements, }); + // Accumulate results from returned values + totalSuccess += result.success; + totalFailures += result.failures; + totalSkipped += result.skipped; + if (result.failureDetails) { + failureDetails.push(...result.failureDetails); + } } } catch (error: any) { // Re-throw validation errors immediately to stop sync @@ -229,18 +256,38 @@ export class Pushers { ); const targetData = await targetDataLoader.loadGuidEntities(locale); - await this.executePushOperation({ + // Track IDs for this specific locale + const localeContentIds: number[] = []; + const localePageIds: number[] = []; + + const result = await this.executePushOperation({ config, sourceData, targetData, locale, - totalSuccess, - totalFailures, - totalSkipped, - publishableContentIds, - publishablePageIds, + publishableContentIds: config === PUSH_OPERATIONS.content ? localeContentIds : publishableContentIds, + publishablePageIds: config === PUSH_OPERATIONS.pages ? localePageIds : publishablePageIds, elements, }); + // Accumulate results from returned values + totalSuccess += result.success; + totalFailures += result.failures; + totalSkipped += result.skipped; + if (result.failureDetails) { + failureDetails.push(...result.failureDetails); + } + + // Store per-locale IDs and also add to combined list + if (config === PUSH_OPERATIONS.content && localeContentIds.length > 0) { + const existing = publishableContentIdsByLocale.get(locale) || []; + publishableContentIdsByLocale.set(locale, [...existing, ...localeContentIds]); + publishableContentIds.push(...localeContentIds); + } + if (config === PUSH_OPERATIONS.pages && localePageIds.length > 0) { + const existing = publishablePageIdsByLocale.get(locale) || []; + publishablePageIdsByLocale.set(locale, [...existing, ...localePageIds]); + publishablePageIds.push(...localePageIds); + } } } @@ -250,6 +297,9 @@ export class Pushers { totalSkipped, publishableContentIds, publishablePageIds, + publishableContentIdsByLocale, + publishablePageIdsByLocale, + failureDetails, }; } catch (error) { console.error(ansiColors.red("Error during pusher execution:"), error); @@ -262,9 +312,6 @@ export class Pushers { sourceData, targetData, locale, - totalSuccess, - totalFailures, - totalSkipped, publishableContentIds, publishablePageIds, elements, @@ -273,13 +320,10 @@ export class Pushers { sourceData: GuidEntities; targetData: GuidEntities; locale: string; - totalSuccess: number; - totalSkipped: number; - totalFailures: number; publishableContentIds?: number[]; publishablePageIds?: number[]; elements: string[]; - }) { + }): Promise<{ success: number; failures: number; skipped: number; failureDetails?: FailureDetail[] }> { const elementData = sourceData[config.dataKey as keyof GuidEntities] || []; // Skip if no data for this element type or element not requested @@ -288,23 +332,18 @@ export class Pushers { !elements.some((element) => config.elements.includes(element)) ) { console.log(ansiColors.yellow(`⚠️ Skipping ${config.description} for locale ${locale} - no data or filtered by --locales`)); - return; + return { success: 0, failures: 0, skipped: 0, failureDetails: [] }; } this.config.onOperationStart?.(config.name, state.sourceGuid[0], state.targetGuid[0]); const pusherResult: PusherResult = await config.handler(sourceData, targetData, locale); - // Accumulate results using standardized pattern - totalSuccess += pusherResult.successful || 0; - totalSkipped += pusherResult.skipped || 0; - totalFailures += pusherResult.failed || 0; - // Collect publishable IDs for workflow operations if (pusherResult.publishableIds && pusherResult.publishableIds.length > 0) { - if (config.elements.includes("Content")) { + if (config.elements.includes("Content") && publishableContentIds) { publishableContentIds.push(...pusherResult.publishableIds); - } else if (config.elements.includes("Pages")) { + } else if (config.elements.includes("Pages") && publishablePageIds) { publishablePageIds.push(...pusherResult.publishableIds); } } @@ -328,8 +367,13 @@ export class Pushers { pusherResult.status === "success", ); - // Save mappings after each pusher - // await referenceMapper.saveAllMappings(); + // Return the counts so they can be accumulated by the caller + return { + success: pusherResult.successful || 0, + failures: pusherResult.failed || 0, + skipped: pusherResult.skipped || 0, + failureDetails: pusherResult.failureDetails || [], + }; } /** diff --git a/src/lib/pushers/page-pusher/process-page.ts b/src/lib/pushers/page-pusher/process-page.ts index f93ee32..b0c717c 100644 --- a/src/lib/pushers/page-pusher/process-page.ts +++ b/src/lib/pushers/page-pusher/process-page.ts @@ -6,7 +6,7 @@ import { TemplateMapper } from "lib/mappers/template-mapper";// Internal helper import { translateZoneNames } from "./translate-zone-names"; import { findPageInOtherLocale, OtherLocaleMapping } from "./find-page-in-other-locale"; import { Logs } from "core/logs"; -import { state } from "core"; +import { state, getFailedContent, contentExistsInSourceData, contentExistsInOtherLocale } from "core/state"; interface Props { channel: string, @@ -22,6 +22,8 @@ interface Props { logger: Logs } +export type PageProcessResult = { status: "success" | "skip" | "failure"; error?: string; contentID?: number }; + export async function processPage({ channel, page, @@ -34,8 +36,8 @@ export async function processPage({ pageMapper, parentPageID, logger -}: Props): Promise<"success" | "skip" | "failure"> { - // Returns 'success', 'skip', or 'failure' +}: Props): Promise { + // Returns object with status and optional error message let existingPage: mgmtApi.PageItem | null = null; let channelID = -1; @@ -50,7 +52,7 @@ export async function processPage({ let templateRef = templateMapper.getTemplateMappingByPageTemplateName(page.templateName, 'source'); if (!templateRef) { logger.page.error(page, `Missing page template ${page.templateName} in source data, skipping`, locale, channel, targetGuid); - return "skip"; + return { status: "skip" }; } targetTemplate = templateMapper.getMappedEntity(templateRef, 'target') as mgmtApi.PageModel; } @@ -99,6 +101,7 @@ export async function processPage({ if (isConflict) { // CONFLICT: Target has changes, source has changes, and we're not in overwrite mode + // Skip processing - do NOT push to target or add to publishable IDs const sourceUrl = `https://app.agilitycms.com/instance/${sourceGuid}/${locale}/pages/${page.pageID}`; const targetUrl = `https://app.agilitycms.com/instance/${targetGuid}/${locale}/pages/${existingPage.pageID}`; @@ -108,6 +111,8 @@ export async function processPage({ ); console.warn(` - Source: ${sourceUrl}`); console.warn(` - Target: ${targetUrl}`); + + return { status: "skip" }; // Prevent conflicting pages from being processed and auto-published } else if (createRequired) { //CREATE NEW PAGE - nothing to do here yet... } else if (!updateRequired) { @@ -117,7 +122,7 @@ export async function processPage({ } logger.page.skipped(page, "up to date, skipping", locale, channel, targetGuid); - return "skip"; // Skip processing - page already exists + return { status: "skip" }; // Skip processing - page already exists } // Map Content IDs in Zones @@ -145,6 +150,9 @@ export async function processPage({ // Content mapping validation (silent unless errors) const contentMapper = new ContentItemMapper(sourceGuid, targetGuid, locale); + // Track first missing content mapping for error summary + let firstMissingContentError: string | null = null; + let firstMissingContentID: number | null = null; for (const [zoneName, zoneModules] of Object.entries(mappedZones)) { const newZoneContent = []; @@ -160,7 +168,8 @@ export async function processPage({ const sourceContentId = module.item.contentid || module.item.contentId; if (sourceContentId && sourceContentId > 0) { - const { targetContentID } = contentMapper.getContentItemMappingByContentID(sourceContentId, 'source'); + const contentMapping = contentMapper.getContentItemMappingByContentID(sourceContentId, 'source'); + const targetContentID = contentMapping?.targetContentID; if (targetContentID) { // CRITICAL FIX: Map to target content ID and remove duplicate fields const targetContentId = targetContentID; @@ -173,25 +182,36 @@ export async function processPage({ delete newModule.item.contentId; newZoneContent.push(newModule); } else { - // Content mapping failed - log detailed debug info for troubleshooting - console.error( - `❌ No content mapping found for ${module.module}: contentID ${sourceContentId} in page ${page.name}` - ); - // const contentMappings = contentMapper.getRecordsByType("content"); - - // console.log("Page", JSON.stringify(page, null, 2)); - // console.error(`Total content mappings available: ${contentMappings.length}`); - // const allContentRecords = pageMapper.getRecordsByType("content"); - // const matchingRecord = allContentRecords.find((r) => r.source.contentID === sourceContentId); - // if (matchingRecord) { - // console.error(`Found matching source record but issue with target:`, { - // sourceID: matchingRecord.source.contentID, - // targetID: matchingRecord.target?.contentID, - // hasTarget: !!matchingRecord.target, - // }); - // } else { - // console.error(`No record found with source contentID: ${sourceContentId}`); - // } + // Content mapping failed - check why (in priority order) + const failedContent = getFailedContent(sourceContentId); + const isPageUnpublished = page.properties?.state !== 2; + const existsInSource = contentExistsInSourceData(sourceGuid, locale, sourceContentId); + const otherLocale = !existsInSource ? contentExistsInOtherLocale(sourceGuid, locale, sourceContentId) : null; + let mappingError: string; + + if (failedContent) { + // Content failed earlier - show the upstream error + mappingError = `No content mapping for ${module.module} (contentID ${sourceContentId}) - content '${failedContent.referenceName}' failed earlier: ${failedContent.error}`; + } else if (isPageUnpublished) { + // Page is unpublished - explains why content isn't in source (it wasn't pulled) + mappingError = `No content mapping for ${module.module} (contentID ${sourceContentId}) - page is unpublished and referenced content may not have been pulled`; + } else if (otherLocale) { + // Content exists in another locale but not this one - not initialized for this locale + mappingError = `No content mapping for ${module.module} (contentID ${sourceContentId}) - content not initialized in ${locale} (exists in ${otherLocale})`; + } else if (!existsInSource) { + // Published page but content file doesn't exist anywhere - needs re-pull + mappingError = `No content mapping for ${module.module} (contentID ${sourceContentId}) - content item not found in source data (may need to re-pull)`; + } else { + // Content exists but wasn't synced - may be model issue + mappingError = `No content mapping for ${module.module} (contentID ${sourceContentId}) - content has never been synced or model may have changed`; + } + + // Don't log individual errors inline - they'll appear in the final summary + // Capture first error and contentID for summary + if (!firstMissingContentError) { + firstMissingContentError = mappingError; + firstMissingContentID = sourceContentId; + } } } else { // Module without content reference - keep it @@ -213,7 +233,8 @@ export async function processPage({ let missingMappings = 0; contentIdsToValidate.forEach((sourceContentId) => { - const { targetContentID } = contentMapper.getContentItemMappingByContentID(sourceContentId, 'source'); + const contentMapping = contentMapper.getContentItemMappingByContentID(sourceContentId, 'source'); + const targetContentID = contentMapping?.targetContentID; if (targetContentID) { mappingResults[sourceContentId] = { found: true, @@ -235,7 +256,11 @@ export async function processPage({ `✗ Page "${page.name}" failed - ${missingMappings}/${contentIdsToValidate.length} missing content mappings` ) ); - return "failure"; + return { + status: "failure", + error: firstMissingContentError || `${missingMappings} missing content mappings`, + contentID: firstMissingContentID || undefined + }; } } @@ -289,8 +314,9 @@ export async function processPage({ // If the page originally had modules but now has none, that's a problem // If it never had modules, that's fine (folder pages, etc.) if (originalModuleCount > 0 && !existingPage && !isLegitimateEmptyPage(page)) { - console.error(`✗ Page "${page.name}" lost all ${originalModuleCount} modules during content mapping`); - return "failure"; + const lostModulesError = `Lost all ${originalModuleCount} modules during content mapping`; + console.error(`✗ Page "${page.name}" ${lostModulesError}`); + return { status: "failure", error: lostModulesError }; } } @@ -334,13 +360,15 @@ export async function processPage({ } let placeBeforeIDArg = -1; - if (insertBeforePageId && insertBeforePageId > 0) { + // Only set placeBeforeIDArg for NEW pages, not updates + // For updates, we preserve the existing position unless explicitly moving + if (!existingPage && insertBeforePageId && insertBeforePageId > 0) { //map the insertBeforePageId to the correct target page ID const mapping = pageMapper.getPageMappingByPageID(insertBeforePageId, 'source'); if ((mapping?.targetPageID || 0) > 0) { placeBeforeIDArg = mapping.targetPageID; } - } + } const pageIDInOtherLocale = mappingToOtherLocale ? mappingToOtherLocale.PageIDOtherLanguage : -1; const otherLocale = mappingToOtherLocale ? mappingToOtherLocale.OtherLanguageCode : null; @@ -439,22 +467,32 @@ export async function processPage({ } else { logger.page.created(page, "created", locale, channel, targetGuid); } - return "success"; // Success + return { status: "success" }; // Success } else { - // Show errorData if available, otherwise generic failure - if (completedBatch.errorData && completedBatch.errorData.trim()) { - logger.page.error(page, `✗ Page "${page.name}" failed - ${completedBatch.errorData}, locale:${locale}`, locale, channel, targetGuid); + // Extract error message - prefer structured failedItems from new API + let errorMsg: string; + if (Array.isArray(completedBatch.failedItems) && completedBatch.failedItems.length > 0) { + // Use structured error from new API + errorMsg = completedBatch.failedItems[0].errorMessage || 'Unknown batch error'; + } else if (batchFailedItems.length > 0 && batchFailedItems[0].error) { + // Use error from extractBatchResults + errorMsg = batchFailedItems[0].error; + } else if (completedBatch.errorData && typeof completedBatch.errorData === 'string' && !completedBatch.errorData.startsWith('{')) { + // Use errorData only if it's a simple string (not JSON) + errorMsg = completedBatch.errorData.trim(); } else { - logger.page.error(page, `✗ Page "${page.name}" failed - invalid page ID: ${actualPageID}, locale:${locale}`, locale, channel, targetGuid); + errorMsg = `Invalid page ID: ${actualPageID}`; } - return "failure"; + logger.page.error(page, `✗ Page "${page.name}" failed - ${errorMsg}, locale:${locale}`, locale, channel, targetGuid); + return { status: "failure", error: errorMsg }; } } else { - logger.page.error(page, `✗ Page "${page.name}" failed in locale:${locale} - unexpected response format`, locale, channel, targetGuid); - return "failure"; // Failure + const errorMsg = "Unexpected response format"; + logger.page.error(page, `✗ Page "${page.name}" failed in locale:${locale} - ${errorMsg}`, locale, channel, targetGuid); + return { status: "failure", error: errorMsg }; } } catch (error: any) { logger.page.error(page, `✗ Page "${page.name}" failed in locale:${locale} - ${error.message}`, locale, channel, targetGuid); - return "failure"; // Failure + return { status: "failure", error: error.message }; } } diff --git a/src/lib/pushers/page-pusher/process-sitemap.ts b/src/lib/pushers/page-pusher/process-sitemap.ts index 8437015..e778a8e 100644 --- a/src/lib/pushers/page-pusher/process-sitemap.ts +++ b/src/lib/pushers/page-pusher/process-sitemap.ts @@ -1,4 +1,5 @@ import * as mgmtApi from "@agility/management-sdk"; +import ansiColors from "ansi-colors"; import { state, getApiClient } from "../../../core/state"; import { PusherResult } from "../../../types/sourceData"; import { SitemapHierarchy } from "./sitemap-hierarchy"; @@ -11,7 +12,8 @@ interface ReturnType { successful: number; failed: number; skipped: number; - publishableIds: number[] + publishableIds: number[]; + failureDetails: Array<{ name: string; error: string; type?: 'content' | 'page'; pageID?: number; contentID?: number; guid?: string; locale?: string }>; } interface Props { @@ -61,7 +63,8 @@ export async function processSitemap({ successful: 0, failed: 0, skipped: 0, - publishableIds: [] + publishableIds: [], + failureDetails: [] }; // Reverse the sitemap nodes to process them in the correct order @@ -76,8 +79,17 @@ export async function processSitemap({ const sourcePage = sourcePages.find(page => page.pageID === node.pageID); if (!sourcePage) { - logger.page.error(node, `source page with ID ${node.pageID} not found in source data.`, locale, channel, targetGuid); + const errorMsg = `source page with ID ${node.pageID} not found in source data.`; + logger.page.error(node, errorMsg, locale, channel, targetGuid); returnData.failed++; + returnData.failureDetails.push({ + name: `PageID ${node.pageID}`, + error: errorMsg, + type: 'page', + pageID: node.pageID, + guid: sourceGuid, + locale + }); continue; // Skip if source page is missing } @@ -106,18 +118,35 @@ export async function processSitemap({ logger }) - if (pageRes === "success") { + if (pageRes.status === "success") { returnData.successful++; - const mapping = pageMapper.getPageMappingByPageID(sourcePage.pageID, 'source'); - if (mapping) { - returnData.publishableIds.push(mapping.targetPageID); + // Only add to publishableIds if source page is in Published state (state === 2) + // Staging pages (state === 1) should not be auto-published + const isSourcePublished = sourcePage.properties?.state === 2; + if (isSourcePublished) { + const mapping = pageMapper.getPageMappingByPageID(sourcePage.pageID, 'source'); + if (mapping) { + returnData.publishableIds.push(mapping.targetPageID); + } + } else { + console.log(ansiColors.gray(` 📋 Skipping auto-publish for page "${sourcePage.name}" (state: ${sourcePage.properties?.state} - not published in source)`)); } - } else if (pageRes === "skip") { + } else if (pageRes.status === "skip") { returnData.skipped++; } else { + // pageRes.status is "failure" returnData.failed++; + returnData.failureDetails.push({ + name: sourcePage.name || `Page ${sourcePage.pageID}`, + error: pageRes.error || 'Unknown error', + type: 'page', + pageID: sourcePage.pageID, + contentID: pageRes.contentID, // Include contentID for linking to the missing content + guid: sourceGuid, + locale + }); } @@ -141,11 +170,17 @@ export async function processSitemap({ returnData.successful += childRes.successful; returnData.failed += childRes.failed; returnData.skipped += childRes.skipped; + returnData.publishableIds.push(...childRes.publishableIds); + returnData.failureDetails.push(...childRes.failureDetails); // Update previousPageID for next iteration previousPageID = node.pageID; } + + // Deduplicate publishableIds at the sitemap level to prevent duplicates from recursive calls + returnData.publishableIds = Array.from(new Set(returnData.publishableIds)); + return returnData; } \ No newline at end of file diff --git a/src/lib/pushers/page-pusher/push-pages.ts b/src/lib/pushers/page-pusher/push-pages.ts index 148b3f7..b6cd47b 100644 --- a/src/lib/pushers/page-pusher/push-pages.ts +++ b/src/lib/pushers/page-pusher/push-pages.ts @@ -19,7 +19,7 @@ export async function pushPages( if (!pages || pages.length === 0) { console.log("No pages found to process."); - return { status: "success", successful: 0, failed: 0, skipped: 0 }; + return { status: "success", successful: 0, failed: 0, skipped: 0, failureDetails: [] }; } const sitemapHierarchy = new SitemapHierarchy(); @@ -37,6 +37,7 @@ export async function pushPages( let skipped = 0; // No duplicates to skip since API prevents true duplicates at same hierarchy level let status: "success" | "error" = "success"; let publishableIds: number[] = []; // Track target page IDs for workflow operations + let failureDetails: Array<{ name: string; error: string; type?: 'content' | 'page'; pageID?: number; contentID?: number; guid?: string; locale?: string }> = []; //loop all the channels @@ -74,17 +75,30 @@ export async function pushPages( if (res.publishableIds && res.publishableIds.length > 0) { publishableIds.push(...res.publishableIds); } + if (res.failureDetails && res.failureDetails.length > 0) { + failureDetails.push(...res.failureDetails); + } if (res.failed > 0) { status = "error"; } - } catch (error) { - logger.page.error(null, `⚠️ Error in page processing for channel: ${channel}: ${JSON.stringify(error, null, 2)}`, locale, channel, targetGuid[0]); + } catch (error: any) { + // Use error.message instead of JSON.stringify to avoid circular reference issues with SDK errors + const errorMessage = error?.message || String(error); + logger.page.error(null, `⚠️ Error in page processing for channel: ${channel}: ${errorMessage}`, locale, channel, targetGuid[0]); status = "error"; } } - return { status, successful, failed, skipped, publishableIds }; + // Deduplicate publishableIds to prevent "item already in batch" errors during workflow + const uniquePublishableIds = Array.from(new Set(publishableIds)); + + if (publishableIds.length !== uniquePublishableIds.length && state.verbose) { + console.log(ansiColors.gray(` 📋 Deduplicated publishable page IDs: ${publishableIds.length} → ${uniquePublishableIds.length}`)); + } + + return { status, successful, failed, skipped, publishableIds: uniquePublishableIds, failureDetails }; } + diff --git a/src/lib/shared/source-publish-status-checker.ts b/src/lib/shared/source-publish-status-checker.ts index de91b70..7fd099b 100644 --- a/src/lib/shared/source-publish-status-checker.ts +++ b/src/lib/shared/source-publish-status-checker.ts @@ -133,6 +133,7 @@ export function filterPublishedPages( /** * Check publish status for all content and page mappings * Returns filtered lists of target IDs that should be published + * CRITICAL: All ID arrays are deduplicated to prevent "item already in batch" errors */ export function checkSourcePublishStatus( contentMappings: ContentMapping[], @@ -143,11 +144,14 @@ export function checkSourcePublishStatus( const contentResult = filterPublishedContent(contentMappings, sourceGuid, locales); const pageResult = filterPublishedPages(pageMappings, sourceGuid, locales); + // CRITICAL: Deduplicate all ID arrays to prevent "item already in batch" API errors + // Duplicate IDs can occur when the same source→target mapping appears multiple times + // (e.g., from multiple locales or duplicate entries in mapping files) return { - publishedContentIds: contentResult.publishedContentIds, - unpublishedContentIds: contentResult.unpublishedContentIds, - publishedPageIds: pageResult.publishedPageIds, - unpublishedPageIds: pageResult.unpublishedPageIds, + publishedContentIds: Array.from(new Set(contentResult.publishedContentIds)), + unpublishedContentIds: Array.from(new Set(contentResult.unpublishedContentIds)), + publishedPageIds: Array.from(new Set(pageResult.publishedPageIds)), + unpublishedPageIds: Array.from(new Set(pageResult.unpublishedPageIds)), errors: [...contentResult.errors, ...pageResult.errors] }; } diff --git a/src/lib/workflows/process-batches.ts b/src/lib/workflows/process-batches.ts index b742c25..c8f7033 100644 --- a/src/lib/workflows/process-batches.ts +++ b/src/lib/workflows/process-batches.ts @@ -155,8 +155,14 @@ export async function processBatches( errors: string[] ): Promise { const logLines: string[] = []; + + // CRITICAL: Deduplicate IDs to prevent "item already in batch" API errors + // This is a defensive measure in case upstream code doesn't dedupe properly + const uniqueIds = Array.from(new Set(ids)); + const duplicatesRemoved = ids.length - uniqueIds.length; + const results: BatchProcessingResult = { - total: ids.length, + total: uniqueIds.length, processed: 0, failed: 0, batches: 0, @@ -164,27 +170,32 @@ export async function processBatches( logLines: [] }; - if (ids.length === 0) return results; + if (uniqueIds.length === 0) return results; const label = type === 'content' ? 'Content' : 'Page'; const operationName = getOperationName(operation); const operationVerb = getOperationVerb(operation); - logLine(ansiColors.cyan(`\n${operationName}ing ${ids.length} ${label.toLowerCase()} items...`), logLines); + // Log deduplication if any duplicates were removed + if (duplicatesRemoved > 0) { + logLine(ansiColors.gray(` 📋 Deduplicated ${label.toLowerCase()} IDs: ${ids.length} → ${uniqueIds.length} (${duplicatesRemoved} duplicates removed)`), logLines); + } + + logLine(ansiColors.cyan(`\n${operationName}ing ${uniqueIds.length} ${label.toLowerCase()} items...`), logLines); // Get item display info and show breakdown (ALL items, no truncation) const targetGuid = state.targetGuid?.[0]; if (targetGuid) { const displayMap = type === 'content' - ? getContentDisplayInfo(ids, targetGuid, locale) - : getPageDisplayInfo(ids, targetGuid, locale); + ? getContentDisplayInfo(uniqueIds, targetGuid, locale) + : getPageDisplayInfo(uniqueIds, targetGuid, locale); if (displayMap.size > 0) { - displayItemBreakdown(ids, type, targetGuid, locale, operationName, displayMap, logLines); + displayItemBreakdown(uniqueIds, type, targetGuid, locale, operationName, displayMap, logLines); } } - const batches = createBatches(ids); + const batches = createBatches(uniqueIds); results.batches = batches.length; for (let i = 0; i < batches.length; i++) { @@ -192,17 +203,28 @@ export async function processBatches( const batchNum = i + 1; const progress = Math.round((batchNum / batches.length) * 100); + // Initial log - batch ID will be shown in the progress display from batchWorkflow logLine(ansiColors.gray(`[${progress}%] ${label} batch ${batchNum}/${batches.length}: ${operationName}ing ${batch.length} items...`), logLines); try { const batchResult = await batchWorkflow(batch, locale, operation, type); + const batchIdStr = batchResult.batchId ? ` (batch ID: ${batchResult.batchId})` : ''; if (batchResult.success) { results.processed += batchResult.processedIds.length; results.processedIds.push(...batchResult.processedIds); + + // Handle partial success - some items succeeded, some failed + if (batchResult.partialSuccess) { + const { successCount, failureCount, batchId } = batchResult.partialSuccess; + results.failed += failureCount; + + // Add a "completed with errors" message instead of failure message + errors.push(`${label} batch ${batchNum} (ID: ${batchId}): Completed with errors - ${successCount} succeeded, ${failureCount} failed`); + } } else { results.failed += batch.length; - errors.push(`${label} batch ${batchNum}: ${batchResult.error}`); + errors.push(`${label} batch ${batchNum}${batchIdStr}: ${batchResult.error}`); } } catch (error: any) { results.failed += batch.length; diff --git a/src/lib/workflows/workflow-operation.ts b/src/lib/workflows/workflow-operation.ts index d942288..e7d15f8 100644 --- a/src/lib/workflows/workflow-operation.ts +++ b/src/lib/workflows/workflow-operation.ts @@ -92,20 +92,72 @@ export class WorkflowOperation { let pageIds: number[]; if (useExplicitIDs) { - // Explicit IDs mode - bypass mappings lookup + // Explicit IDs mode - bypass mappings lookup but still check source publish status console.log(ansiColors.cyan('\n🔧 Using explicit IDs (bypassing mappings lookup)')); - contentIds = hasExplicitContentIDs ? state.explicitContentIDs : []; - pageIds = hasExplicitPageIDs ? state.explicitPageIDs : []; + let explicitContentIds = hasExplicitContentIDs ? state.explicitContentIDs : []; + let explicitPageIds = hasExplicitPageIDs ? state.explicitPageIDs : []; - console.log(ansiColors.gray(` Explicit content IDs: ${contentIds.length > 0 ? contentIds.join(', ') : '(none)'}`)); - console.log(ansiColors.gray(` Explicit page IDs: ${pageIds.length > 0 ? pageIds.join(', ') : '(none)'}`)); + console.log(ansiColors.gray(` Explicit content IDs: ${explicitContentIds.length > 0 ? explicitContentIds.join(', ') : '(none)'}`)); + console.log(ansiColors.gray(` Explicit page IDs: ${explicitPageIds.length > 0 ? explicitPageIds.join(', ') : '(none)'}`)); - if (contentIds.length === 0 && pageIds.length === 0) { + if (explicitContentIds.length === 0 && explicitPageIds.length === 0) { console.log(ansiColors.yellow('\n⚠️ No valid IDs provided.')); result.elapsedTime = Date.now() - startTime; return result; } + + // For publish operations, check source publish status even with explicit IDs + if (operationType === WorkflowOperationType.Publish) { + console.log(ansiColors.cyan('\nChecking source instance publish status for explicit IDs...')); + + // Read mappings to get source→target relationships (needed for reverse lookup) + const mappingResult = readMappingsForGuidPair(source, target, locales); + + // Create reverse lookup maps (target ID → source mapping) + const targetToSourceContent = new Map(); + const targetToSourcePage = new Map(); + + for (const mapping of mappingResult.contentMappings) { + targetToSourceContent.set(mapping.targetContentID, mapping); + } + for (const mapping of mappingResult.pageMappings) { + targetToSourcePage.set(mapping.targetPageID, mapping); + } + + // Filter explicit IDs to only those that have mappings + const contentMappingsForExplicit = explicitContentIds + .filter(id => targetToSourceContent.has(id)) + .map(id => targetToSourceContent.get(id)); + const pageMappingsForExplicit = explicitPageIds + .filter(id => targetToSourcePage.has(id)) + .map(id => targetToSourcePage.get(id)); + + // Check source publish status + const publishStatus = checkSourcePublishStatus( + contentMappingsForExplicit, + pageMappingsForExplicit, + source, + locales + ); + + // Report filtering results + const contentPublishedInSource = publishStatus.publishedContentIds.length; + const pagesPublishedInSource = publishStatus.publishedPageIds.length; + const contentSkipped = publishStatus.unpublishedContentIds.length; + const pagesSkipped = publishStatus.unpublishedPageIds.length; + + console.log(ansiColors.gray(`Content: ${contentPublishedInSource}/${explicitContentIds.length} published in source (${contentSkipped} staging/unpublished skipped)`)); + console.log(ansiColors.gray(`Pages: ${pagesPublishedInSource}/${explicitPageIds.length} published in source (${pagesSkipped} staging/unpublished skipped)`)); + + // Use only IDs that are published in source + contentIds = publishStatus.publishedContentIds; + pageIds = publishStatus.publishedPageIds; + } else { + // For non-publish operations, use all explicit IDs + contentIds = explicitContentIds; + pageIds = explicitPageIds; + } } else { // Standard mode - use mappings files console.log(ansiColors.gray(`\nMapping Summary:`)); diff --git a/src/types/sourceData.ts b/src/types/sourceData.ts index 229a379..d97703e 100644 --- a/src/types/sourceData.ts +++ b/src/types/sourceData.ts @@ -26,6 +26,19 @@ export type PusherProgressCallback = ( itemName?: string ) => void; +/** + * Individual failure detail with optional link metadata + */ +export interface FailureDetail { + name: string; + error: string; + type?: 'content' | 'page'; // For generating appropriate link + pageID?: number; // Source page ID for page links + contentID?: number; // Source content ID for content links + guid?: string; // Source instance GUID + locale?: string; // Locale code +} + /** * Standardized pusher result interface for all pusher operations * Replaces inline type definitions with consistent response structure @@ -36,6 +49,7 @@ export interface PusherResult { skipped: number; status: 'success' | 'error'; publishableIds?: number[]; // Optional: target instance IDs for workflow operations (content items and pages only) + failureDetails?: FailureDetail[]; // Individual failure details for error summary } /** diff --git a/src/types/workflows.ts b/src/types/workflows.ts index cc6785b..ba8dbf7 100644 --- a/src/types/workflows.ts +++ b/src/types/workflows.ts @@ -20,7 +20,14 @@ export interface BatchWorkflowResult { success: boolean; processedIds: number[]; failedCount: number; + batchId?: number; // The batch ID from the API error?: string; + /** Partial success details when some items succeed and some fail */ + partialSuccess?: { + successCount: number; + failureCount: number; + batchId: number; + }; } /** diff --git a/yarn.lock b/yarn.lock index 2a70273..78cde92 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1720,6 +1720,11 @@ fs.realpath@^1.0.0: resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== +fsevents@^2.3.2: + version "2.3.3" + resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz" + integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== + function-bind@^1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz" From b78318ae0b2e4fcd08961d0f7ccfd276a2db071e Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 18 Feb 2026 12:41:20 -0500 Subject: [PATCH 09/19] Refactor contentExistsInOtherLocale function to only consider API-known locales, improving accuracy in locale checks. --- src/core/state.ts | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/core/state.ts b/src/core/state.ts index ce2786a..a7f8c2a 100644 --- a/src/core/state.ts +++ b/src/core/state.ts @@ -849,29 +849,27 @@ export function contentExistsInSourceData(guid: string, locale: string, contentI } /** - * Check if a content item exists in another locale but not the current one - * Returns the locale where it exists, or null if not found anywhere + * Check if a content item exists in another locale but not the current one. + * Only considers API-known locales (state.availableLocales) so deleted locales + * with leftover folders (e.g. DONOTUSE) are not treated as valid "other locale". + * Returns the locale where it exists, or null if not found anywhere. */ export function contentExistsInOtherLocale(guid: string, currentLocale: string, contentID: number): string | null { const fs = require('fs'); const path = require('path'); - - // Get the guid folder and find all locale folders + + const validLocales = (state.availableLocales || []).filter((l) => l !== currentLocale); + if (validLocales.length === 0) return null; + const guidPath = path.join(process.cwd(), state.rootPath, guid); if (!fs.existsSync(guidPath)) return null; - - const entries = fs.readdirSync(guidPath, { withFileTypes: true }); - const localeFolders = entries - .filter((entry: any) => entry.isDirectory() && entry.name !== 'models' && entry.name !== currentLocale) - .map((entry: any) => entry.name); - - // Check each locale for the content item - for (const locale of localeFolders) { + + for (const locale of validLocales) { const contentPath = path.join(guidPath, locale, 'item', `${contentID}.json`); if (fs.existsSync(contentPath)) { return locale; } } - + return null; } From ccea5125ec0b2278305ace3ced0ef9265ff3120e Mon Sep 17 00:00:00 2001 From: Aaron Date: Wed, 18 Feb 2026 17:22:35 -0500 Subject: [PATCH 10/19] Update process-page.ts Bug fix --- src/lib/pushers/page-pusher/process-page.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/pushers/page-pusher/process-page.ts b/src/lib/pushers/page-pusher/process-page.ts index b0c717c..1f7031e 100644 --- a/src/lib/pushers/page-pusher/process-page.ts +++ b/src/lib/pushers/page-pusher/process-page.ts @@ -333,7 +333,9 @@ export async function processPage({ const payload: any = { ...pageCopy, - pageID: existingPage ? existingPage.pageID : -1, // Use existing page ID if available + // If local target file is missing but mapping exists, use the known target container ID + // to force an UPDATE instead of INSERT (prevents duplicate name server errors) + pageID: existingPage ? existingPage.pageID : (pageMapping?.targetPageID ?? -1), title: pageTitle, // CRITICAL: Ensure title is always present channelID: channelID, // CRITICAL: Always use target instance channel ID to avoid FK constraint errors zones: formattedZones, From 1511815d702b512485261222a45e83e65a294c05 Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 19 Feb 2026 12:24:56 -0500 Subject: [PATCH 11/19] Updates to page conflict detection logic --- src/lib/mappers/page-mapper.ts | 8 +++++--- src/lib/pushers/page-pusher/process-page.ts | 22 +++++++++++++-------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/lib/mappers/page-mapper.ts b/src/lib/mappers/page-mapper.ts index e36a612..d8432e9 100644 --- a/src/lib/mappers/page-mapper.ts +++ b/src/lib/mappers/page-mapper.ts @@ -121,10 +121,12 @@ export class PageMapper { return sourcePage.properties.versionID > mapping.sourceVersionID; } - hasTargetChanged(targetPage: mgmtApi.PageItem) { - if (!targetPage) return false; - const mapping = this.getPageMapping(targetPage, 'target'); + hasTargetChanged(sourcePage: mgmtApi.PageItem) { + if (!sourcePage) return false; + const mapping = this.getPageMapping(sourcePage, 'source'); if (!mapping) return false; + const targetPage = this.getMappedEntity(mapping, 'target'); + if (!targetPage) return false; // no downloaded target file yet (e.g. first sync) — can't detect change return targetPage.properties.versionID > mapping.targetVersionID; } diff --git a/src/lib/pushers/page-pusher/process-page.ts b/src/lib/pushers/page-pusher/process-page.ts index 1f7031e..b0ac8cf 100644 --- a/src/lib/pushers/page-pusher/process-page.ts +++ b/src/lib/pushers/page-pusher/process-page.ts @@ -85,10 +85,13 @@ export async function processPage({ channelID = sitemap?.[0]?.digitalChannelID || 1; // Fallback to first channel or default } - const hasTargetChanged = pageMapper.hasTargetChanged(existingPage); const hasSourceChanged = pageMapper.hasSourceChanged(page); + const hasTargetChanged = pageMapper.hasTargetChanged(page); - const isConflict = hasTargetChanged && hasSourceChanged; + // A conflict exists whenever the target has changed independently — regardless of whether + // the source also changed. Even if source is unchanged today, a future source push would + // silently overwrite the target's independent changes without this guard. + const isConflict = hasTargetChanged; const updateRequired = (hasSourceChanged && !isConflict) || overwrite; const createRequired = !existingPage; @@ -100,14 +103,18 @@ export async function processPage({ }[page.pageType] || page.pageType; if (isConflict) { - // CONFLICT: Target has changes, source has changes, and we're not in overwrite mode - // Skip processing - do NOT push to target or add to publishable IDs - + // CONFLICT: Target has independent changes — skip to prevent overwriting them. + // Use mapping's targetPageID as fallback in case existingPage wasn't loaded. + const targetPageID = existingPage?.pageID ?? pageMapping?.targetPageID; const sourceUrl = `https://app.agilitycms.com/instance/${sourceGuid}/${locale}/pages/${page.pageID}`; - const targetUrl = `https://app.agilitycms.com/instance/${targetGuid}/${locale}/pages/${existingPage.pageID}`; + const targetUrl = `https://app.agilitycms.com/instance/${targetGuid}/${locale}/pages/${targetPageID}`; + + const reason = hasSourceChanged + ? "changes detected in both source and target" + : "target has been changed independently"; console.warn( - `⚠️ Conflict detected ${pageTypeDisplay} ${ansiColors.underline(page.name)} ${ansiColors.bold.grey("changes detected in both source and target")}. Please resolve manually.` + `⚠️ Conflict detected ${pageTypeDisplay} ${ansiColors.underline(page.name)} ${ansiColors.bold.grey(reason)}. Please resolve manually.` ); console.warn(` - Source: ${sourceUrl}`); console.warn(` - Target: ${targetUrl}`); @@ -116,7 +123,6 @@ export async function processPage({ } else if (createRequired) { //CREATE NEW PAGE - nothing to do here yet... } else if (!updateRequired) { - // Add to reference mapper for future lookups if (existingPage) { pageMapper.addMapping(page, existingPage); } From 4ad5c4988617ebeab02118bdedee8031dcb9fb28 Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 19 Feb 2026 12:30:39 -0500 Subject: [PATCH 12/19] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c6bc1ba..f47e561 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agility/cli", - "version": "1.0.0-beta.13.1", + "version": "1.0.0-beta.13.2", "description": "Agility CLI for working with your content. (Public Beta)", "repository": { "type": "git", From 39da89a6fba202a981e01327601dc288e5d95ff8 Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 19 Feb 2026 12:52:03 -0500 Subject: [PATCH 13/19] Better conflict messages --- src/lib/mappers/page-mapper.ts | 13 ++++---- src/lib/pushers/page-pusher/process-page.ts | 33 +++++++++++---------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/lib/mappers/page-mapper.ts b/src/lib/mappers/page-mapper.ts index d8432e9..41724a4 100644 --- a/src/lib/mappers/page-mapper.ts +++ b/src/lib/mappers/page-mapper.ts @@ -121,13 +121,16 @@ export class PageMapper { return sourcePage.properties.versionID > mapping.sourceVersionID; } - hasTargetChanged(sourcePage: mgmtApi.PageItem) { - if (!sourcePage) return false; + hasTargetChanged(sourcePage: mgmtApi.PageItem): 'version_changed' | 'file_missing' | null { + if (!sourcePage) return null; const mapping = this.getPageMapping(sourcePage, 'source'); - if (!mapping) return false; + if (!mapping) return null; const targetPage = this.getMappedEntity(mapping, 'target'); - if (!targetPage) return false; // no downloaded target file yet (e.g. first sync) — can't detect change - return targetPage.properties.versionID > mapping.targetVersionID; + // Mapping exists but no downloaded file — page was previously synced and its file has + // since been removed (e.g. unpublished or deleted in the target instance). + if (!targetPage) return 'file_missing'; + if (targetPage.properties.versionID > mapping.targetVersionID) return 'version_changed'; + return null; } /** diff --git a/src/lib/pushers/page-pusher/process-page.ts b/src/lib/pushers/page-pusher/process-page.ts index b0ac8cf..b9267da 100644 --- a/src/lib/pushers/page-pusher/process-page.ts +++ b/src/lib/pushers/page-pusher/process-page.ts @@ -86,12 +86,12 @@ export async function processPage({ } const hasSourceChanged = pageMapper.hasSourceChanged(page); - const hasTargetChanged = pageMapper.hasTargetChanged(page); + const targetChangeResult = pageMapper.hasTargetChanged(page); // A conflict exists whenever the target has changed independently — regardless of whether // the source also changed. Even if source is unchanged today, a future source push would // silently overwrite the target's independent changes without this guard. - const isConflict = hasTargetChanged; + const isConflict = targetChangeResult !== null; const updateRequired = (hasSourceChanged && !isConflict) || overwrite; const createRequired = !existingPage; @@ -103,23 +103,31 @@ export async function processPage({ }[page.pageType] || page.pageType; if (isConflict) { - // CONFLICT: Target has independent changes — skip to prevent overwriting them. + // CONFLICT: Target has independent changes. // Use mapping's targetPageID as fallback in case existingPage wasn't loaded. const targetPageID = existingPage?.pageID ?? pageMapping?.targetPageID; const sourceUrl = `https://app.agilitycms.com/instance/${sourceGuid}/${locale}/pages/${page.pageID}`; const targetUrl = `https://app.agilitycms.com/instance/${targetGuid}/${locale}/pages/${targetPageID}`; - const reason = hasSourceChanged - ? "changes detected in both source and target" - : "target has been changed independently"; + let reason: string; + if (targetChangeResult === 'file_missing') { + reason = "target page may have been unpublished or deleted — cannot verify its current state"; + } else if (hasSourceChanged) { + reason = "changes detected in both source and target"; + } else { + reason = "target has been changed independently"; + } console.warn( - `⚠️ Conflict detected ${pageTypeDisplay} ${ansiColors.underline(page.name)} ${ansiColors.bold.grey(reason)}. Please resolve manually.` + `⚠️ Conflict detected ${pageTypeDisplay} ${ansiColors.underline(page.name)} — ${ansiColors.bold.grey(reason)}. Use --overwrite to force.` ); console.warn(` - Source: ${sourceUrl}`); console.warn(` - Target: ${targetUrl}`); - return { status: "skip" }; // Prevent conflicting pages from being processed and auto-published + if (!overwrite) { + return { status: "skip" }; // Prevent conflicting pages from being processed and auto-published + } + // overwrite mode: warn but continue processing } else if (createRequired) { //CREATE NEW PAGE - nothing to do here yet... } else if (!updateRequired) { @@ -465,13 +473,8 @@ export async function processPage({ folder: "Folder", }[page.pageType] || page.pageType; - if (existingPage) { - if (overwrite) { - logger.page.updated(page, "updated", locale, channel, targetGuid); - - } else { - logger.page.updated(page, "updated", locale, channel, targetGuid); - } + if (existingPage || pageMapping) { + logger.page.updated(page, "updated", locale, channel, targetGuid); } else { logger.page.created(page, "created", locale, channel, targetGuid); } From e4f99cae1e5be6b62ccd612910f486ade873e439 Mon Sep 17 00:00:00 2001 From: Aaron Date: Mon, 23 Feb 2026 11:27:54 -0500 Subject: [PATCH 14/19] Fixing page conflict detection logic --- package.json | 2 +- src/core/system-args.ts | 2 +- src/lib/mappers/page-mapper.ts | 7 ++----- src/lib/pushers/page-pusher/process-page.ts | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index f47e561..b849ad9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agility/cli", - "version": "1.0.0-beta.13.2", + "version": "1.0.0-beta.13.3", "description": "Agility CLI for working with your content. (Public Beta)", "repository": { "type": "git", diff --git a/src/core/system-args.ts b/src/core/system-args.ts index c9c2f7a..ebfbd78 100644 --- a/src/core/system-args.ts +++ b/src/core/system-args.ts @@ -194,7 +194,7 @@ export const systemArgs = { autoPublish: { describe: "Automatically publish content and/or pages after sync completes. Options: 'content' (publish only content), 'pages' (publish only pages), 'both' (publish content and pages). Default: both when flag is provided.", demandOption: false, - alias: ["auto-publish", "autoPublish", "AutoPublish", "AUTO_PUBLISH"], + alias: ["auto-publish", "autoPublish", "AutoPublish", "AUTO_PUBLISH", "autopublish"], type: "string" as const, coerce: (value: string | boolean) => { // Handle --autoPublish without value (defaults to 'both') diff --git a/src/lib/mappers/page-mapper.ts b/src/lib/mappers/page-mapper.ts index 41724a4..3710faf 100644 --- a/src/lib/mappers/page-mapper.ts +++ b/src/lib/mappers/page-mapper.ts @@ -1,7 +1,7 @@ import { fileOperations } from "../../core"; import * as mgmtApi from "@agility/management-sdk"; -interface PageMapping { +export interface PageMapping { sourceGuid: string; targetGuid: string; sourcePageID: number; @@ -121,11 +121,8 @@ export class PageMapper { return sourcePage.properties.versionID > mapping.sourceVersionID; } - hasTargetChanged(sourcePage: mgmtApi.PageItem): 'version_changed' | 'file_missing' | null { - if (!sourcePage) return null; - const mapping = this.getPageMapping(sourcePage, 'source'); + hasTargetChanged(targetPage: mgmtApi.PageItem | null, mapping: PageMapping | null): 'version_changed' | 'file_missing' | null { if (!mapping) return null; - const targetPage = this.getMappedEntity(mapping, 'target'); // Mapping exists but no downloaded file — page was previously synced and its file has // since been removed (e.g. unpublished or deleted in the target instance). if (!targetPage) return 'file_missing'; diff --git a/src/lib/pushers/page-pusher/process-page.ts b/src/lib/pushers/page-pusher/process-page.ts index b9267da..db15be1 100644 --- a/src/lib/pushers/page-pusher/process-page.ts +++ b/src/lib/pushers/page-pusher/process-page.ts @@ -86,7 +86,7 @@ export async function processPage({ } const hasSourceChanged = pageMapper.hasSourceChanged(page); - const targetChangeResult = pageMapper.hasTargetChanged(page); + const targetChangeResult = pageMapper.hasTargetChanged(existingPage, pageMapping); // A conflict exists whenever the target has changed independently — regardless of whether // the source also changed. Even if source is unchanged today, a future source push would From 5b6019ba005d947b1a264980e7d53689a8ebd5ef Mon Sep 17 00:00:00 2001 From: Aaron Date: Tue, 10 Mar 2026 10:33:13 -0400 Subject: [PATCH 15/19] Adding docs, and additional debugging --- README.md | 58 +++++++++++++++++++++++++++-- package.json | 2 +- src/core/logs.ts | 7 +++- src/lib/pushers/guid-data-loader.ts | 20 ++-------- src/lib/pushers/model-pusher.ts | 5 +++ 5 files changed, 69 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 531b2cb..5bf41bc 100644 --- a/README.md +++ b/README.md @@ -108,10 +108,11 @@ agility sync [options] **Operation Control Options:** -| Option | Type | Default | Description | -| ------------- | ------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `--update` | boolean | `false` | Download fresh data from source instance before operations, if left false, incremental sync is performed to only get changed data. | -| `--overwrite` | boolean | `false` | Force update existing items in target instance instead of creating new items with -1 IDs. Default: false (Warning: may cause duplicate items in lists, overwriting existing content) | +| Option | Type | Default | Description | +| ---------------- | ------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `--update` | boolean | `false` | Download fresh data from source instance before operations, if left false, incremental sync is performed to only get changed data. | +| `--overwrite` | boolean | `false` | Force update existing items in target instance instead of creating new items with -1 IDs. Default: false (Warning: may cause duplicate items in lists, overwriting existing content) | +| `--autoPublish` | string | _(disabled)_ | Automatically publish synced items that were published in the source instance. Values: `content`, `pages`, `both`. If flag is provided without a value, defaults to `both`. Items that are only in staging (not published) in the source are skipped. | **UI & Output Options:** @@ -141,10 +142,59 @@ agility sync --sourceGuid="abc123" --targetGuid="def456" --models="BlogPost,Blog # Sync models with dependencies (includes content, assets, galleries, containers, lists, but not pages) agility sync --sourceGuid="abc123" --targetGuid="def456" --models-with-deps="BlogPost,BlogCategory" + +# Sync and auto-publish everything that was published in source +agility sync --sourceGuid="abc123" --targetGuid="def456" --autoPublish + +# Sync and auto-publish only content (not pages) +agility sync --sourceGuid="abc123" --targetGuid="def456" --autoPublish=content + +# Sync and auto-publish only pages +agility sync --sourceGuid="abc123" --targetGuid="def456" --autoPublish=pages ``` --- ## Advanced Topics +### Auto-Publish + +The `--autoPublish` flag lets you automatically publish synced content and/or pages in the target instance immediately after a sync completes. Only items that are **published in the source instance** will be published in the target — staging-only items are skipped. + +#### Modes + +| Value | Behavior | +| --------- | --------------------------------------------- | +| `both` | Publish both content items and pages (default when flag is provided without a value) | +| `content` | Publish only content items | +| `pages` | Publish only pages | + +#### How It Works + +1. During sync, the CLI tracks which content items and pages were successfully pushed to the target instance +2. It also checks the publish state of each item in the source — only items with a published state are eligible +3. After all sync operations complete, the CLI publishes the eligible items in the target using the batch workflow API +4. Publishing is done per-locale to match the workflow API requirements +5. After publishing, reference mappings are updated to reflect the new published versions + +#### Examples + +```bash +# Auto-publish everything (content + pages) — flag without value defaults to 'both' +agility sync --sourceGuid="abc123" --targetGuid="def456" --autoPublish + +# Explicitly publish both +agility sync --sourceGuid="abc123" --targetGuid="def456" --autoPublish=both + +# Publish only content items (skip pages) +agility sync --sourceGuid="abc123" --targetGuid="def456" --autoPublish=content + +# Publish only pages (skip content) +agility sync --sourceGuid="abc123" --targetGuid="def456" --autoPublish=pages +``` + +> **Note:** Auto-publish only works with `sync` operations (not `pull`). Items that fail to sync will not be published. Any publish errors are reported in the final summary alongside sync errors. + +--- + ### Model-Specific Sync The CLI provides two options for selective synchronization based on specific content models: `--models` and `--models-with-deps`. This is particularly useful for large instances where you only want to sync certain content types. diff --git a/package.json b/package.json index b849ad9..a963a33 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agility/cli", - "version": "1.0.0-beta.13.3", + "version": "1.0.0-beta.13.4", "description": "Agility CLI for working with your content. (Public Beta)", "repository": { "type": "git", diff --git a/src/core/logs.ts b/src/core/logs.ts index b83e36c..c13a4f3 100644 --- a/src/core/logs.ts +++ b/src/core/logs.ts @@ -746,8 +746,11 @@ export class Logs { error: (payload: any, apiError: any, targetGuid?: string) => { const itemName = payload?.referenceName || payload?.displayName || `Model ${payload?.id || "Unknown"}`; - const errorDetails = apiError?.message || apiError || "Unknown error"; - // we need a better error logger for data elements + const baseMessage = apiError?.message || "Unknown error"; + const responseData = apiError?.response?.data ?? apiError?.response?.body ?? apiError?.data; + const responseStatus = apiError?.response?.status ?? apiError?.status; + const serverDetail = responseData ? ` | Server response (${responseStatus ?? "?"}): ${typeof responseData === "string" ? responseData : safeStringify(responseData)}` : ""; + const errorDetails = `${baseMessage}${serverDetail}`; this.logDataElement("model", "error", "failed", itemName, targetGuid || this.guid, errorDetails); }, }; diff --git a/src/lib/pushers/guid-data-loader.ts b/src/lib/pushers/guid-data-loader.ts index a622e8a..3fb8287 100644 --- a/src/lib/pushers/guid-data-loader.ts +++ b/src/lib/pushers/guid-data-loader.ts @@ -175,7 +175,7 @@ export class GuidDataLoader { // Validate that specified models exist - const validation = treeBuilder.validateModels(modelNames, completeEntities.models); + const validation = treeBuilder.validateModels(modelNames, (completeEntities ?? guidEntities).models); if (validation.invalid.length > 0) { // Use the correct source for available models (same as validation) const sourceForValidation = useFullDependencyTree ? completeEntities : guidEntities; @@ -305,21 +305,9 @@ export class GuidDataLoader { return { models: guidEntities.models.filter((m: any) => modelSet.has(m.referenceName)), - containers: guidEntities.containers.filter((c: any) => { - // Include containers that use the specified models - const model = guidEntities.models.find((m: any) => m.id === c.contentDefinitionID); - return model && modelSet.has(model.referenceName); - }), - lists: guidEntities.lists.filter((l: any) => { - // Include lists that use the specified models - const model = guidEntities.models.find((m: any) => m.id === l.contentDefinitionID); - return model && modelSet.has(model.referenceName); - }), - content: guidEntities.content.filter((c: any) => { - // Include content that uses the specified models - return modelSet.has(c.properties?.definitionName); - }), - // For simple filtering, don't include templates, pages, assets, galleries unless they're directly related + containers: [], + lists: [], + content: [], templates: [], pages: [], assets: [], diff --git a/src/lib/pushers/model-pusher.ts b/src/lib/pushers/model-pusher.ts index 6913f87..cf1c011 100644 --- a/src/lib/pushers/model-pusher.ts +++ b/src/lib/pushers/model-pusher.ts @@ -164,6 +164,11 @@ async function updateExistingModel( referenceMapper.addMapping(sourceModel, updatedModel); return "updated"; } catch (error: any) { + const axiosErr = error?.innerError; + console.error(`[model-pusher] SAVE FAILED for ${sourceModel?.referenceName}:`); + console.error(` message: ${error?.message}`); + console.error(` status: ${axiosErr?.response?.status ?? axiosErr?.status ?? "n/a"}`); + console.error(` responseData: ${JSON.stringify(axiosErr?.response?.data ?? axiosErr?.data ?? null, null, 2)}`); logger.model.error(sourceModel, error, targetGuid) return "failed"; } From fae36b992b457ebcaec6f2089a9a427c039b2deb Mon Sep 17 00:00:00 2001 From: Aaron Date: Tue, 10 Mar 2026 14:38:01 -0400 Subject: [PATCH 16/19] Detect auth/GUID failures early instead of cascading into locale fallback When all GUIDs fail API key retrieval (HTTP 500), bail immediately with a clear error pointing to auth expiry or bad GUIDs. In the locale detection catch block, distinguish between auth problems (keys also failed) and transient issues, and stop silently falling back to en-us. Co-Authored-By: Claude Opus 4.6 --- src/core/auth.ts | 48 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/src/core/auth.ts b/src/core/auth.ts index 4efc1a4..f1465c8 100644 --- a/src/core/auth.ts +++ b/src/core/auth.ts @@ -399,6 +399,7 @@ export class Auth { // Step 3: Get API keys for all GUIDs const allGuids = [...state.sourceGuid, ...state.targetGuid]; state.apiKeys = []; + const failedGuids: string[] = []; for (const guid of allGuids) { if (guid) { @@ -408,11 +409,23 @@ export class Auth { state.apiKeys.push({ guid, previewKey, fetchKey }); } catch (error) { + failedGuids.push(guid); console.log(ansiColors.yellow(`Warning: Could not get keys for GUID ${guid}: ${error.message}`)); } } } + // If ALL GUIDs failed key retrieval, this is almost certainly an auth or GUID problem + const validGuids = allGuids.filter(g => g); + if (validGuids.length > 0 && failedGuids.length === validGuids.length) { + console.log(ansiColors.red(`\nError: Failed to retrieve API keys for all specified GUIDs.`)); + console.log(ansiColors.red(`This usually means either:`)); + console.log(ansiColors.red(` 1. Your authentication has expired — run 'agility auth' to re-authenticate`)); + console.log(ansiColors.red(` 2. The GUID(s) are incorrect — verify your --sourceGuid / --targetGuid values`)); + console.log(ansiColors.red(` 3. Your account does not have access to these instances\n`)); + return false; + } + // Step 4: Set up UI mode in state state.useHeadless = state.headless; // headless takes precedence state.useVerbose = !state.useHeadless && state.verbose; @@ -524,17 +537,34 @@ export class Auth { } catch (error) { - console.log(ansiColors.yellow(`Note: Could not auto-detect locales: ${error.message}`)); - state.availableLocales = ["en-us"]; // Fallback to default - - // Create fallback mapping for all GUIDs - const fallbackLocales = state.locale.length > 0 ? [state.locale[0]] : ["en-us"]; - for (const guid of allGuids) { - if (guid) { - state.guidLocaleMap.set(guid, fallbackLocales); + // If we also failed to get keys for any GUIDs, this is likely an auth/GUID problem — fail fast + if (failedGuids.length > 0) { + console.log(ansiColors.red(`\nError: Unable to retrieve locales, and API key retrieval also failed.`)); + console.log(ansiColors.red(`This strongly indicates an authentication or GUID configuration problem.`)); + console.log(ansiColors.red(` - Run 'agility auth' to re-authenticate`)); + console.log(ansiColors.red(` - Verify your --sourceGuid / --targetGuid values are correct\n`)); + return false; + } + + // Keys succeeded but locales failed — could be a transient issue, fall back gracefully + console.log(ansiColors.yellow(`Warning: Could not auto-detect locales: ${error.message}`)); + if (state.locale.length > 0) { + // User specified locales explicitly, use those + const guidLocaleMap = new Map(); + for (const guid of allGuids) { + if (guid) { + guidLocaleMap.set(guid, state.locale); + } } + state.guidLocaleMap = guidLocaleMap; + state.availableLocales = state.locale; + console.log(`📝 Using user-specified locales: ${state.locale.join(", ")}`); + } else { + // No explicit locales and auto-detect failed — can't safely assume en-us + console.log(ansiColors.red(`Error: Could not detect available locales and no --locale flag was provided.`)); + console.log(ansiColors.red(`Please specify locales explicitly with --locale (e.g., --locale en-us)\n`)); + return false; } - console.log(`📝 Using fallback mapping: all GUIDs → ${fallbackLocales.join(", ")}`); } } From 5a544314ae4ed988afbeda94673c2c927c4775f6 Mon Sep 17 00:00:00 2001 From: Aaron Date: Tue, 10 Mar 2026 14:51:43 -0400 Subject: [PATCH 17/19] correcting llm commands --- src/core/auth.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/auth.ts b/src/core/auth.ts index f1465c8..429e86c 100644 --- a/src/core/auth.ts +++ b/src/core/auth.ts @@ -420,7 +420,7 @@ export class Auth { if (validGuids.length > 0 && failedGuids.length === validGuids.length) { console.log(ansiColors.red(`\nError: Failed to retrieve API keys for all specified GUIDs.`)); console.log(ansiColors.red(`This usually means either:`)); - console.log(ansiColors.red(` 1. Your authentication has expired — run 'agility auth' to re-authenticate`)); + console.log(ansiColors.red(` 1. Your authentication has expired — run 'agility login' to re-authenticate`)); console.log(ansiColors.red(` 2. The GUID(s) are incorrect — verify your --sourceGuid / --targetGuid values`)); console.log(ansiColors.red(` 3. Your account does not have access to these instances\n`)); return false; @@ -541,7 +541,7 @@ export class Auth { if (failedGuids.length > 0) { console.log(ansiColors.red(`\nError: Unable to retrieve locales, and API key retrieval also failed.`)); console.log(ansiColors.red(`This strongly indicates an authentication or GUID configuration problem.`)); - console.log(ansiColors.red(` - Run 'agility auth' to re-authenticate`)); + console.log(ansiColors.red(` - Run 'agility login' to re-authenticate`)); console.log(ansiColors.red(` - Verify your --sourceGuid / --targetGuid values are correct\n`)); return false; } From 8257873f58ad63ff3ef26e91c577277a45a133d2 Mon Sep 17 00:00:00 2001 From: Aaron Date: Tue, 10 Mar 2026 15:04:46 -0400 Subject: [PATCH 18/19] fail fast if we fail ANY key retrieval --- src/core/auth.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/auth.ts b/src/core/auth.ts index 429e86c..62b6569 100644 --- a/src/core/auth.ts +++ b/src/core/auth.ts @@ -415,9 +415,8 @@ export class Auth { } } - // If ALL GUIDs failed key retrieval, this is almost certainly an auth or GUID problem - const validGuids = allGuids.filter(g => g); - if (validGuids.length > 0 && failedGuids.length === validGuids.length) { + // If any GUIDs failed key retrieval, this is almost certainly an auth or GUID problem + if (failedGuids.length > 0) { console.log(ansiColors.red(`\nError: Failed to retrieve API keys for all specified GUIDs.`)); console.log(ansiColors.red(`This usually means either:`)); console.log(ansiColors.red(` 1. Your authentication has expired — run 'agility login' to re-authenticate`)); @@ -538,6 +537,7 @@ export class Auth { } catch (error) { // If we also failed to get keys for any GUIDs, this is likely an auth/GUID problem — fail fast + // This should never happen, but just in case if (failedGuids.length > 0) { console.log(ansiColors.red(`\nError: Unable to retrieve locales, and API key retrieval also failed.`)); console.log(ansiColors.red(`This strongly indicates an authentication or GUID configuration problem.`)); From 8a6105f4efba1862d80b937bbfc8d3214e6bd12d Mon Sep 17 00:00:00 2001 From: Aaron Date: Tue, 10 Mar 2026 15:08:54 -0400 Subject: [PATCH 19/19] Cleaning up output --- src/core/auth.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/auth.ts b/src/core/auth.ts index 62b6569..acacd80 100644 --- a/src/core/auth.ts +++ b/src/core/auth.ts @@ -410,14 +410,13 @@ export class Auth { state.apiKeys.push({ guid, previewKey, fetchKey }); } catch (error) { failedGuids.push(guid); - console.log(ansiColors.yellow(`Warning: Could not get keys for GUID ${guid}: ${error.message}`)); } } } // If any GUIDs failed key retrieval, this is almost certainly an auth or GUID problem if (failedGuids.length > 0) { - console.log(ansiColors.red(`\nError: Failed to retrieve API keys for all specified GUIDs.`)); + console.log(ansiColors.red(`\nError: Failed to retrieve API keys for one or more specified GUIDs.`)); console.log(ansiColors.red(`This usually means either:`)); console.log(ansiColors.red(` 1. Your authentication has expired — run 'agility login' to re-authenticate`)); console.log(ansiColors.red(` 2. The GUID(s) are incorrect — verify your --sourceGuid / --targetGuid values`)); @@ -539,7 +538,7 @@ export class Auth { // If we also failed to get keys for any GUIDs, this is likely an auth/GUID problem — fail fast // This should never happen, but just in case if (failedGuids.length > 0) { - console.log(ansiColors.red(`\nError: Unable to retrieve locales, and API key retrieval also failed.`)); + console.log(ansiColors.red(`Error: Unable to retrieve locales, and API key retrieval also failed.`)); console.log(ansiColors.red(`This strongly indicates an authentication or GUID configuration problem.`)); console.log(ansiColors.red(` - Run 'agility login' to re-authenticate`)); console.log(ansiColors.red(` - Verify your --sourceGuid / --targetGuid values are correct\n`));