diff --git a/packages/boxel-cli/src/commands/pull.ts b/packages/boxel-cli/src/commands/pull.ts new file mode 100644 index 0000000000..64f8780fa0 --- /dev/null +++ b/packages/boxel-cli/src/commands/pull.ts @@ -0,0 +1,180 @@ +import { + RealmSyncBase, + validateMatrixEnvVars, + type SyncOptions, +} from '../lib/realm-sync-base.js'; +import { + CheckpointManager, + type CheckpointChange, +} from '../lib/checkpoint-manager.js'; +import * as fs from 'fs'; +import * as path from 'path'; + +interface PullOptions extends SyncOptions { + deleteLocal?: boolean; +} + +class RealmPuller extends RealmSyncBase { + hasError = false; + + constructor( + private pullOptions: PullOptions, + matrixUrl: string, + username: string, + password: string, + ) { + super(pullOptions, matrixUrl, username, password); + } + + async sync(): Promise { + console.log( + `Starting pull from ${this.options.workspaceUrl} to ${this.options.localDir}`, + ); + + console.log('Testing workspace access...'); + try { + await this.getRemoteFileList(''); + } catch (error) { + console.error('Failed to access workspace:', error); + throw new Error( + 'Cannot proceed with pull: Authentication or access failed. ' + + 'Please check your Matrix credentials and workspace permissions.', + ); + } + console.log('Workspace access verified'); + + const remoteFiles = await this.getRemoteFileList(); + console.log(`Found ${remoteFiles.size} files in remote workspace`); + + const localFiles = await this.getLocalFileList(); + console.log(`Found ${localFiles.size} files in local directory`); + + if (!fs.existsSync(this.options.localDir)) { + if (this.options.dryRun) { + console.log( + `[DRY RUN] Would create directory: ${this.options.localDir}`, + ); + } else { + fs.mkdirSync(this.options.localDir, { recursive: true }); + console.log(`Created directory: ${this.options.localDir}`); + } + } + + const downloadedFiles: string[] = []; + for (const [relativePath] of remoteFiles) { + try { + const localPath = path.join(this.options.localDir, relativePath); + await this.downloadFile(relativePath, localPath); + downloadedFiles.push(relativePath); + } catch (error) { + this.hasError = true; + console.error(`Error downloading ${relativePath}:`, error); + } + } + + if (this.pullOptions.deleteLocal) { + const filesToDelete = new Set(localFiles.keys()); + for (const relativePath of remoteFiles.keys()) { + filesToDelete.delete(relativePath); + } + + if (filesToDelete.size > 0) { + const checkpointManager = new CheckpointManager(this.options.localDir); + const deleteChanges: CheckpointChange[] = Array.from(filesToDelete).map( + (f) => ({ + file: f, + status: 'deleted' as const, + }), + ); + const preDeleteCheckpoint = checkpointManager.createCheckpoint( + 'remote', + deleteChanges, + `Pre-delete checkpoint: ${filesToDelete.size} files not on server`, + ); + if (preDeleteCheckpoint) { + console.log( + `\nCheckpoint created before deletion: ${preDeleteCheckpoint.shortHash}`, + ); + } + + console.log( + `\nDeleting ${filesToDelete.size} local files that don't exist in workspace...`, + ); + + for (const relativePath of filesToDelete) { + try { + const localPath = localFiles.get(relativePath); + if (localPath) { + await this.deleteLocalFile(localPath); + console.log(` Deleted: ${relativePath}`); + } + } catch (error) { + this.hasError = true; + console.error(`Error deleting local file ${relativePath}:`, error); + } + } + } + } + + if (!this.options.dryRun && downloadedFiles.length > 0) { + const checkpointManager = new CheckpointManager(this.options.localDir); + const pullChanges: CheckpointChange[] = downloadedFiles.map((f) => ({ + file: f, + status: 'modified' as const, + })); + const checkpoint = checkpointManager.createCheckpoint( + 'remote', + pullChanges, + ); + if (checkpoint) { + const tag = checkpoint.isMajor ? '[MAJOR]' : '[minor]'; + console.log( + `\nCheckpoint created: ${checkpoint.shortHash} ${tag} ${checkpoint.message}`, + ); + } + } + + console.log('Pull completed'); + } +} + +export interface PullCommandOptions { + delete?: boolean; + dryRun?: boolean; +} + +export async function pullCommand( + workspaceUrl: string, + localDir: string, + options: PullCommandOptions, +): Promise { + const { matrixUrl, username, password } = + await validateMatrixEnvVars(workspaceUrl); + + try { + const puller = new RealmPuller( + { + workspaceUrl, + localDir, + deleteLocal: options.delete, + dryRun: options.dryRun, + }, + matrixUrl, + username, + password, + ); + + await puller.initialize(); + await puller.sync(); + + if (puller.hasError) { + console.log('Pull did not complete successfully. View logs for details'); + process.exit(2); + } else { + console.log('Pull completed successfully'); + } + } catch (error) { + console.error('Pull failed:', error); + process.exit(1); + } +} diff --git a/packages/boxel-cli/src/index.ts b/packages/boxel-cli/src/index.ts index b1350a2ab9..efcf53fed5 100644 --- a/packages/boxel-cli/src/index.ts +++ b/packages/boxel-cli/src/index.ts @@ -3,6 +3,7 @@ import { Command } from 'commander'; import { readFileSync } from 'fs'; import { resolve } from 'path'; import { profileCommand } from './commands/profile.js'; +import { pullCommand } from './commands/pull.js'; const pkg = JSON.parse( readFileSync(resolve(__dirname, '../package.json'), 'utf-8'), @@ -39,4 +40,28 @@ program }, ); +const workspace = program + .command('workspace') + .description('Workspace sync and management commands'); + +workspace + .command('pull') + .description('Pull files from a Boxel workspace to a local directory') + .argument( + '', + 'The URL of the source workspace (e.g., https://app.boxel.ai/demo/)', + ) + .argument('', 'The local directory to sync files to') + .option('--delete', 'Delete local files that do not exist in the workspace') + .option('--dry-run', 'Show what would be done without making changes') + .action( + async ( + workspaceUrl: string, + localDir: string, + options: { delete?: boolean; dryRun?: boolean }, + ) => { + await pullCommand(workspaceUrl, localDir, options); + }, + ); + program.parse(); diff --git a/packages/boxel-cli/src/lib/checkpoint-manager.ts b/packages/boxel-cli/src/lib/checkpoint-manager.ts new file mode 100644 index 0000000000..71136aded1 --- /dev/null +++ b/packages/boxel-cli/src/lib/checkpoint-manager.ts @@ -0,0 +1,549 @@ +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { isProtectedFile } from './realm-sync-base.js'; + +export interface Checkpoint { + hash: string; + shortHash: string; + message: string; + description: string; + date: Date; + isMajor: boolean; + filesChanged: number; + insertions: number; + deletions: number; + source: 'local' | 'remote' | 'manual'; + isMilestone: boolean; + milestoneName?: string; +} + +export interface CheckpointChange { + file: string; + status: 'added' | 'modified' | 'deleted'; +} + +export class CheckpointManager { + private workspaceDir: string; + private gitDir: string; + + constructor(workspaceDir: string) { + this.workspaceDir = path.resolve(workspaceDir); + this.gitDir = path.join(this.workspaceDir, '.boxel-history'); + } + + init(): void { + if (!fs.existsSync(this.gitDir)) { + fs.mkdirSync(this.gitDir, { recursive: true }); + } + + const gitPath = path.join(this.gitDir, '.git'); + if (!fs.existsSync(gitPath)) { + this.git('init'); + this.git('config', 'user.email', 'boxel-cli@local'); + this.git('config', 'user.name', 'Boxel CLI'); + this.git( + 'commit', + '--allow-empty', + '-m', + '[init] Initialize checkpoint history', + ); + } + } + + isInitialized(): boolean { + return fs.existsSync(path.join(this.gitDir, '.git')); + } + + private syncFilesToHistory(): void { + const files = this.getWorkspaceFiles(); + + const historyFiles = this.getHistoryFiles(); + for (const file of historyFiles) { + if (!files.includes(file)) { + const historyPath = path.join(this.gitDir, file); + if (fs.existsSync(historyPath)) { + fs.unlinkSync(historyPath); + } + } + } + + for (const file of files) { + const srcPath = path.join(this.workspaceDir, file); + const destPath = path.join(this.gitDir, file); + + const destDir = path.dirname(destPath); + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } + + fs.copyFileSync(srcPath, destPath); + } + } + + private getWorkspaceFiles(): string[] { + const files: string[] = []; + + const scan = (dir: string, prefix = '') => { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + if ( + entry.name === '.boxel-history' || + entry.name === '.boxel-sync.json' + ) { + continue; + } + if (entry.name.startsWith('.')) { + continue; + } + + const relPath = prefix ? `${prefix}/${entry.name}` : entry.name; + + if (entry.isDirectory()) { + scan(path.join(dir, entry.name), relPath); + } else { + files.push(relPath); + } + } + }; + + scan(this.workspaceDir); + return files; + } + + private getHistoryFiles(): string[] { + const files: string[] = []; + + const scan = (dir: string, prefix = '') => { + if (!fs.existsSync(dir)) return; + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + if (entry.name === '.git') continue; + + const relPath = prefix ? `${prefix}/${entry.name}` : entry.name; + + if (entry.isDirectory()) { + scan(path.join(dir, entry.name), relPath); + } else { + files.push(relPath); + } + } + }; + + scan(this.gitDir); + return files; + } + + detectCurrentChanges(): CheckpointChange[] { + if (!this.isInitialized()) { + const files = this.getWorkspaceFiles(); + return files.map((file) => ({ file, status: 'added' as const })); + } + + this.syncFilesToHistory(); + + const status = spawnSync('git', ['status', '--porcelain'], { + cwd: this.gitDir, + encoding: 'utf-8', + }); + + const statusOutput = status.stdout.trim(); + if (!statusOutput) { + return []; + } + + const changes: CheckpointChange[] = []; + for (const line of statusOutput.split('\n')) { + if (!line) continue; + + const statusCode = line.substring(0, 2); + const file = line.substring(3); + + if (statusCode.includes('R')) { + const arrowIndex = file.indexOf(' -> '); + if (arrowIndex !== -1) { + const oldFile = file.substring(0, arrowIndex); + const newFile = file.substring(arrowIndex + 4); + changes.push({ file: oldFile, status: 'deleted' }); + changes.push({ file: newFile, status: 'added' }); + continue; + } + } + + if (statusCode.includes('D')) { + changes.push({ file, status: 'deleted' }); + } else if ( + statusCode.includes('A') || + statusCode.includes('C') || + statusCode === '??' + ) { + changes.push({ file, status: 'added' }); + } else if ( + statusCode.includes('M') || + statusCode.includes('U') || + statusCode.includes('T') + ) { + changes.push({ file, status: 'modified' }); + } + } + + return changes; + } + + createCheckpoint( + source: 'local' | 'remote' | 'manual', + changes: CheckpointChange[], + customMessage?: string, + ): Checkpoint | null { + if (!this.isInitialized()) { + this.init(); + } + + this.syncFilesToHistory(); + + this.git('add', '-A'); + + const status = spawnSync('git', ['status', '--porcelain'], { + cwd: this.gitDir, + encoding: 'utf-8', + }); + + if (!status.stdout.trim()) { + return null; + } + + const isMajor = this.classifyChanges(changes); + + const { message, description } = customMessage + ? { message: customMessage, description: '' } + : this.generateCommitMessage(source, changes, isMajor); + + const prefix = isMajor ? '[MAJOR]' : '[minor]'; + const sourceTag = `[${source}]`; + const fullMessage = `${prefix} ${sourceTag} ${message}${description ? '\n\n' + description : ''}`; + + this.git('commit', '-m', fullMessage); + + const hash = this.git('rev-parse', 'HEAD').trim(); + const shortHash = hash.substring(0, 7); + + return { + hash, + shortHash, + message, + description, + date: new Date(), + isMajor, + filesChanged: changes.length, + insertions: 0, + deletions: 0, + source, + isMilestone: false, + }; + } + + private classifyChanges(changes: CheckpointChange[]): boolean { + if (changes.length > 3) return true; + + for (const change of changes) { + if (change.status === 'added' || change.status === 'deleted') return true; + if (change.file.endsWith('.gts')) return true; + } + + return false; + } + + private generateCommitMessage( + source: 'local' | 'remote' | 'manual', + changes: CheckpointChange[], + _isMajor: boolean, + ): { message: string; description: string } { + const sourceLabel = + source === 'local' ? 'Push' : source === 'remote' ? 'Pull' : 'Manual'; + + if (changes.length === 0) { + return { + message: `${sourceLabel}: No changes detected`, + description: '', + }; + } + + if (changes.length === 1) { + const change = changes[0]; + const action = + change.status === 'added' + ? 'Add' + : change.status === 'deleted' + ? 'Delete' + : 'Update'; + return { + message: `${sourceLabel}: ${action} ${change.file}`, + description: '', + }; + } + + const added = changes.filter((c) => c.status === 'added'); + const modified = changes.filter((c) => c.status === 'modified'); + const deleted = changes.filter((c) => c.status === 'deleted'); + + const parts: string[] = []; + if (added.length > 0) parts.push(`+${added.length}`); + if (modified.length > 0) parts.push(`~${modified.length}`); + if (deleted.length > 0) parts.push(`-${deleted.length}`); + + const message = `${sourceLabel}: ${changes.length} files (${parts.join(', ')})`; + + const lines: string[] = []; + if (added.length > 0) { + lines.push('Added:'); + added.forEach((c) => lines.push(` + ${c.file}`)); + } + if (modified.length > 0) { + lines.push('Modified:'); + modified.forEach((c) => lines.push(` ~ ${c.file}`)); + } + if (deleted.length > 0) { + lines.push('Deleted:'); + deleted.forEach((c) => lines.push(` - ${c.file}`)); + } + + return { message, description: lines.join('\n') }; + } + + getCheckpoints(limit = 50): Checkpoint[] { + if (!this.isInitialized()) { + return []; + } + + const format = '%H|%h|%s|%aI|%an'; + const log = this.git('log', `--format=${format}`, `-${limit}`); + + if (!log.trim()) { + return []; + } + + const milestones = this.getAllMilestones(); + + return log + .trim() + .split('\n') + .map((line) => { + const [hash, shortHash, subject, dateStr] = line.split('|'); + + const isMajor = subject.includes('[MAJOR]'); + const source = subject.includes('[local]') + ? ('local' as const) + : subject.includes('[remote]') + ? ('remote' as const) + : ('manual' as const); + + const message = subject + .replace(/\[(MAJOR|minor)\]\s*/i, '') + .replace(/\[(local|remote|manual)\]\s*/i, ''); + + const stats = this.getCommitStats(hash); + + const milestoneName = milestones.get(hash); + const isMilestone = !!milestoneName; + + return { + hash, + shortHash, + message, + description: '', + date: new Date(dateStr), + isMajor, + source, + isMilestone, + milestoneName, + ...stats, + }; + }); + } + + private getCommitStats(hash: string): { + filesChanged: number; + insertions: number; + deletions: number; + } { + try { + const stat = this.git('show', '--stat', '--format=', hash); + const lines = stat.trim().split('\n'); + const summaryLine = lines[lines.length - 1] || ''; + + const filesMatch = summaryLine.match(/(\d+) files? changed/); + const insertMatch = summaryLine.match(/(\d+) insertions?/); + const deleteMatch = summaryLine.match(/(\d+) deletions?/); + + return { + filesChanged: filesMatch ? parseInt(filesMatch[1]) : 0, + insertions: insertMatch ? parseInt(insertMatch[1]) : 0, + deletions: deleteMatch ? parseInt(deleteMatch[1]) : 0, + }; + } catch { + return { filesChanged: 0, insertions: 0, deletions: 0 }; + } + } + + getChangedFiles(hash: string): string[] { + const output = this.git('show', '--name-only', '--format=', hash); + return output.trim().split('\n').filter(Boolean); + } + + getDiff(hash: string): string { + return this.git('show', '--format=', hash); + } + + restore(hash: string): void { + const currentFiles = this.getHistoryFiles(); + for (const file of currentFiles) { + const filePath = path.join(this.gitDir, file); + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + } + + this.git('checkout', hash, '--', '.'); + + const historyFiles = this.getHistoryFiles(); + const workspaceFiles = this.getWorkspaceFiles(); + + for (const file of workspaceFiles) { + if (isProtectedFile(file)) continue; + if (!historyFiles.includes(file)) { + const filePath = path.join(this.workspaceDir, file); + fs.unlinkSync(filePath); + } + } + + for (const file of historyFiles) { + if (isProtectedFile(file)) continue; + const srcPath = path.join(this.gitDir, file); + const destPath = path.join(this.workspaceDir, file); + + const destDir = path.dirname(destPath); + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } + + fs.copyFileSync(srcPath, destPath); + } + + this.git('checkout', 'HEAD', '--', '.'); + } + + markMilestone( + hashOrIndex: string | number, + name: string, + ): { hash: string; name: string } | null { + if (!this.isInitialized()) { + return null; + } + + let hash: string; + if (typeof hashOrIndex === 'number') { + const checkpoints = this.getCheckpoints(hashOrIndex + 1); + if (hashOrIndex < 1 || hashOrIndex > checkpoints.length) { + return null; + } + hash = checkpoints[hashOrIndex - 1].hash; + } else { + hash = hashOrIndex; + } + + const tagName = `milestone/${name.replace(/\s+/g, '-').replace(/[^a-zA-Z0-9\-_.]/g, '')}`; + + try { + this.git('tag', '-a', tagName, hash, '-m', `Milestone: ${name}`); + return { hash, name }; + } catch { + return null; + } + } + + unmarkMilestone(hashOrIndex: string | number): boolean { + if (!this.isInitialized()) { + return false; + } + + let hash: string; + if (typeof hashOrIndex === 'number') { + const checkpoints = this.getCheckpoints(hashOrIndex + 1); + if (hashOrIndex < 1 || hashOrIndex > checkpoints.length) { + return false; + } + hash = checkpoints[hashOrIndex - 1].hash; + } else { + hash = hashOrIndex; + } + + const tags = this.getMilestoneTags(hash); + if (tags.length === 0) { + return false; + } + + for (const tag of tags) { + try { + this.git('tag', '-d', tag); + } catch { + // Ignore errors + } + } + + return true; + } + + private getMilestoneTags(hash: string): string[] { + try { + const output = this.git('tag', '--points-at', hash); + return output + .trim() + .split('\n') + .filter((tag) => tag.startsWith('milestone/')) + .filter(Boolean); + } catch { + return []; + } + } + + private getAllMilestones(): Map { + const milestones = new Map(); + try { + const tags = this.git('tag', '-l', 'milestone/*'); + for (const tag of tags.trim().split('\n').filter(Boolean)) { + try { + const hash = this.git('rev-list', '-1', tag).trim(); + const name = tag.replace('milestone/', '').replace(/-/g, ' '); + milestones.set(hash, name); + } catch { + // Ignore invalid tags + } + } + } catch { + // No tags + } + return milestones; + } + + getMilestones(): Checkpoint[] { + const all = this.getCheckpoints(100); + return all.filter((cp) => cp.isMilestone); + } + + private git(...args: string[]): string { + const result = spawnSync('git', args, { + cwd: this.gitDir, + encoding: 'utf-8', + }); + + if (result.error) { + throw result.error; + } + + if (result.status !== 0 && !args.includes('status')) { + throw new Error(`git ${args.join(' ')} failed: ${result.stderr}`); + } + + return result.stdout; + } +} diff --git a/packages/boxel-cli/src/lib/matrix-client.ts b/packages/boxel-cli/src/lib/matrix-client.ts new file mode 100644 index 0000000000..3a7dd4ecc0 --- /dev/null +++ b/packages/boxel-cli/src/lib/matrix-client.ts @@ -0,0 +1,252 @@ +import { Sha256 } from '@aws-crypto/sha256-js'; + +export interface MatrixAccess { + accessToken: string; + deviceId: string; + userId: string; +} + +export class MatrixClient { + readonly matrixURL: URL; + readonly username: string; + private access: MatrixAccess | undefined; + private password?: string; + private seed?: string; + private loginPromise: Promise | undefined; + + constructor({ + matrixURL, + username, + password, + seed, + }: { + matrixURL: URL; + username: string; + password?: string; + seed?: string; + }) { + if (!password && !seed) { + throw new Error( + 'Either password or a seed must be specified when creating a matrix client', + ); + } + this.matrixURL = matrixURL; + this.username = username; + this.password = password; + this.seed = seed; + } + + getUserId(): string | undefined { + return this.access?.userId; + } + + isLoggedIn(): boolean { + return this.access !== undefined; + } + + getAccessToken(): string | undefined { + return this.access?.accessToken; + } + + private async request( + path: string, + method: 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'GET' = 'GET', + options: RequestInit = {}, + includeAuth = true, + ): Promise { + options.method = method; + + if (includeAuth) { + if (!this.access) { + throw new Error('Missing matrix access token'); + } + options.headers = { + ...options.headers, + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.access.accessToken}`, + }; + } + return fetch(`${this.matrixURL.href}${path}`, options); + } + + async login(): Promise { + if (this.loginPromise) { + return this.loginPromise; + } + + this.loginPromise = this.performLogin(); + return this.loginPromise; + } + + private async performLogin(): Promise { + let password: string | undefined; + if (this.password) { + password = this.password; + } else if (this.seed) { + password = await passwordFromSeed(this.username, this.seed); + } else { + throw new Error( + 'bug: should never be here, we ensure password or seed exists in constructor', + ); + } + + const response = await this.request( + '_matrix/client/v3/login', + 'POST', + { + body: JSON.stringify({ + identifier: { + type: 'm.id.user', + user: this.username, + }, + password, + type: 'm.login.password', + }), + }, + false, + ); + + const json = (await response.json()) as Record; + + if (!response.ok) { + throw new Error( + `Unable to login to matrix ${this.matrixURL.href} as user ${this.username}: status ${response.status} - ${JSON.stringify(json)}`, + ); + } + + const { + access_token: accessToken, + device_id: deviceId, + user_id: userId, + } = json as { access_token: string; device_id: string; user_id: string }; + + this.access = { accessToken, deviceId, userId }; + } + + async getJoinedRooms(): Promise<{ joined_rooms: string[] }> { + const response = await this.request('_matrix/client/v3/joined_rooms'); + return (await response.json()) as { joined_rooms: string[] }; + } + + async joinRoom(roomId: string): Promise { + const response = await this.request( + `_matrix/client/v3/rooms/${roomId}/join`, + 'POST', + ); + if (!response.ok) { + const json = await response.json(); + throw new Error( + `Unable to join room ${roomId}: status ${response.status} - ${JSON.stringify(json)}`, + ); + } + } + + async getAccountData(type: string): Promise { + if (!this.access) { + throw new Error('Must be logged in to get account data'); + } + + const response = await this.request( + `_matrix/client/v3/user/${encodeURIComponent(this.access.userId)}/account_data/${type}`, + ); + + if (response.status === 404) { + return null; + } + + if (!response.ok) { + const errorBody = await this.safeReadErrorBody(response); + throw new Error( + `Unable to get account data '${type}' for ${this.access.userId}: status ${response.status} - ${JSON.stringify(errorBody)}`, + ); + } + + return (await response.json()) as T; + } + + async setAccountData(type: string, data: T): Promise { + if (!this.access) { + throw new Error('Must be logged in to set account data'); + } + + const response = await this.request( + `_matrix/client/v3/user/${encodeURIComponent(this.access.userId)}/account_data/${type}`, + 'PUT', + { + body: JSON.stringify(data), + }, + ); + + if (!response.ok) { + const errorBody = await this.safeReadErrorBody(response); + throw new Error( + `Unable to set account data '${type}' for ${this.access.userId}: status ${response.status} - ${JSON.stringify(errorBody)}`, + ); + } + } + + private async safeReadErrorBody(response: Response): Promise { + try { + const text = await response.text(); + if (!text) { + return undefined; + } + try { + return JSON.parse(text) as unknown; + } catch { + return text; + } + } catch { + return ''; + } + } + + async getOpenIdToken(): Promise< + | { + access_token: string; + expires_in: number; + matrix_server_name: string; + token_type: string; + } + | undefined + > { + if (!this.access) { + throw new Error('Must be logged in to get OpenID token'); + } + const response = await this.request( + `_matrix/client/v3/user/${encodeURIComponent(this.access.userId)}/openid/request_token`, + 'POST', + { body: '{}' }, + ); + if (!response.ok) { + return undefined; + } + return response.json() as Promise<{ + access_token: string; + expires_in: number; + matrix_server_name: string; + token_type: string; + }>; + } +} + +function uint8ArrayToHex(uint8: Uint8Array): string { + return Array.from(uint8) + .map((i) => i.toString(16).padStart(2, '0')) + .join(''); +} + +function getMatrixUsername(userId: string): string { + return userId.replace(/^@/, '').replace(/:.*$/, ''); +} + +export async function passwordFromSeed( + username: string, + seed: string, +): Promise { + const hash = new Sha256(); + const cleanUsername = getMatrixUsername(username); + hash.update(cleanUsername); + hash.update(seed); + return uint8ArrayToHex(await hash.digest()); +} diff --git a/packages/boxel-cli/src/lib/realm-auth-client.ts b/packages/boxel-cli/src/lib/realm-auth-client.ts new file mode 100644 index 0000000000..95b78ce078 --- /dev/null +++ b/packages/boxel-cli/src/lib/realm-auth-client.ts @@ -0,0 +1,116 @@ +import type { MatrixClient } from './matrix-client.js'; + +export interface JWTPayload { + iat: number; + exp: number; + user: string; + realm: string; + permissions: string[]; +} + +const MAX_ATTEMPTS = 3; +const BACK_OFF_MS = 1000; + +export class RealmAuthClient { + private _jwt: string | undefined; + + constructor( + private realmURL: URL, + private matrixClient: MatrixClient, + ) {} + + get jwt(): string | undefined { + return this._jwt; + } + + async getJWT(): Promise { + const tokenRefreshLeadTimeSeconds = 60; + + if (!this._jwt) { + this._jwt = await this.createRealmSession(); + return this._jwt; + } + + const jwtData = JSON.parse(atob(this._jwt.split('.')[1])) as JWTPayload; + const nowSeconds = Math.floor(Date.now() / 1000); + + if (jwtData.exp - tokenRefreshLeadTimeSeconds < nowSeconds) { + this._jwt = await this.createRealmSession(); + return this._jwt; + } + + return this._jwt; + } + + private async createRealmSession(): Promise { + if (!this.matrixClient.isLoggedIn()) { + throw new Error( + 'Must be logged in to matrix before a realm session can be created', + ); + } + + const initialResponse = await this.initiateSessionRequest(); + const jwt = initialResponse.headers.get('Authorization'); + + if (!jwt) { + throw new Error( + "Expected 'Authorization' header in response to POST session but it was missing", + ); + } + + const [, payload] = jwt.split('.'); + const jwtBody = JSON.parse(atob(payload)) as { sessionRoom?: string }; + const { sessionRoom } = jwtBody; + + if (sessionRoom) { + const { joined_rooms: rooms } = await this.matrixClient.getJoinedRooms(); + if (!rooms.includes(sessionRoom)) { + await this.matrixClient.joinRoom(sessionRoom); + } + } + + return jwt; + } + + private async initiateSessionRequest(): Promise { + const userId = this.matrixClient.getUserId(); + if (!userId) { + throw new Error('userId is undefined'); + } + + const openAccessToken = await this.matrixClient.getOpenIdToken(); + if (!openAccessToken) { + throw new Error('Failed to fetch OpenID token from matrix'); + } + + return this.withRetries(() => + fetch(`${this.realmURL.href}_session`, { + method: 'POST', + headers: { + Accept: 'application/json', + }, + body: JSON.stringify(openAccessToken), + }), + ); + } + + private async withRetries( + fetchFn: () => Promise, + ): Promise { + let attempt = 0; + + for (;;) { + const response = await fetchFn(); + + if (response.status === 500 && ++attempt <= MAX_ATTEMPTS) { + await this.delay(attempt * BACK_OFF_MS); + } else { + return response; + } + } + } + + private delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/packages/boxel-cli/src/lib/realm-sync-base.ts b/packages/boxel-cli/src/lib/realm-sync-base.ts new file mode 100644 index 0000000000..96c00338c9 --- /dev/null +++ b/packages/boxel-cli/src/lib/realm-sync-base.ts @@ -0,0 +1,593 @@ +import { MatrixClient, passwordFromSeed } from './matrix-client.js'; +import { RealmAuthClient } from './realm-auth-client.js'; +import * as fs from 'fs'; +import * as path from 'path'; +import ignoreModule from 'ignore'; + +const ignore = (ignoreModule as any).default || ignoreModule; +type Ignore = ReturnType; + +// Files that must never be pushed, deleted, or overwritten on the server via CLI. +export const PROTECTED_FILES = new Set(['.realm.json']); + +export function isProtectedFile(relativePath: string): boolean { + const normalizedPath = relativePath.replace(/\\/g, '/').replace(/^\/+/, ''); + return PROTECTED_FILES.has(normalizedPath); +} + +export const SupportedMimeType = { + CardSource: 'application/vnd.card+source', + DirectoryListing: 'application/vnd.api+json', + Mtimes: 'application/vnd.api+json', +} as const; + +export interface SyncOptions { + workspaceUrl: string; + localDir: string; + dryRun?: boolean; +} + +export abstract class RealmSyncBase { + protected matrixClient: MatrixClient; + protected realmAuthClient: RealmAuthClient; + protected normalizedRealmUrl: string; + private ignoreCache = new Map(); + + constructor( + protected options: SyncOptions, + matrixUrl: string, + username: string, + password: string, + ) { + this.matrixClient = new MatrixClient({ + matrixURL: new URL(matrixUrl), + username, + password, + }); + + this.normalizedRealmUrl = this.normalizeRealmUrl(options.workspaceUrl); + + this.realmAuthClient = new RealmAuthClient( + new URL(this.normalizedRealmUrl), + this.matrixClient, + ); + } + + async initialize(): Promise { + console.log('Logging into Matrix...'); + await this.matrixClient.login(); + console.log('Matrix login successful'); + } + + private normalizeRealmUrl(url: string): string { + try { + const urlObj = new URL(url); + + const pathPart = urlObj.pathname; + const lastSegment = pathPart.split('/').filter(Boolean).pop() || ''; + + if (lastSegment.includes('.')) { + console.warn( + `Warning: "${url}" looks like a file URL, not a realm URL.` + + `\n Realm URLs should point to a directory (e.g., ${urlObj.origin}${pathPart.replace(/\/[^/]*\.[^/]*$/, '/')})`, + ); + } else if (!url.endsWith('/')) { + console.warn( + `Warning: Realm URL should end with a trailing slash.` + + `\n Did you mean "${url}/"?`, + ); + } + + return urlObj.href.replace(/\/+$/, '') + '/'; + } catch { + throw new Error(`Invalid workspace URL: ${url}`); + } + } + + protected buildDirectoryUrl(dir = ''): string { + if (!dir) { + return this.normalizedRealmUrl; + } + const cleanDir = dir.replace(/^\/+|\/+$/g, ''); + return `${this.normalizedRealmUrl}${cleanDir}/`; + } + + protected buildFileUrl(relativePath: string): string { + const cleanPath = relativePath.replace(/^\/+/, ''); + return `${this.normalizedRealmUrl}${cleanPath}`; + } + + protected async getRemoteFileList(dir = ''): Promise> { + const files = new Map(); + + try { + const url = this.buildDirectoryUrl(dir); + const jwt = await this.realmAuthClient.getJWT(); + + const response = await fetch(url, { + headers: { + Accept: 'application/vnd.api+json', + Authorization: jwt, + }, + }); + + if (!response.ok) { + if (response.status === 404) { + return files; + } + if (response.status === 401 || response.status === 403) { + throw new Error( + `Authentication failed (${response.status}): Cannot access workspace. Check your Matrix credentials and workspace permissions.`, + ); + } + throw new Error( + `Failed to get directory listing: ${response.status} ${response.statusText}`, + ); + } + + const data = (await response.json()) as { + data?: { + relationships?: Record; + }; + }; + + if (data.data && data.data.relationships) { + for (const [name, info] of Object.entries(data.data.relationships)) { + const entry = info as { meta: { kind: string } }; + const isFile = entry.meta.kind === 'file'; + const entryPath = dir ? path.posix.join(dir, name) : name; + + if (isFile) { + if (!this.shouldIgnoreRemoteFile(entryPath)) { + files.set(entryPath, true); + } + } else { + const subdirFiles = await this.getRemoteFileList(entryPath); + for (const [subPath, isFileEntry] of subdirFiles) { + files.set(subPath, isFileEntry); + } + } + } + } + } catch (error) { + if (error instanceof Error) { + if ( + error.message.includes('Authentication failed') || + error.message.includes('Cannot access workspace') || + error.message.includes('401') || + error.message.includes('403') + ) { + throw error; + } + } + console.error(`Error reading remote directory ${dir}:`, error); + throw error; + } + + return files; + } + + protected async getRemoteMtimes(): Promise> { + const mtimes = new Map(); + + try { + const url = `${this.normalizedRealmUrl}_mtimes`; + const jwt = await this.realmAuthClient.getJWT(); + + const response = await fetch(url, { + headers: { + Accept: SupportedMimeType.Mtimes, + Authorization: jwt, + }, + }); + + if (!response.ok) { + if (response.status === 404) { + console.log( + 'Note: _mtimes endpoint not available, will upload all files', + ); + return mtimes; + } + throw new Error( + `Failed to get mtimes: ${response.status} ${response.statusText}`, + ); + } + + const data = (await response.json()) as { + data?: { + attributes?: { + mtimes?: Record; + }; + }; + }; + + if (data.data?.attributes?.mtimes) { + const remoteMtimeEntries = Object.entries(data.data.attributes.mtimes); + if (process.env.DEBUG) { + console.log( + `Remote mtimes received: ${remoteMtimeEntries.length} entries`, + ); + if (remoteMtimeEntries.length > 0) { + console.log( + `Sample: ${remoteMtimeEntries[0][0]} = ${remoteMtimeEntries[0][1]}`, + ); + } + } + for (const [fileUrl, mtime] of remoteMtimeEntries) { + const relativePath = fileUrl.replace(this.normalizedRealmUrl, ''); + if (!this.shouldIgnoreRemoteFile(relativePath)) { + mtimes.set(relativePath, mtime); + } + } + } else if (process.env.DEBUG) { + console.log( + 'No mtimes in response:', + JSON.stringify(data).slice(0, 200), + ); + } + } catch (error) { + console.warn( + 'Could not fetch remote mtimes, will upload all files:', + error, + ); + } + + return mtimes; + } + + protected async getLocalFileListWithMtimes( + dir = '', + ): Promise> { + const files = new Map(); + const fullDir = path.join(this.options.localDir, dir); + + if (!fs.existsSync(fullDir)) { + return files; + } + + const entries = fs.readdirSync(fullDir); + + for (const entry of entries) { + const fullPath = path.join(fullDir, entry); + const relativePath = dir ? path.posix.join(dir, entry) : entry; + const stats = fs.statSync(fullPath); + + if (this.shouldIgnoreFile(relativePath, fullPath)) { + continue; + } + + if (stats.isFile()) { + files.set(relativePath, { + path: fullPath, + mtime: stats.mtimeMs, + }); + } else if (stats.isDirectory()) { + const subdirFiles = await this.getLocalFileListWithMtimes(relativePath); + for (const [subPath, fileInfo] of subdirFiles) { + files.set(subPath, fileInfo); + } + } + } + + return files; + } + + protected async getLocalFileList(dir = ''): Promise> { + const files = new Map(); + const fullDir = path.join(this.options.localDir, dir); + + if (!fs.existsSync(fullDir)) { + return files; + } + + const entries = fs.readdirSync(fullDir); + + for (const entry of entries) { + const fullPath = path.join(fullDir, entry); + const relativePath = dir ? path.posix.join(dir, entry) : entry; + const stats = fs.statSync(fullPath); + + if (this.shouldIgnoreFile(relativePath, fullPath)) { + continue; + } + + if (stats.isFile()) { + files.set(relativePath, fullPath); + } else if (stats.isDirectory()) { + const subdirFiles = await this.getLocalFileList(relativePath); + for (const [subPath, fullSubPath] of subdirFiles) { + files.set(subPath, fullSubPath); + } + } + } + + return files; + } + + protected async uploadFile( + relativePath: string, + localPath: string, + ): Promise { + if (isProtectedFile(relativePath)) { + console.log(` Skipped (protected): ${relativePath}`); + return; + } + + console.log(`Uploading: ${relativePath}`); + + if (this.options.dryRun) { + console.log(`[DRY RUN] Would upload ${relativePath}`); + return; + } + + const content = fs.readFileSync(localPath, 'utf8'); + const url = this.buildFileUrl(relativePath); + const jwt = await this.realmAuthClient.getJWT(); + + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'text/plain;charset=UTF-8', + Authorization: jwt, + Accept: SupportedMimeType.CardSource, + }, + body: content, + }); + + if (!response.ok) { + throw new Error( + `Failed to upload: ${response.status} ${response.statusText}`, + ); + } + + console.log(` Uploaded: ${relativePath}`); + } + + protected async downloadFile( + relativePath: string, + localPath: string, + ): Promise { + console.log(`Downloading: ${relativePath}`); + + if (this.options.dryRun) { + console.log(`[DRY RUN] Would download ${relativePath}`); + return; + } + + const url = this.buildFileUrl(relativePath); + const jwt = await this.realmAuthClient.getJWT(); + + const response = await fetch(url, { + headers: { + Authorization: jwt, + Accept: SupportedMimeType.CardSource, + }, + }); + + if (!response.ok) { + throw new Error( + `Failed to download: ${response.status} ${response.statusText}`, + ); + } + + const content = await response.text(); + + const localDir = path.dirname(localPath); + if (!fs.existsSync(localDir)) { + fs.mkdirSync(localDir, { recursive: true }); + } + + fs.writeFileSync(localPath, content, 'utf8'); + console.log(` Downloaded: ${relativePath}`); + } + + protected async deleteFile(relativePath: string): Promise { + if (isProtectedFile(relativePath)) { + console.log(` Skipped (protected): ${relativePath}`); + return; + } + + console.log(`Deleting remote: ${relativePath}`); + + if (this.options.dryRun) { + console.log(`[DRY RUN] Would delete ${relativePath}`); + return; + } + + const url = this.buildFileUrl(relativePath); + const jwt = await this.realmAuthClient.getJWT(); + + const response = await fetch(url, { + method: 'DELETE', + headers: { + Authorization: jwt, + Accept: SupportedMimeType.CardSource, + }, + }); + + if (!response.ok && response.status !== 404) { + throw new Error( + `Failed to delete: ${response.status} ${response.statusText}`, + ); + } + + console.log(` Deleted: ${relativePath}`); + } + + protected async deleteLocalFile(localPath: string): Promise { + console.log(`Deleting local: ${localPath}`); + + if (this.options.dryRun) { + console.log(`[DRY RUN] Would delete local file ${localPath}`); + return; + } + + if (fs.existsSync(localPath)) { + fs.unlinkSync(localPath); + console.log(` Deleted: ${localPath}`); + } + } + + private getIgnoreInstance(dirPath: string): Ignore { + if (this.ignoreCache.has(dirPath)) { + return this.ignoreCache.get(dirPath)!; + } + + const ig = ignore(); + let currentPath = dirPath; + const rootPath = this.options.localDir; + + while (currentPath.startsWith(rootPath)) { + const gitignorePath = path.join(currentPath, '.gitignore'); + if (fs.existsSync(gitignorePath)) { + try { + const gitignoreContent = fs.readFileSync(gitignorePath, 'utf8'); + ig.add(gitignoreContent); + } catch (error) { + console.warn( + `Warning: Could not read .gitignore file at ${gitignorePath}:`, + error, + ); + } + } + + const boxelignorePath = path.join(currentPath, '.boxelignore'); + if (fs.existsSync(boxelignorePath)) { + try { + const boxelignoreContent = fs.readFileSync(boxelignorePath, 'utf8'); + ig.add(boxelignoreContent); + } catch (error) { + console.warn( + `Warning: Could not read .boxelignore file at ${boxelignorePath}:`, + error, + ); + } + } + + const parentPath = path.dirname(currentPath); + if (parentPath === currentPath) break; + currentPath = parentPath; + } + + this.ignoreCache.set(dirPath, ig); + return ig; + } + + private shouldIgnoreFile(relativePath: string, fullPath: string): boolean { + const fileName = path.basename(relativePath); + + if (fileName === '.boxel-sync.json') { + return true; + } + + if (fileName.startsWith('.')) { + return true; + } + + const dirPath = path.dirname(fullPath); + const ig = this.getIgnoreInstance(dirPath); + const normalizedPath = relativePath.replace(/\\/g, '/'); + + return ig.ignores(normalizedPath); + } + + private shouldIgnoreRemoteFile(relativePath: string): boolean { + const fileName = path.basename(relativePath); + if (fileName.startsWith('.')) { + return true; + } + return false; + } + + abstract sync(): Promise; +} + +function deriveRealmUsername(workspaceUrl: string): string { + let url: URL; + try { + url = new URL(workspaceUrl); + } catch { + throw new Error(`Invalid workspace URL: ${workspaceUrl}`); + } + + const segments = url.pathname.split('/').filter(Boolean); + if (segments.length === 0) { + throw new Error( + `Cannot derive realm username from workspace URL (${workspaceUrl}). Please provide MATRIX_USERNAME`, + ); + } + + if (segments[0] === 'published') { + if (!segments[1]) { + throw new Error( + `Cannot derive published realm username from workspace URL (${workspaceUrl}). Missing published realm id.`, + ); + } + return `realm/published_${segments[1]}`; + } + + if (segments.length >= 2) { + return `realm/${segments[0]}_${segments[1]}`; + } + + return `${segments[0]}_realm`; +} + +export async function validateMatrixEnvVars(workspaceUrl: string): Promise<{ + matrixUrl: string; + username: string; + password: string; +}> { + const { getProfileManager } = await import('./profile-manager.js'); + const profileManager = getProfileManager(); + const credentials = await profileManager.getActiveCredentials(); + + if (credentials) { + return { + matrixUrl: credentials.matrixUrl, + username: credentials.username, + password: credentials.password, + }; + } + + const matrixUrl = process.env.MATRIX_URL; + const envUsername = process.env.MATRIX_USERNAME; + let password = process.env.MATRIX_PASSWORD; + const realmSecret = process.env.REALM_SECRET_SEED; + let username = envUsername; + + if (!matrixUrl) { + console.error('MATRIX_URL environment variable is required'); + console.error('Or run "boxel profile add" to create a profile.'); + process.exit(1); + } + + if (!username) { + if (!realmSecret) { + console.error( + 'Either MATRIX_USERNAME or REALM_SECRET_SEED environment variable is required', + ); + process.exit(1); + } + username = deriveRealmUsername(workspaceUrl); + console.log( + `Derived realm Matrix username '${username}' from workspace URL using REALM_SECRET_SEED`, + ); + } + + if (!password && realmSecret) { + password = await passwordFromSeed(username, realmSecret); + console.log( + 'Generated password from REALM_SECRET_SEED for realm user authentication', + ); + } + + if (!password) { + console.error( + 'Either MATRIX_PASSWORD or REALM_SECRET_SEED environment variable is required', + ); + process.exit(1); + } + + return { matrixUrl, username, password }; +} diff --git a/packages/boxel-cli/tests/commands/pull.test.ts b/packages/boxel-cli/tests/commands/pull.test.ts new file mode 100644 index 0000000000..3c6e133e8f --- /dev/null +++ b/packages/boxel-cli/tests/commands/pull.test.ts @@ -0,0 +1,236 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import { + createRealmState, + type RealmServerState, +} from '../helpers/mock-realm-server.js'; +import { createMockFetch } from '../helpers/mock-fetch.js'; +import { TEST_REALM_URL } from '../helpers/mock-credentials.js'; + +const mockCredentials = vi.hoisted(() => ({ + matrixUrl: 'https://matrix.test.local/', + username: 'testuser', + password: 'testpassword', +})); + +vi.mock('../../src/lib/realm-sync-base.js', async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + validateMatrixEnvVars: vi.fn().mockResolvedValue(mockCredentials), + }; +}); + +vi.mock('../../src/lib/checkpoint-manager.js', () => ({ + CheckpointManager: vi.fn().mockImplementation(() => ({ + createCheckpoint: vi.fn().mockReturnValue({ + shortHash: 'abc1234', + message: 'test checkpoint', + isMajor: false, + }), + })), +})); + +// @ts-expect-error vitest supports top-level await even in CJS mode +const { pullCommand } = await import('../../src/commands/pull.js'); + +describe('pull integration', () => { + let tmpDir: string; + let realmState: RealmServerState; + let originalFetch: typeof fetch; + + beforeEach(() => { + tmpDir = fs.mkdtempSync( + path.join(process.env.TMPDIR || '/tmp', 'boxel-pull-test-'), + ); + originalFetch = globalThis.fetch; + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.spyOn(process, 'exit').mockImplementation( + (code?: string | number | null | undefined) => { + throw new Error(`process.exit(${code})`); + }, + ); + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + vi.mocked(console.log).mockRestore?.(); + vi.mocked(console.error).mockRestore?.(); + vi.mocked(console.warn).mockRestore?.(); + vi.mocked(process.exit).mockRestore?.(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + function setupFetch( + state: RealmServerState, + onRequest?: (url: string, method: string) => void, + ) { + const result = createMockFetch({ realmState: state, onRequest }); + globalThis.fetch = result.mockFetch; + return result; + } + + it('pulls 3 files into empty directory', async () => { + realmState = createRealmState({ + 'BlogPost/hello.json': { + content: '{"data":{"attributes":{"title":"Hello"}}}', + }, + 'BlogPost/world.json': { + content: '{"data":{"attributes":{"title":"World"}}}', + }, + 'my-card.gts': { + content: 'export class MyCard extends CardDef {}', + }, + }); + setupFetch(realmState); + + const localDir = path.join(tmpDir, 'workspace'); + await pullCommand(TEST_REALM_URL, localDir, {}); + + expect(fs.existsSync(path.join(localDir, 'BlogPost', 'hello.json'))).toBe( + true, + ); + expect(fs.existsSync(path.join(localDir, 'BlogPost', 'world.json'))).toBe( + true, + ); + expect(fs.existsSync(path.join(localDir, 'my-card.gts'))).toBe(true); + + expect( + fs.readFileSync(path.join(localDir, 'BlogPost', 'hello.json'), 'utf-8'), + ).toBe('{"data":{"attributes":{"title":"Hello"}}}'); + expect(fs.readFileSync(path.join(localDir, 'my-card.gts'), 'utf-8')).toBe( + 'export class MyCard extends CardDef {}', + ); + }); + + it('preserves local-only files without --delete', async () => { + realmState = createRealmState({ + 'BlogPost/hello.json': { + content: '{"data":{"attributes":{"title":"Hello"}}}', + }, + }); + setupFetch(realmState); + + const localDir = path.join(tmpDir, 'workspace'); + fs.mkdirSync(path.join(localDir, 'Notes'), { recursive: true }); + fs.writeFileSync( + path.join(localDir, 'Notes', 'local-only.json'), + '{"local":"only"}', + ); + + await pullCommand(TEST_REALM_URL, localDir, {}); + + expect(fs.existsSync(path.join(localDir, 'BlogPost', 'hello.json'))).toBe( + true, + ); + expect(fs.existsSync(path.join(localDir, 'Notes', 'local-only.json'))).toBe( + true, + ); + }); + + it('with --delete removes local-only files', async () => { + realmState = createRealmState({ + 'BlogPost/hello.json': { + content: '{"data":{"attributes":{"title":"Hello"}}}', + }, + }); + setupFetch(realmState); + + const localDir = path.join(tmpDir, 'workspace'); + fs.mkdirSync(path.join(localDir, 'Notes'), { recursive: true }); + fs.writeFileSync( + path.join(localDir, 'Notes', 'local-only.json'), + '{"local":"only"}', + ); + + await pullCommand(TEST_REALM_URL, localDir, { delete: true }); + + expect(fs.existsSync(path.join(localDir, 'BlogPost', 'hello.json'))).toBe( + true, + ); + expect(fs.existsSync(path.join(localDir, 'Notes', 'local-only.json'))).toBe( + false, + ); + }); + + it('with --dry-run writes no files', async () => { + realmState = createRealmState({ + 'BlogPost/hello.json': { + content: '{"data":{"attributes":{"title":"Hello"}}}', + }, + }); + setupFetch(realmState); + + const localDir = path.join(tmpDir, 'workspace'); + fs.mkdirSync(localDir, { recursive: true }); + + await pullCommand(TEST_REALM_URL, localDir, { dryRun: true }); + + const entries = fs.readdirSync(localDir); + expect(entries).toEqual([]); + }); + + it('with one file returning 500 still downloads others', async () => { + realmState = createRealmState({ + 'BlogPost/good.json': { + content: '{"data":{"attributes":{"title":"Good"}}}', + }, + 'BlogPost/broken.json': { content: 'broken' }, + }); + realmState.failingPaths = new Set(['BlogPost/broken.json']); + setupFetch(realmState); + + const localDir = path.join(tmpDir, 'workspace'); + + await expect(pullCommand(TEST_REALM_URL, localDir, {})).rejects.toThrow( + /process\.exit/, + ); + + expect(fs.existsSync(path.join(localDir, 'BlogPost', 'good.json'))).toBe( + true, + ); + expect(fs.existsSync(path.join(localDir, 'BlogPost', 'broken.json'))).toBe( + false, + ); + }); + + it('pulls subdirectories recursively', async () => { + realmState = createRealmState({ + 'BlogPost/hello.json': { content: '{"title":"Hello"}' }, + 'BlogPost/drafts/draft1.json': { content: '{"title":"Draft"}' }, + 'Author/author1.json': { content: '{"name":"Author"}' }, + }); + setupFetch(realmState); + + const localDir = path.join(tmpDir, 'workspace'); + await pullCommand(TEST_REALM_URL, localDir, {}); + + expect(fs.existsSync(path.join(localDir, 'BlogPost', 'hello.json'))).toBe( + true, + ); + expect( + fs.existsSync(path.join(localDir, 'BlogPost', 'drafts', 'draft1.json')), + ).toBe(true); + expect(fs.existsSync(path.join(localDir, 'Author', 'author1.json'))).toBe( + true, + ); + }); + + it('pulls index.json', async () => { + realmState = createRealmState({ + 'index.json': { content: '{"data":{"id":"test"}}' }, + }); + setupFetch(realmState); + + const localDir = path.join(tmpDir, 'workspace'); + await pullCommand(TEST_REALM_URL, localDir, {}); + + expect(fs.existsSync(path.join(localDir, 'index.json'))).toBe(true); + expect(fs.readFileSync(path.join(localDir, 'index.json'), 'utf-8')).toBe( + '{"data":{"id":"test"}}', + ); + }); +}); diff --git a/packages/boxel-cli/tests/helpers/mock-credentials.ts b/packages/boxel-cli/tests/helpers/mock-credentials.ts new file mode 100644 index 0000000000..22edba7c17 --- /dev/null +++ b/packages/boxel-cli/tests/helpers/mock-credentials.ts @@ -0,0 +1,43 @@ +export const TEST_MATRIX_URL = 'https://matrix.test.local/'; +export const TEST_REALM_URL = 'https://realm.test.local/testuser/workspace/'; +export const TEST_USERNAME = 'testuser'; +export const TEST_PASSWORD = 'testpassword'; +export const TEST_USER_ID = '@testuser:test.local'; +export const TEST_ACCESS_TOKEN = 'test_access_token_abc123'; +export const TEST_DEVICE_ID = 'TESTDEVICE01'; +export const TEST_SESSION_ROOM = '!testroom:test.local'; + +export function createMockJWT( + overrides: Partial<{ + exp: number; + user: string; + realm: string; + permissions: string[]; + sessionRoom: string; + }> = {}, +): string { + const header = { alg: 'none', typ: 'JWT' }; + const payload = { + iat: Math.floor(Date.now() / 1000), + exp: overrides.exp ?? Math.floor(Date.now() / 1000) + 3600, + user: overrides.user ?? TEST_USER_ID, + realm: overrides.realm ?? TEST_REALM_URL, + permissions: overrides.permissions ?? ['read', 'write'], + sessionRoom: overrides.sessionRoom ?? TEST_SESSION_ROOM, + }; + const signature = 'mock_signature'; + + const encode = (obj: unknown) => + Buffer.from(JSON.stringify(obj)).toString('base64'); + + return `${encode(header)}.${encode(payload)}.${Buffer.from(signature).toString('base64')}`; +} + +export function createMockOpenIdToken() { + return { + access_token: 'openid_token_xyz', + expires_in: 3600, + matrix_server_name: 'test.local', + token_type: 'Bearer', + }; +} diff --git a/packages/boxel-cli/tests/helpers/mock-fetch.ts b/packages/boxel-cli/tests/helpers/mock-fetch.ts new file mode 100644 index 0000000000..746702b60c --- /dev/null +++ b/packages/boxel-cli/tests/helpers/mock-fetch.ts @@ -0,0 +1,99 @@ +import { TEST_MATRIX_URL, TEST_REALM_URL } from './mock-credentials.js'; +import { + handleMatrixRequest, + type MatrixServerState, +} from './mock-matrix-server.js'; +import { + handleRealmRequest, + type RealmServerState, +} from './mock-realm-server.js'; + +export interface FetchCall { + url: string; + method: string; + headers: Record; + body?: string; +} + +export interface MockFetchOptions { + matrixState?: MatrixServerState; + realmState: RealmServerState; + matrixUrl?: string; + realmUrl?: string; + onRequest?: (url: string, method: string) => void; +} + +export function createMockFetch(options: MockFetchOptions): { + mockFetch: typeof fetch; + calls: FetchCall[]; +} { + const { + matrixState = {}, + realmState, + matrixUrl = TEST_MATRIX_URL, + realmUrl = TEST_REALM_URL, + onRequest, + } = options; + + const calls: FetchCall[] = []; + + const mockFetch = async ( + input: string | URL | Request, + init?: RequestInit, + ): Promise => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : input.url; + const method = init?.method || 'GET'; + const rawHeaders = init?.headers || {}; + const headers: Record = {}; + + if (rawHeaders instanceof Headers) { + rawHeaders.forEach((v, k) => { + headers[k] = v; + }); + } else if (Array.isArray(rawHeaders)) { + for (const [k, v] of rawHeaders) { + headers[k] = v; + } + } else { + Object.assign(headers, rawHeaders); + } + + const body = init?.body ? String(init.body) : undefined; + + calls.push({ url, method, headers, body }); + + if (onRequest) { + onRequest(url, method); + } + + if (url.startsWith(matrixUrl)) { + const response = handleMatrixRequest(url, method, body, matrixState); + if (response) return response; + } + + if (url.startsWith(realmUrl)) { + const response = handleRealmRequest( + url, + method, + body, + headers, + realmState, + realmUrl, + ); + if (response) return response; + } + + console.warn(`[mock-fetch] Unmatched request: ${method} ${url}`); + return new Response(JSON.stringify({ error: 'Not Found' }), { + status: 404, + headers: { 'Content-Type': 'application/json' }, + }); + }; + + return { mockFetch: mockFetch as typeof fetch, calls }; +} diff --git a/packages/boxel-cli/tests/helpers/mock-matrix-server.ts b/packages/boxel-cli/tests/helpers/mock-matrix-server.ts new file mode 100644 index 0000000000..0a524a3dea --- /dev/null +++ b/packages/boxel-cli/tests/helpers/mock-matrix-server.ts @@ -0,0 +1,69 @@ +import { + TEST_USER_ID, + TEST_ACCESS_TOKEN, + TEST_DEVICE_ID, + createMockOpenIdToken, +} from './mock-credentials.js'; + +export interface MatrixServerState { + loginShouldFail?: boolean; + joinedRooms?: string[]; +} + +function jsonResponse( + body: unknown, + status = 200, + headers?: Record, +): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json', ...headers }, + }); +} + +export function handleMatrixRequest( + url: string, + method: string, + _body: string | undefined, + state: MatrixServerState, +): Response | null { + const urlPath = new URL(url).pathname; + + if (urlPath === '/_matrix/client/v3/login' && method === 'POST') { + if (state.loginShouldFail) { + return jsonResponse( + { errcode: 'M_FORBIDDEN', error: 'Invalid password' }, + 403, + ); + } + return jsonResponse({ + access_token: TEST_ACCESS_TOKEN, + device_id: TEST_DEVICE_ID, + user_id: TEST_USER_ID, + }); + } + + if (urlPath === '/_matrix/client/v3/joined_rooms' && method === 'GET') { + return jsonResponse({ + joined_rooms: state.joinedRooms || [], + }); + } + + if ( + urlPath.match(/^\/_matrix\/client\/v3\/rooms\/[^/]+\/join$/) && + method === 'POST' + ) { + return jsonResponse({}); + } + + if ( + urlPath.match( + /^\/_matrix\/client\/v3\/user\/[^/]+\/openid\/request_token$/, + ) && + method === 'POST' + ) { + return jsonResponse(createMockOpenIdToken()); + } + + return null; +} diff --git a/packages/boxel-cli/tests/helpers/mock-realm-server.ts b/packages/boxel-cli/tests/helpers/mock-realm-server.ts new file mode 100644 index 0000000000..9e929d6645 --- /dev/null +++ b/packages/boxel-cli/tests/helpers/mock-realm-server.ts @@ -0,0 +1,192 @@ +import { createMockJWT } from './mock-credentials.js'; + +export interface RealmFile { + path: string; + content: string; + mtime: number; +} + +export interface RealmServerState { + files: Map; + sessionShouldFail?: boolean; + failingPaths?: Set; +} + +export function createRealmState( + initialFiles?: Record, +): RealmServerState { + const files = new Map(); + if (initialFiles) { + for (const [filePath, info] of Object.entries(initialFiles)) { + files.set(filePath, { + path: filePath, + content: info.content, + mtime: info.mtime ?? Math.floor(Date.now() / 1000), + }); + } + } + return { files }; +} + +function jsonResponse( + body: unknown, + status = 200, + headers?: Record, +): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json', ...headers }, + }); +} + +function textResponse( + body: string, + status = 200, + headers?: Record, +): Response { + return new Response(body, { + status, + headers: { 'Content-Type': 'text/plain', ...headers }, + }); +} + +function buildDirectoryListing( + state: RealmServerState, + dirPrefix: string, + _realmUrl: string, +): Response { + const relationships: Record = {}; + const seenDirs = new Set(); + + for (const filePath of state.files.keys()) { + if (!filePath.startsWith(dirPrefix)) continue; + + const rest = filePath.slice(dirPrefix.length); + const slashIdx = rest.indexOf('/'); + + if (slashIdx === -1) { + relationships[rest] = { meta: { kind: 'file' } }; + } else { + const dirName = rest.slice(0, slashIdx); + if (!seenDirs.has(dirName)) { + seenDirs.add(dirName); + relationships[dirName] = { meta: { kind: 'directory' } }; + } + } + } + + return jsonResponse({ + data: { relationships }, + }); +} + +function buildMtimesResponse( + state: RealmServerState, + realmUrl: string, +): Response { + const mtimes: Record = {}; + for (const [filePath, file] of state.files) { + const basename = filePath.split('/').pop() || ''; + if (basename.startsWith('.') && basename !== '.realm.json') continue; + + mtimes[`${realmUrl}${filePath}`] = file.mtime; + } + + return jsonResponse({ + data: { + attributes: { mtimes }, + }, + }); +} + +export function handleRealmRequest( + url: string, + method: string, + body: string | undefined, + headers: Record, + state: RealmServerState, + realmUrl: string, +): Response | null { + if (!url.startsWith(realmUrl)) return null; + + const relativePath = url.slice(realmUrl.length); + + if (relativePath === '_session' && method === 'POST') { + if (state.sessionShouldFail) { + return jsonResponse({ error: 'Unauthorized' }, 403); + } + const jwt = createMockJWT(); + return new Response(JSON.stringify({}), { + status: 200, + headers: { + 'Content-Type': 'application/json', + Authorization: jwt, + }, + }); + } + + if (relativePath === '_mtimes' && method === 'GET') { + return buildMtimesResponse(state, realmUrl); + } + + if (relativePath.endsWith('/') && method === 'GET') { + const accept = headers['Accept'] || headers['accept'] || ''; + if (accept.includes('application/vnd.api+json')) { + return buildDirectoryListing( + state, + relativePath === '' ? '' : relativePath, + realmUrl, + ); + } + } + + if (relativePath === '' && method === 'GET') { + const accept = headers['Accept'] || headers['accept'] || ''; + if (accept.includes('application/vnd.api+json')) { + return buildDirectoryListing(state, '', realmUrl); + } + } + + if (method === 'HEAD') { + if (state.files.has(relativePath)) { + return new Response(null, { status: 200 }); + } + return new Response(null, { status: 404 }); + } + + if (method === 'GET' && !relativePath.endsWith('/')) { + if (state.failingPaths?.has(relativePath)) { + return jsonResponse({ error: 'Internal Server Error' }, 500); + } + + const file = state.files.get(relativePath); + if (file) { + return textResponse(file.content); + } + return jsonResponse({ error: 'Not Found' }, 404); + } + + if (method === 'POST' && !relativePath.startsWith('_')) { + if (state.failingPaths?.has(relativePath)) { + return jsonResponse({ error: 'Internal Server Error' }, 500); + } + + state.files.set(relativePath, { + path: relativePath, + content: body || '', + mtime: Math.floor(Date.now() / 1000), + }); + return new Response(null, { status: 204 }); + } + + if (method === 'DELETE') { + state.files.delete(relativePath); + return new Response(null, { status: 204 }); + } + + if (method === 'QUERY') { + return jsonResponse({ data: [] }); + } + + return null; +}