diff --git a/src/content/__tests__/resolveAudioClip.test.ts b/src/content/__tests__/resolveAudioClip.test.ts new file mode 100644 index 0000000..c7fe8e1 --- /dev/null +++ b/src/content/__tests__/resolveAudioClip.test.ts @@ -0,0 +1,84 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { resolveAudioClip } from "../resolveAudioClip"; + +vi.mock("../../sandboxes/logStep", () => ({ + logStep: vi.fn(), +})); + +vi.mock("../selectAudioClip", () => ({ + selectAudioClip: vi.fn(), +})); + +vi.mock("../selectAttachedAudioClip", () => ({ + selectAttachedAudioClip: vi.fn(), +})); + +const { selectAudioClip } = await import("../selectAudioClip"); +const { selectAttachedAudioClip } = await import("../selectAttachedAudioClip"); + +const mockClip = { + songFilename: "song.mp3", + songTitle: "song", + songBuffer: Buffer.from("audio"), + startSeconds: 0, + durationSeconds: 15, + lyrics: { title: "song", fullLyrics: "", segments: [] }, + clipLyrics: "", + clipReason: "best", + clipMood: "happy", +}; + +describe("resolveAudioClip", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("uses selectAttachedAudioClip when songs contain a URL", async () => { + vi.mocked(selectAttachedAudioClip).mockResolvedValue(mockClip); + + const result = await resolveAudioClip({ + songs: ["hiccups", "https://example.com/track.mp3"], + lipsync: false, + githubRepo: "https://github.com/test/repo", + artistSlug: "artist", + }); + + expect(selectAttachedAudioClip).toHaveBeenCalledWith({ + audioUrl: "https://example.com/track.mp3", + lipsync: false, + }); + expect(selectAudioClip).not.toHaveBeenCalled(); + expect(result).toBe(mockClip); + }); + + it("uses selectAudioClip when songs are all slugs", async () => { + vi.mocked(selectAudioClip).mockResolvedValue(mockClip); + + const payload = { + songs: ["hiccups", "adhd"], + lipsync: true, + githubRepo: "https://github.com/test/repo", + artistSlug: "artist", + }; + const result = await resolveAudioClip(payload); + + expect(selectAudioClip).toHaveBeenCalledWith(payload); + expect(selectAttachedAudioClip).not.toHaveBeenCalled(); + expect(result).toBe(mockClip); + }); + + it("uses selectAudioClip when songs is undefined", async () => { + vi.mocked(selectAudioClip).mockResolvedValue(mockClip); + + const payload = { + songs: undefined, + lipsync: false, + githubRepo: "https://github.com/test/repo", + artistSlug: "artist", + }; + const result = await resolveAudioClip(payload); + + expect(selectAudioClip).toHaveBeenCalledWith(payload); + expect(result).toBe(mockClip); + }); +}); diff --git a/src/content/__tests__/resolveFaceGuide.test.ts b/src/content/__tests__/resolveFaceGuide.test.ts new file mode 100644 index 0000000..e603a60 --- /dev/null +++ b/src/content/__tests__/resolveFaceGuide.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { resolveFaceGuide } from "../resolveFaceGuide"; + +vi.mock("../../sandboxes/logStep", () => ({ + logStep: vi.fn(), +})); + +vi.mock("../fetchImageFromUrl", () => ({ + fetchImageFromUrl: vi.fn(), +})); + +vi.mock("../fetchGithubFile", () => ({ + fetchGithubFile: vi.fn(), +})); + +vi.mock("@fal-ai/client", () => ({ + fal: { storage: { upload: vi.fn() } }, +})); + +const { fetchImageFromUrl } = await import("../fetchImageFromUrl"); +const { fetchGithubFile } = await import("../fetchGithubFile"); +const { fal } = await import("@fal-ai/client"); + +describe("resolveFaceGuide", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("returns null when template does not use face guide", async () => { + const result = await resolveFaceGuide({ + usesFaceGuide: false, + images: ["https://example.com/face.png"], + githubRepo: "https://github.com/test/repo", + artistSlug: "artist", + }); + + expect(result).toBeNull(); + expect(fetchImageFromUrl).not.toHaveBeenCalled(); + }); + + it("uses fetchImageFromUrl when images array has entries", async () => { + vi.mocked(fetchImageFromUrl).mockResolvedValue("https://fal.ai/uploaded.png"); + + const result = await resolveFaceGuide({ + usesFaceGuide: true, + images: ["https://example.com/face.png"], + githubRepo: "https://github.com/test/repo", + artistSlug: "artist", + }); + + expect(fetchImageFromUrl).toHaveBeenCalledWith("https://example.com/face.png"); + expect(result).toBe("https://fal.ai/uploaded.png"); + }); + + it("fetches from GitHub when no images provided", async () => { + const buffer = Buffer.from("image-data"); + vi.mocked(fetchGithubFile).mockResolvedValue(buffer); + vi.mocked(fal.storage.upload).mockResolvedValue("https://fal.ai/github.png"); + + const result = await resolveFaceGuide({ + usesFaceGuide: true, + images: undefined, + githubRepo: "https://github.com/test/repo", + artistSlug: "artist", + }); + + expect(fetchGithubFile).toHaveBeenCalledWith( + "https://github.com/test/repo", + "artists/artist/context/images/face-guide.png", + ); + expect(result).toBe("https://fal.ai/github.png"); + }); + + it("throws when GitHub face-guide is not found", async () => { + vi.mocked(fetchGithubFile).mockResolvedValue(null); + + await expect( + resolveFaceGuide({ + usesFaceGuide: true, + images: undefined, + githubRepo: "https://github.com/test/repo", + artistSlug: "artist", + }), + ).rejects.toThrow("face-guide.png not found"); + }); +}); diff --git a/src/content/__tests__/selectAttachedAudioClip.test.ts b/src/content/__tests__/selectAttachedAudioClip.test.ts new file mode 100644 index 0000000..ed2c357 --- /dev/null +++ b/src/content/__tests__/selectAttachedAudioClip.test.ts @@ -0,0 +1,153 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { selectAttachedAudioClip } from "../selectAttachedAudioClip"; + +vi.mock("../../sandboxes/logStep", () => ({ + logStep: vi.fn(), +})); + +vi.mock("../transcribeSong", () => ({ + transcribeSong: vi.fn(), +})); + +vi.mock("../analyzeClips", () => ({ + analyzeClips: vi.fn(), +})); + +vi.mock("../defaultPipelineConfig", () => ({ + DEFAULT_PIPELINE_CONFIG: { clipDuration: 15 }, +})); + +const { transcribeSong } = await import("../transcribeSong"); +const { analyzeClips } = await import("../analyzeClips"); + +describe("selectAttachedAudioClip", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(new ArrayBuffer(100)), + }), + ); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("downloads audio from the provided URL", async () => { + vi.mocked(transcribeSong).mockResolvedValue({ + title: "my-song", + fullLyrics: "hello world", + segments: [{ start: 0, end: 5, text: "hello world" }], + }); + vi.mocked(analyzeClips).mockResolvedValue([ + { startSeconds: 0, lyrics: "hello", reason: "good", mood: "happy", hasLyrics: true }, + ]); + + await selectAttachedAudioClip({ + audioUrl: "https://blob.vercel-storage.com/song.mp3", + lipsync: false, + }); + + expect(fetch).toHaveBeenCalledWith("https://blob.vercel-storage.com/song.mp3"); + }); + + it("throws when download fails", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ ok: false, status: 404, statusText: "Not Found" }), + ); + + await expect( + selectAttachedAudioClip({ + audioUrl: "https://example.com/missing.mp3", + lipsync: false, + }), + ).rejects.toThrow("Failed to download attached audio: 404 Not Found"); + }); + + it("derives filename from URL path", async () => { + vi.mocked(transcribeSong).mockResolvedValue({ + title: "my-track", + fullLyrics: "", + segments: [], + }); + vi.mocked(analyzeClips).mockResolvedValue([]); + + const result = await selectAttachedAudioClip({ + audioUrl: "https://blob.vercel-storage.com/content-attachments/audio/my-track.mp3", + lipsync: false, + }); + + expect(result.songFilename).toBe("my-track.mp3"); + expect(result.songTitle).toBe("my-track"); + }); + + it("transcribes the downloaded audio", async () => { + vi.mocked(transcribeSong).mockResolvedValue({ + title: "song", + fullLyrics: "lyrics here", + segments: [{ start: 0, end: 10, text: "lyrics here" }], + }); + vi.mocked(analyzeClips).mockResolvedValue([ + { startSeconds: 0, lyrics: "lyrics", reason: "best", mood: "chill", hasLyrics: true }, + ]); + + const result = await selectAttachedAudioClip({ + audioUrl: "https://blob.vercel-storage.com/song.mp3", + lipsync: false, + }); + + expect(transcribeSong).toHaveBeenCalledWith(expect.any(Buffer), "song.mp3"); + expect(result.lyrics.fullLyrics).toBe("lyrics here"); + }); + + it("prefers clips with lyrics when lipsync is true", async () => { + vi.mocked(transcribeSong).mockResolvedValue({ + title: "song", + fullLyrics: "", + segments: [], + }); + vi.mocked(analyzeClips).mockResolvedValue([ + { startSeconds: 0, lyrics: "", reason: "instrumental", mood: "chill", hasLyrics: false }, + { startSeconds: 30, lyrics: "words", reason: "vocal", mood: "happy", hasLyrics: true }, + ]); + + const result = await selectAttachedAudioClip({ + audioUrl: "https://blob.vercel-storage.com/song.mp3", + lipsync: true, + }); + + expect(result.startSeconds).toBe(30); + }); + + it("returns SelectedAudioClip interface shape", async () => { + vi.mocked(transcribeSong).mockResolvedValue({ + title: "song", + fullLyrics: "full lyrics", + segments: [{ start: 0, end: 10, text: "clip text" }], + }); + vi.mocked(analyzeClips).mockResolvedValue([ + { startSeconds: 0, lyrics: "clip", reason: "best clip", mood: "energetic", hasLyrics: true }, + ]); + + const result = await selectAttachedAudioClip({ + audioUrl: "https://blob.vercel-storage.com/song.mp3", + lipsync: false, + }); + + expect(result).toMatchObject({ + songFilename: expect.any(String), + songTitle: expect.any(String), + songBuffer: expect.any(Buffer), + startSeconds: expect.any(Number), + durationSeconds: 15, + lyrics: expect.any(Object), + clipLyrics: expect.any(String), + clipReason: expect.any(String), + clipMood: expect.any(String), + }); + }); +}); diff --git a/src/content/fetchImageFromUrl.ts b/src/content/fetchImageFromUrl.ts new file mode 100644 index 0000000..daa7a51 --- /dev/null +++ b/src/content/fetchImageFromUrl.ts @@ -0,0 +1,28 @@ +import { fal } from "@fal-ai/client"; +import { logStep } from "../sandboxes/logStep"; + +/** + * Downloads an image from a public URL and uploads it to fal.ai storage. + * + * @param imageUrl - Public URL of the image + * @returns fal.ai storage URL for the uploaded image + */ +export async function fetchImageFromUrl(imageUrl: string): Promise { + logStep("Downloading image from URL"); + const response = await fetch(imageUrl); + if (!response.ok) { + throw new Error(`Failed to download image: ${response.status}`); + } + const imageBuffer = Buffer.from(await response.arrayBuffer()); + + logStep("Uploading image to fal.ai storage", true, { + sizeBytes: imageBuffer.byteLength, + }); + const contentType = response.headers.get("content-type") || "image/png"; + const originalName = new URL(imageUrl).pathname.split("/").pop() || "image.png"; + const faceGuideFile = new File([new Uint8Array(imageBuffer)], originalName, { type: contentType }); + const falUrl = await fal.storage.upload(faceGuideFile); + + logStep("Image uploaded", false, { falUrl }); + return falUrl; +} diff --git a/src/content/resolveAudioClip.ts b/src/content/resolveAudioClip.ts new file mode 100644 index 0000000..9baad47 --- /dev/null +++ b/src/content/resolveAudioClip.ts @@ -0,0 +1,23 @@ +import { logStep } from "../sandboxes/logStep"; +import { selectAudioClip, type SelectedAudioClip } from "./selectAudioClip"; +import { selectAttachedAudioClip } from "./selectAttachedAudioClip"; +import type { CreateContentPayload } from "../schemas/contentCreationSchema"; + +/** + * Resolves the audio clip for the content pipeline. + * If any entry in songs is a URL, downloads and processes it directly. + * Otherwise delegates to selectAudioClip to pick from the artist's repo. + */ +export async function resolveAudioClip( + payload: Pick, +): Promise { + const songUrl = payload.songs?.find(s => s.startsWith("http")); + + if (songUrl) { + logStep("Using song URL from songs array"); + return selectAttachedAudioClip({ audioUrl: songUrl, lipsync: payload.lipsync }); + } + + logStep("Selecting audio clip from repo"); + return selectAudioClip(payload); +} diff --git a/src/content/resolveFaceGuide.ts b/src/content/resolveFaceGuide.ts new file mode 100644 index 0000000..ac6459f --- /dev/null +++ b/src/content/resolveFaceGuide.ts @@ -0,0 +1,47 @@ +import { fal } from "@fal-ai/client"; +import { logStep } from "../sandboxes/logStep"; +import { fetchImageFromUrl } from "./fetchImageFromUrl"; +import { fetchGithubFile } from "./fetchGithubFile"; + +/** + * Resolves the face guide URL for the content pipeline. + * Uses the first image from the images array if provided, + * otherwise fetches face-guide.png from the artist's GitHub repo. + * + * @returns fal.ai storage URL, or null if template doesn't use face guide + */ +export async function resolveFaceGuide({ + usesFaceGuide, + images, + githubRepo, + artistSlug, +}: { + usesFaceGuide: boolean; + images: string[] | undefined; + githubRepo: string; + artistSlug: string; +}): Promise { + if (!usesFaceGuide) return null; + + const imageUrl = images?.[0]; + if (imageUrl) { + return fetchImageFromUrl(imageUrl); + } + + logStep("Fetching face-guide from GitHub"); + const buffer = await fetchGithubFile( + githubRepo, + `artists/${artistSlug}/context/images/face-guide.png`, + ); + if (!buffer) { + throw new Error(`face-guide.png not found for artist ${artistSlug}`); + } + + logStep("Uploading face-guide to fal.ai storage", true, { + sizeBytes: buffer.byteLength, + }); + const file = new File([buffer], "face-guide.png", { type: "image/png" }); + const falUrl = await fal.storage.upload(file); + logStep("Face-guide uploaded", false, { faceGuideUrl: falUrl }); + return falUrl; +} diff --git a/src/content/selectAttachedAudioClip.ts b/src/content/selectAttachedAudioClip.ts new file mode 100644 index 0000000..c10d112 --- /dev/null +++ b/src/content/selectAttachedAudioClip.ts @@ -0,0 +1,89 @@ +import { logStep } from "../sandboxes/logStep"; +import { transcribeSong } from "./transcribeSong"; +import { analyzeClips, type SongClip } from "./analyzeClips"; +import { DEFAULT_PIPELINE_CONFIG } from "./defaultPipelineConfig"; +import type { SelectedAudioClip } from "./selectAudioClip"; + +/** + * Selects an audio clip from a user-attached audio file URL. + * Downloads the file, transcribes it, analyzes clips, and picks the best one. + * + * @param audioUrl - Public URL of the user-attached audio file + * @param lipsync - Whether to prefer clips with lyrics (for lipsync mode) + * @returns Selected audio clip with all metadata + */ +export async function selectAttachedAudioClip({ + audioUrl, + lipsync, +}: { + audioUrl: string; + lipsync: boolean; +}): Promise { + const clipDuration = DEFAULT_PIPELINE_CONFIG.clipDuration; + + // Download the attached audio + logStep("Downloading attached audio", { url: audioUrl }); + const response = await fetch(audioUrl); + if (!response.ok) { + throw new Error(`Failed to download attached audio: ${response.status} ${response.statusText}`); + } + const arrayBuffer = await response.arrayBuffer(); + const songBuffer = Buffer.from(arrayBuffer); + + // Derive filename from URL + const urlPath = new URL(audioUrl).pathname; + const songFilename = urlPath.split("/").pop() ?? "attached-audio.mp3"; + const songTitle = songFilename.replace(/\.(mp3|wav|m4a|ogg|aac)$/i, ""); + + logStep("Attached audio downloaded", { songTitle, sizeBytes: songBuffer.byteLength }); + + // Transcribe + const lyrics = await transcribeSong(songBuffer, songFilename); + + // Analyze clips + const clips = await analyzeClips(songTitle, lyrics); + + if (clips.length === 0) { + clips.push({ + startSeconds: 0, + lyrics: "", + reason: "fallback — no clips analyzed", + mood: "unknown", + hasLyrics: false, + }); + } + + let selectedClip: SongClip; + if (lipsync) { + const lyricsClips = clips.filter(c => c.hasLyrics !== false); + selectedClip = lyricsClips.length > 0 ? lyricsClips[0] : clips[0]; + } else { + selectedClip = clips[0]; + } + + // Get lyrics for the clip window + const clipEnd = selectedClip.startSeconds + clipDuration; + const clipSegments = lyrics.segments.filter( + s => s.start < clipEnd && s.end > selectedClip.startSeconds, + ); + const clipLyrics = clipSegments.map(s => s.text).join(" "); + + logStep("Attached audio clip selected", { + songTitle, + startSeconds: selectedClip.startSeconds, + clipLyrics: clipLyrics.slice(0, 80), + mood: selectedClip.mood, + }); + + return { + songFilename, + songTitle, + songBuffer, + startSeconds: selectedClip.startSeconds, + durationSeconds: clipDuration, + lyrics, + clipLyrics, + clipReason: selectedClip.reason, + clipMood: selectedClip.mood, + }; +} diff --git a/src/schemas/__tests__/contentCreationSchema.test.ts b/src/schemas/__tests__/contentCreationSchema.test.ts index 5ed44ca..1d1cbc4 100644 --- a/src/schemas/__tests__/contentCreationSchema.test.ts +++ b/src/schemas/__tests__/contentCreationSchema.test.ts @@ -38,5 +38,77 @@ describe("createContentPayloadSchema", () => { expect(result.success).toBe(false); }); + + it("accepts songs with a mix of slugs and URLs", () => { + const result = createContentPayloadSchema.safeParse({ + accountId: "acc_123", + artistSlug: "gatsby-grace", + template: "artist-caption-bedroom", + githubRepo: "https://github.com/recoupable/test-repo", + songs: ["hiccups", "https://example.com/unreleased-track.mp3"], + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.songs).toEqual(["hiccups", "https://example.com/unreleased-track.mp3"]); + } + }); + + it("accepts images as an array of URLs", () => { + const result = createContentPayloadSchema.safeParse({ + accountId: "acc_123", + artistSlug: "gatsby-grace", + template: "artist-caption-bedroom", + githubRepo: "https://github.com/recoupable/test-repo", + images: ["https://example.com/face.png"], + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.images).toEqual(["https://example.com/face.png"]); + } + }); + + it("rejects images with non-URL strings", () => { + const result = createContentPayloadSchema.safeParse({ + accountId: "acc_123", + artistSlug: "gatsby-grace", + template: "artist-caption-bedroom", + githubRepo: "https://github.com/recoupable/test-repo", + images: ["not-a-url"], + }); + + expect(result.success).toBe(false); + }); + + it("does not accept attachedAudioUrl (removed)", () => { + const result = createContentPayloadSchema.safeParse({ + accountId: "acc_123", + artistSlug: "gatsby-grace", + template: "artist-caption-bedroom", + githubRepo: "https://github.com/recoupable/test-repo", + attachedAudioUrl: "https://example.com/song.mp3", + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data).not.toHaveProperty("attachedAudioUrl"); + } + }); + + it("does not accept attachedImageUrl (removed)", () => { + const result = createContentPayloadSchema.safeParse({ + accountId: "acc_123", + artistSlug: "gatsby-grace", + template: "artist-caption-bedroom", + githubRepo: "https://github.com/recoupable/test-repo", + attachedImageUrl: "https://example.com/face.png", + }); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data).not.toHaveProperty("attachedImageUrl"); + } + }); }); diff --git a/src/schemas/contentCreationSchema.ts b/src/schemas/contentCreationSchema.ts index 9d31eaa..1a85f1d 100644 --- a/src/schemas/contentCreationSchema.ts +++ b/src/schemas/contentCreationSchema.ts @@ -14,8 +14,10 @@ export const createContentPayloadSchema = z.object({ upscale: z.boolean().default(false), /** GitHub repo URL so the task can fetch artist files (face-guide, songs). */ githubRepo: z.string().url("githubRepo must be a valid URL"), - /** Optional list of song slugs to restrict which songs the pipeline picks from. */ + /** Optional list of song slugs or public URLs. Slugs select from the artist's repo; URLs are downloaded directly. */ songs: z.array(z.string()).optional(), + /** Optional list of public image URLs to use as face guides instead of the artist's default face-guide.png. */ + images: z.array(z.string().url()).optional(), }); export type CreateContentPayload = z.infer; diff --git a/src/tasks/createContentTask.ts b/src/tasks/createContentTask.ts index ba30ffe..5309f63 100644 --- a/src/tasks/createContentTask.ts +++ b/src/tasks/createContentTask.ts @@ -8,7 +8,8 @@ import { generateContentVideo } from "../content/generateContentVideo"; import { generateAudioVideo } from "../content/generateAudioVideo"; import { upscaleImage } from "../content/upscaleImage"; import { upscaleVideo } from "../content/upscaleVideo"; -import { selectAudioClip } from "../content/selectAudioClip"; +import { resolveFaceGuide } from "../content/resolveFaceGuide"; +import { resolveAudioClip } from "../content/resolveAudioClip"; import { generateCaption } from "../content/generateCaption"; import { fetchArtistContext } from "../content/fetchArtistContext"; import { fetchAudienceContext } from "../content/fetchAudienceContext"; @@ -69,27 +70,15 @@ export const createContentTask = schemaTask({ const template = await loadTemplate(payload.template); // --- Step 2: Fetch face-guide (only if template uses it) --- - let faceGuideUrl: string | null = null; - if (template.usesFaceGuide) { - logStep("Fetching face-guide"); - const faceGuideBuffer = await fetchGithubFile( - payload.githubRepo, - `artists/${payload.artistSlug}/context/images/face-guide.png`, - ); - if (!faceGuideBuffer) { - throw new Error(`face-guide.png not found for artist ${payload.artistSlug}`); - } - logStep("Uploading face-guide to fal.ai storage", true, { - sizeBytes: faceGuideBuffer.byteLength, - }); - const faceGuideFile = new File([faceGuideBuffer], "face-guide.png", { type: "image/png" }); - faceGuideUrl = await fal.storage.upload(faceGuideFile); - logStep("Face-guide uploaded", false, { faceGuideUrl }); - } + const faceGuideUrl = await resolveFaceGuide({ + usesFaceGuide: template.usesFaceGuide, + images: payload.images, + githubRepo: payload.githubRepo, + artistSlug: payload.artistSlug, + }); // --- Step 3: Select audio clip --- - logStep("Selecting audio clip"); - const audioClip = await selectAudioClip(payload); + const audioClip = await resolveAudioClip(payload); // --- Step 4: Fetch artist/audience context --- logStep("Fetching artist context");