From 23daf16711a9c751e64ad95abb7be635e9207a75 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Thu, 4 Dec 2025 20:24:38 -0700
Subject: [PATCH 01/27] refactor: streamline Azure storage handling and update
 documentation

- Consolidated Azure storage container handling to use a single container defined in environment variables, removing support for multiple containers.
- Updated related code and tests to reflect the new single-container approach, ensuring consistency across the application.
- Enhanced documentation in INTERFACE.md to clarify storage architecture and operations, including retention management and URL generation.
- Added tests for retention setting and ensured short-lived URLs are consistently included in responses.
---
 .../cortex-file-handler/.env.test.azure.ci    |   2 +-
 .../.env.test.azure.sample                    |   2 +-
 .../cortex-file-handler/.env.test.gcs.ci      |   2 +-
 .../cortex-file-handler/.env.test.gcs.sample  |   2 +-
 helper-apps/cortex-file-handler/INTERFACE.md  |  68 ++-
 .../cortex-file-handler/src/blobHandler.js    | 126 ++---
 .../cortex-file-handler/src/constants.js      |  30 +-
 helper-apps/cortex-file-handler/src/index.js  | 205 ++++---
 helper-apps/cortex-file-handler/src/redis.js  |  84 ++-
 .../src/services/ConversionService.js         |   7 +-
 .../src/services/FileConversionService.js     |   5 +-
 .../services/storage/AzureStorageProvider.js  |  49 +-
 .../services/storage/LocalStorageProvider.js  |   7 +-
 .../src/services/storage/StorageFactory.js    |  27 +-
 .../src/services/storage/StorageService.js    | 225 ++++++--
 helper-apps/cortex-file-handler/src/start.js  |   7 +-
 .../tests/blobHandler.test.js                 |  39 +-
 .../tests/containerConversionFlow.test.js     | 225 +-------
 .../tests/containerNameParsing.test.js        | 226 +-------
 .../tests/containerParameterFlow.test.js      | 396 -------------
 .../tests/deleteOperations.test.js            |  22 +-
 .../tests/hashContainerScoping.test.js        | 415 --------------
 .../tests/setRetention.test.js                | 533 ++++++++++++++++++
 .../tests/storage/StorageFactory.test.js      |  79 +--
 .../tests/storage/StorageService.test.js      |  83 +--
 25 files changed, 1231 insertions(+), 1635 deletions(-)
 delete mode 100644 helper-apps/cortex-file-handler/tests/containerParameterFlow.test.js
 delete mode 100644 helper-apps/cortex-file-handler/tests/hashContainerScoping.test.js
 create mode 100644 helper-apps/cortex-file-handler/tests/setRetention.test.js

diff --git a/helper-apps/cortex-file-handler/.env.test.azure.ci b/helper-apps/cortex-file-handler/.env.test.azure.ci
index 35a54148..80dc0227 100644
--- a/helper-apps/cortex-file-handler/.env.test.azure.ci
+++ b/helper-apps/cortex-file-handler/.env.test.azure.ci
@@ -1,7 +1,7 @@
 # Test environment configuration for Azure tests
 REDIS_CONNECTION_STRING=redis://default:redispw@localhost:32768
 AZURE_STORAGE_CONNECTION_STRING=UseDevelopmentStorage=true
-AZURE_STORAGE_CONTAINER_NAME=default,test-container,test1,test2,test3,container1,container2,container3
+AZURE_STORAGE_CONTAINER_NAME=default
 NODE_ENV=test
 PORT=7072  # Different port for testing 
 MARKITDOWN_CONVERT_URL= #cortex-markitdown url
\ No newline at end of file
diff --git a/helper-apps/cortex-file-handler/.env.test.azure.sample b/helper-apps/cortex-file-handler/.env.test.azure.sample
index 35a54148..80dc0227 100644
--- a/helper-apps/cortex-file-handler/.env.test.azure.sample
+++ b/helper-apps/cortex-file-handler/.env.test.azure.sample
@@ -1,7 +1,7 @@
 # Test environment configuration for Azure tests
 REDIS_CONNECTION_STRING=redis://default:redispw@localhost:32768
 AZURE_STORAGE_CONNECTION_STRING=UseDevelopmentStorage=true
-AZURE_STORAGE_CONTAINER_NAME=default,test-container,test1,test2,test3,container1,container2,container3
+AZURE_STORAGE_CONTAINER_NAME=default
 NODE_ENV=test
 PORT=7072  # Different port for testing 
 MARKITDOWN_CONVERT_URL= #cortex-markitdown url
\ No newline at end of file
diff --git a/helper-apps/cortex-file-handler/.env.test.gcs.ci b/helper-apps/cortex-file-handler/.env.test.gcs.ci
index 7f05c160..fd78b43b 100644
--- a/helper-apps/cortex-file-handler/.env.test.gcs.ci
+++ b/helper-apps/cortex-file-handler/.env.test.gcs.ci
@@ -4,7 +4,7 @@ STORAGE_EMULATOR_HOST=http://localhost:4443
 GCP_SERVICE_ACCOUNT_KEY={"project_id":"test-project"}
 GCS_BUCKETNAME=cortextempfiles
 AZURE_STORAGE_CONNECTION_STRING=UseDevelopmentStorage=true
-AZURE_STORAGE_CONTAINER_NAME=default,test-container,test1,test2,test3,container1,container2,container3
+AZURE_STORAGE_CONTAINER_NAME=default
 NODE_ENV=test
 PORT=7072  # Different port for testing 
 MARKITDOWN_CONVERT_URL= #cortex-markitdown url
\ No newline at end of file
diff --git a/helper-apps/cortex-file-handler/.env.test.gcs.sample b/helper-apps/cortex-file-handler/.env.test.gcs.sample
index 7f05c160..fd78b43b 100644
--- a/helper-apps/cortex-file-handler/.env.test.gcs.sample
+++ b/helper-apps/cortex-file-handler/.env.test.gcs.sample
@@ -4,7 +4,7 @@ STORAGE_EMULATOR_HOST=http://localhost:4443
 GCP_SERVICE_ACCOUNT_KEY={"project_id":"test-project"}
 GCS_BUCKETNAME=cortextempfiles
 AZURE_STORAGE_CONNECTION_STRING=UseDevelopmentStorage=true
-AZURE_STORAGE_CONTAINER_NAME=default,test-container,test1,test2,test3,container1,container2,container3
+AZURE_STORAGE_CONTAINER_NAME=default
 NODE_ENV=test
 PORT=7072  # Different port for testing 
 MARKITDOWN_CONVERT_URL= #cortex-markitdown url
\ No newline at end of file
diff --git a/helper-apps/cortex-file-handler/INTERFACE.md b/helper-apps/cortex-file-handler/INTERFACE.md
index 7936c5dd..8983272c 100644
--- a/helper-apps/cortex-file-handler/INTERFACE.md
+++ b/helper-apps/cortex-file-handler/INTERFACE.md
@@ -4,6 +4,16 @@
 
 The Cortex File Handler is a service that processes files through various operations including uploading, downloading, chunking, and document processing. It supports multiple storage backends (Azure Blob Storage, Google Cloud Storage, and Local File System).
 
+## Storage Architecture
+
+The file handler uses a unified storage approach with Azure Blob Storage:
+- **Single Container**: All files are stored in a single Azure Blob Storage container
+- **Blob Index Tags**: Files are distinguished by blob index tags rather than separate containers
+  - `retention=temporary`: Files that will be automatically deleted after 30 days (default for all uploads)
+  - `retention=permanent`: Files that should be retained indefinitely
+- **Lifecycle Management**: Azure lifecycle management policies automatically delete temporary files after 30 days based on the blob index tag
+- **Set Retention Operation**: Setting a file's retention from temporary to permanent (or vice versa) simply updates the blob index tag (no copying between containers)
+
 ## Request Methods
 
 ### POST
@@ -20,7 +30,8 @@ The Cortex File Handler is a service that processes files through various operat
   - If hash is provided, stores file metadata in Redis
   - Returns upload result with file URLs
 - **Response**: Object containing:
-  - `url`: Primary storage URL
+  - `url`: Primary storage URL (with long-lived SAS token)
+  - `shortLivedUrl`: Short-lived URL (5-minute expiration, always included)
   - `gcs`: GCS URL (if GCS is configured)
   - `hash`: Hash value (if provided)
   - `message`: Success message
@@ -89,12 +100,49 @@ The Cortex File Handler is a service that processes files through various operat
 
 - **Purpose**: Remove files from storage
 - **Parameters** (can be in query string or request body):
-  - `requestId` (required): Unique identifier for the request
+  - `requestId` (optional): Unique identifier for the request (for multi-file deletion)
+  - `hash` (optional): Hash of the file to delete (for single-file deletion)
 - **Behavior**:
+  - Supports two deletion modes:
+    1. **By requestId**: Deletes all files associated with a requestId
+    2. **By hash**: Deletes a single file by its hash
   - Deletes file from primary storage (Azure or Local)
   - Deletes file from GCS if configured
+  - Removes file metadata from Redis
   - Returns deletion result
-- **Response**: Array of deleted file URLs
+- **Response**: 
+  - For requestId deletion: Array of deleted file URLs
+  - For hash deletion: Object containing deletion details with hash, filename, and deletion results
+
+### SET RETENTION (POST/PUT)
+
+- **Purpose**: Set the retention tag for a file (temporary or permanent)
+- **Parameters** (can be in query string or request body):
+  - `hash` (required): Hash of the file
+  - `retention` (required): Retention value - either `'temporary'` or `'permanent'`
+  - `setRetention` (optional): Set to `true` to trigger operation, or use `operation=setRetention` in query string
+- **Behavior**:
+  - Updates the blob index tag to the specified retention value
+  - No file copying is performed - the file stays in the same location
+  - Updates Redis map with new information including shortLivedUrl
+  - Also updates converted file tags if they exist
+  - Preserves file metadata (filename, hash, etc.)
+  - Generates new shortLivedUrl for the file
+- **Response**: Object containing:
+  - `hash`: File hash
+  - `filename`: Original filename
+  - `retention`: The retention value that was set
+  - `url`: Primary storage URL (same as before, file location unchanged)
+  - `shortLivedUrl`: New short-lived URL (5-minute expiration)
+  - `gcs`: GCS URL (if GCS is configured, unchanged)
+  - `converted`: Converted file info (if applicable)
+  - `message`: Success message
+- **Note**: 
+  - This is a simple tag update operation - no file copying occurs
+  - The file URL remains the same, only the blob index tag changes
+  - This operation is fast and idempotent
+  - Both the original file and any converted versions have their tags updated
+  - No locking is required since it's just a tag update
 
 ## Storage Configuration
 
@@ -159,6 +207,11 @@ The Cortex File Handler is a service that processes files through various operat
   - Files are stored with UUID-based names
   - Organized by requestId folders
   - Azure: Uses SAS tokens for access
+    - All files are stored in a single container (configured via `AZURE_STORAGE_CONTAINER_NAME` environment variable)
+    - Files are tagged with `retention=temporary` by default
+    - Files can be set to permanent using the `setRetention` operation (updates tag to `retention=permanent`)
+    - Lifecycle management automatically deletes temporary files after 30 days
+    - No container specification is supported - all files use the single configured container
   - Local: Served via HTTP on configured port
 - **GCS** (if configured):
   - Files stored with gs:// protocol URLs
@@ -169,6 +222,12 @@ The Cortex File Handler is a service that processes files through various operat
   - Used for caching remote file results
   - Tracks file access timestamps
   - Used for progress tracking
+  - Files are stored by hash directly (no container scoping)
+- **Short-Lived URLs**:
+  - All file operations now return a `shortLivedUrl` field
+  - Short-lived URLs expire after 5 minutes (configurable via `shortLivedMinutes`)
+  - Provides secure, time-limited access to files
+  - Always included in responses for consistency
 
 ## Cleanup
 
@@ -201,13 +260,14 @@ GET /file-handler?hash=abc123&checkHash=true&shortLivedMinutes=10
   "message": "File 'document.pdf' uploaded successfully.",
   "filename": "document.pdf",
   "url": "https://storage.blob.core.windows.net/container/file.pdf?original-sas-token",
+  "shortLivedUrl": "https://storage.blob.core.windows.net/container/file.pdf?sv=2023-11-03&se=2024-01-15T10%3A15%3A00Z&sr=b&sp=r&sig=...",
   "gcs": "gs://bucket/file.pdf",
   "hash": "abc123",
-  "shortLivedUrl": "https://storage.blob.core.windows.net/container/file.pdf?sv=2023-11-03&se=2024-01-15T10%3A15%3A00Z&sr=b&sp=r&sig=...",
   "expiresInMinutes": 5,
   "timestamp": "2024-01-15T10:10:00.000Z",
   "converted": {
     "url": "https://storage.blob.core.windows.net/container/converted.pdf",
+    "shortLivedUrl": "https://storage.blob.core.windows.net/container/converted.pdf?sv=2023-11-03&se=2024-01-15T10%3A15%3A00Z&sr=b&sp=r&sig=...",
     "gcs": "gs://bucket/converted.pdf"
   }
 }
diff --git a/helper-apps/cortex-file-handler/src/blobHandler.js b/helper-apps/cortex-file-handler/src/blobHandler.js
index 92109738..78384145 100644
--- a/helper-apps/cortex-file-handler/src/blobHandler.js
+++ b/helper-apps/cortex-file-handler/src/blobHandler.js
@@ -19,12 +19,9 @@ import { publicFolder, port, ipAddress } from "./start.js";
 import { 
   CONVERTED_EXTENSIONS, 
   AZURITE_ACCOUNT_NAME,
-  parseContainerNames,
-  getCurrentContainerNames,
-  AZURE_STORAGE_CONTAINER_NAMES,
   getDefaultContainerName,
   GCS_BUCKETNAME,
-  isValidContainerName
+  AZURE_STORAGE_CONTAINER_NAME
 } from "./constants.js";
 import { FileConversionService } from "./services/FileConversionService.js";
 import { StorageFactory } from "./services/storage/StorageFactory.js";
@@ -179,16 +176,10 @@ async function downloadFromGCS(gcsUrl, destinationPath) {
   }
 }
 
-export const getBlobClient = async (containerName = null) => {
+export const getBlobClient = async () => {
   const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
-  const finalContainerName = containerName || getDefaultContainerName();
-
-  // Validate container name is in whitelist
-  if (!isValidContainerName(finalContainerName)) {
-    throw new Error(
-      `Invalid container name '${finalContainerName}'. Allowed containers: ${AZURE_STORAGE_CONTAINER_NAMES.join(', ')}`,
-    );
-  }
+  // Always use default container from env var
+  const finalContainerName = getDefaultContainerName();
 
   if (!connectionString || !finalContainerName) {
     throw new Error(
@@ -210,17 +201,19 @@ export const getBlobClient = async (containerName = null) => {
   return { blobServiceClient, containerClient };
 };
 
-async function saveFileToBlob(chunkPath, requestId, filename = null, containerName = null) {
+async function saveFileToBlob(chunkPath, requestId, filename = null) {
   // Use provider for consistency with cache control headers
+  // Container parameter is ignored - always uses default container from env var
   const storageFactory = StorageFactory.getInstance();
-  const provider = await storageFactory.getAzureProvider(containerName);
+  const provider = await storageFactory.getAzureProvider();
   return await provider.uploadFile({}, chunkPath, requestId, null, filename);
 }
 
 //deletes blob that has the requestId
-async function deleteBlob(requestId, containerName = null) {
+async function deleteBlob(requestId) {
   if (!requestId) throw new Error("Missing requestId parameter");
-  const { containerClient } = await getBlobClient(containerName);
+  // Container parameter is ignored - always uses default container from env var
+  const { containerClient } = await getBlobClient();
   // List all blobs in the container
   const blobs = containerClient.listBlobsFlat();
 
@@ -250,13 +243,12 @@ function uploadBlob(
   saveToLocal = false,
   filePath = null,
   hash = null,
-  containerParam = null,
 ) {
   return new Promise((resolve, reject) => {
     (async () => {
       try {
         let requestId = uuidv4();
-        let containerName = containerParam;
+        // Container parameter is ignored - always uses default container from env var
         const body = {};
         const fields = {}; // Buffer for all fields
 
@@ -281,7 +273,6 @@ function uploadBlob(
               uploadName, // Use the LLM-friendly filename
               resolve,
               hash,
-              containerName,
             );
             resolve(result);
           } catch (error) {
@@ -302,16 +293,8 @@ function uploadBlob(
             } else if (fieldname === "hash") {
               hash = value;
             } else if (fieldname === "container") {
-              if (value && !isValidContainerName(value)) {
-                // Read current containers from env for error message
-                const currentContainerNames = getCurrentContainerNames();
-                errorOccurred = true;
-                const err = new Error(`Invalid container name '${value}'. Allowed containers: ${currentContainerNames.join(', ')}`);
-                err.status = 400;
-                reject(err);
-                return;
-              }
-              containerName = value;
+              // Container parameter is ignored - always uses default container from env var
+              // No validation or error needed, just ignore it
             }
             fields[fieldname] = value; // Store all fields
           });
@@ -333,16 +316,15 @@ function uploadBlob(
             // Simple approach: small delay to allow container field to be processed
             console.log("File received, giving fields time to process...");
             await new Promise(resolve => setTimeout(resolve, 20));
-            console.log("Processing file with containerName:", containerName);
+            // Container parameter is ignored - always uses default container from env var
             
             if (errorOccurred) return; // Check again after waiting
 
-            // Capture containerName value to avoid closure issues
-            const capturedContainerName = containerName;
-            await processFile(fieldname, file, info, capturedContainerName);
+            // Container parameter is ignored - always uses default container from env var
+            await processFile(fieldname, file, info);
           });
 
-          const processFile = async (fieldname, file, info, capturedContainerName) => {
+          const processFile = async (fieldname, file, info) => {
             if (errorOccurred) return;
 
             // Validate file
@@ -440,7 +422,7 @@ function uploadBlob(
                 context,
                 uploadName,
                 azureStream,
-                capturedContainerName,
+                null, // containerName ignored
                 contentType,
               ).catch(async (err) => {
                 cloudUploadError = err;
@@ -451,7 +433,7 @@ function uploadBlob(
                     highWaterMark: 1024 * 1024,
                     autoClose: true,
                   });
-                  return saveToAzureStorage(context, uploadName, diskStream, capturedContainerName, contentType);
+                  return saveToAzureStorage(context, uploadName, diskStream, null, contentType);
                 }
                 throw err;
               });
@@ -482,10 +464,10 @@ function uploadBlob(
               const results = await Promise.all(
                 [
                   azurePromise
-                    ? azurePromise.then((url) => ({ url, type: "primary" }))
+                    ? azurePromise.then((result) => ({ result, type: "primary" }))
                     : null,
                   !azurePromise && saveToLocal
-                    ? Promise.resolve({ url: null, type: "primary-local" }) // placeholder for local, url handled later
+                    ? Promise.resolve({ result: { url: null }, type: "primary-local" }) // placeholder for local, url handled later
                     : null,
                   gcsPromise
                     ? gcsPromise.then((gcs) => ({ gcs, type: "gcs" }))
@@ -496,15 +478,23 @@ function uploadBlob(
               const result = {
                 message: `File '${uploadName}' uploaded successfully.`,
                 filename: uploadName,
-                ...results.reduce((acc, result) => {
-                  if (result.type === "primary") acc.url = result.url;
-                  if (result.type === "gcs")
-                    acc.gcs = ensureUnencodedGcsUrl(result.gcs);
+                ...results.reduce((acc, item) => {
+                  if (item.type === "primary") {
+                    acc.url = item.result.url || item.result;
+                    acc.shortLivedUrl = item.result.shortLivedUrl || item.result.url || item.result;
+                  }
+                  if (item.type === "gcs")
+                    acc.gcs = ensureUnencodedGcsUrl(item.gcs);
                   return acc;
                 }, {}),
               };
               if (hash) result.hash = hash;
-              if (capturedContainerName) result.container = capturedContainerName;
+              // Container parameter is ignored - always uses default container from env var
+              
+              // Ensure shortLivedUrl is always present
+              if (!result.shortLivedUrl && result.url) {
+                result.shortLivedUrl = result.url;
+              }
 
               // If saving locally, wait for disk write to finish and then move to public folder
               if (saveToLocal) {
@@ -512,7 +502,7 @@ function uploadBlob(
                   if (diskWritePromise) {
                     await diskWritePromise; // ensure file fully written
                   }
-                  const localUrl = await saveToLocalStorage(
+                  const localResult = await saveToLocalStorage(
                     context,
                     requestId,
                     uploadName,
@@ -521,7 +511,9 @@ function uploadBlob(
                       autoClose: true,
                     }),
                   );
-                  result.url = localUrl;
+                  // Handle both old format (string) and new format (object)
+                  result.url = typeof localResult === 'string' ? localResult : localResult.url;
+                  result.shortLivedUrl = localResult.shortLivedUrl || result.url;
                 } catch (err) {
                   console.error("Error saving to local storage:", err);
                   throw err;
@@ -576,7 +568,7 @@ function uploadBlob(
                         conversion.convertedPath,
                         requestId,
                         null,
-                        capturedContainerName,
+                        null, // containerName ignored
                       );
 
                     // Optionally save to GCS
@@ -718,7 +710,6 @@ async function uploadFile(
   filename,
   resolve,
   hash = null,
-  containerName = null,
 ) {
   try {
     if (!file) {
@@ -779,12 +770,15 @@ async function uploadFile(
         context,
         uploadName,
         createOptimizedReadStream(uploadPath),
-        containerName,
+        null, // containerName ignored
       );
     storagePromises.push(
-      primaryPromise.then((url) => {
+      primaryPromise.then((result) => {
         context.log("Primary storage upload completed");
-        return { url, type: "primary" };
+        // Handle both old format (string URL) and new format (object with url and shortLivedUrl)
+        const url = typeof result === 'string' ? result : (result.url || result);
+        const shortLivedUrl = result.shortLivedUrl || url;
+        return { url, shortLivedUrl, type: "primary" };
       }),
     );
 
@@ -811,9 +805,12 @@ async function uploadFile(
     const result = {
       message: `File '${uploadName}' ${saveToLocal ? "saved to folder" : "uploaded"} successfully.`,
       filename: uploadName,
-      ...results.reduce((acc, result) => {
-        if (result.type === "primary") acc.url = result.url;
-        if (result.type === "gcs") acc.gcs = ensureUnencodedGcsUrl(result.gcs);
+      ...results.reduce((acc, item) => {
+        if (item.type === "primary") {
+          acc.url = item.url;
+          acc.shortLivedUrl = item.shortLivedUrl || item.url;
+        }
+        if (item.type === "gcs") acc.gcs = ensureUnencodedGcsUrl(item.gcs);
         return acc;
       }, {}),
     };
@@ -822,8 +819,11 @@ async function uploadFile(
       result.hash = hash;
     }
     
-    if (containerName) {
-      result.container = containerName;
+    // Container parameter is ignored - always uses default container from env var
+    
+    // Ensure shortLivedUrl is always present
+    if (!result.shortLivedUrl && result.url) {
+      result.shortLivedUrl = result.url;
     }
 
     // Initialize conversion service
@@ -895,7 +895,8 @@ async function uploadFile(
     context.log("Error in upload process:", error);
     if (body.url) {
       try {
-        await cleanup(context, [body.url], containerName);
+        // Container parameter is ignored - always uses default container from env var
+        await cleanup(context, [body.url]);
       } catch (cleanupError) {
         context.log("Error during cleanup after failure:", cleanupError);
       }
@@ -915,8 +916,9 @@ async function streamToBuffer(stream) {
 }
 
 // Function to delete files that haven't been used in more than a month
-async function cleanup(context, urls = null, containerName = null) {
-  const { containerClient } = await getBlobClient(containerName);
+// Container parameter is ignored - always uses default container from env var
+async function cleanup(context, urls = null) {
+  const { containerClient } = await getBlobClient();
   const cleanedURLs = [];
 
   if (!urls) {
@@ -1154,10 +1156,8 @@ export {
   gcs,
   uploadChunkToGCS,
   downloadFromGCS,
-  // Re-export container constants for backward compatibility
-  getCurrentContainerNames,
-  AZURE_STORAGE_CONTAINER_NAMES,
+  // Re-export container constants
   getDefaultContainerName,
   GCS_BUCKETNAME,
-  isValidContainerName,
+  AZURE_STORAGE_CONTAINER_NAME,
 };
diff --git a/helper-apps/cortex-file-handler/src/constants.js b/helper-apps/cortex-file-handler/src/constants.js
index 888521fa..586ca97e 100644
--- a/helper-apps/cortex-file-handler/src/constants.js
+++ b/helper-apps/cortex-file-handler/src/constants.js
@@ -133,32 +133,16 @@ export const CONVERTED_EXTENSIONS = [
 // Azure Storage constants
 export const AZURITE_ACCOUNT_NAME = "devstoreaccount1";
 
-// Parse comma-separated container names from environment variable
-export const parseContainerNames = () => {
-  const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
-  return containerStr.split(',').map(name => name.trim());
+// Get single container name from environment variable
+// CFH operates on a single Azure container and single GCS bucket
+export const getContainerName = () => {
+  return process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
 };
 
-// Helper function to get current container names at runtime
-// Useful for runtime validation when env vars might change (e.g., in tests)
-export const getCurrentContainerNames = () => {
-  return parseContainerNames();
-};
-
-export const AZURE_STORAGE_CONTAINER_NAMES = parseContainerNames();
-
-// Helper function to get the default container name at runtime
-// This allows tests to change the environment variable and have the correct default
+// Helper function to get current container name at runtime
 export const getDefaultContainerName = () => {
-  return process.env.DEFAULT_AZURE_STORAGE_CONTAINER_NAME || getCurrentContainerNames()[0];
+  return getContainerName();
 };
 
-export const DEFAULT_AZURE_STORAGE_CONTAINER_NAME = process.env.DEFAULT_AZURE_STORAGE_CONTAINER_NAME || AZURE_STORAGE_CONTAINER_NAMES[0];
+export const AZURE_STORAGE_CONTAINER_NAME = getContainerName();
 export const GCS_BUCKETNAME = process.env.GCS_BUCKETNAME || "cortextempfiles";
-
-// Validate if a container name is allowed
-export const isValidContainerName = (containerName) => {
-  // Read from environment at runtime to support dynamically changing env in tests
-  const currentContainerNames = getCurrentContainerNames();
-  return currentContainerNames.includes(containerName);
-};
diff --git a/helper-apps/cortex-file-handler/src/index.js b/helper-apps/cortex-file-handler/src/index.js
index 3a39d501..b6cff2ed 100644
--- a/helper-apps/cortex-file-handler/src/index.js
+++ b/helper-apps/cortex-file-handler/src/index.js
@@ -14,7 +14,6 @@ import {
   removeFromFileStoreMap,
   setFileStoreMap,
   cleanupRedisFileStoreMapAge,
-  getScopedHashKey,
 } from "./redis.js";
 import { FileConversionService } from "./services/FileConversionService.js";
 import { StorageService } from "./services/storage/StorageService.js";
@@ -79,8 +78,10 @@ async function CortexFileHandler(context, req) {
     fetch,
     load,
     restore,
-    container,
+    setRetention,
+    retention,
   } = source;
+  // Container parameter is ignored - always uses default container from env var
 
   // Normalize boolean parameters
   const shouldSave = save === true || save === "true";
@@ -91,22 +92,27 @@ async function CortexFileHandler(context, req) {
 
 
 
+  const shouldSetRetention = setRetention === true || setRetention === "true" || 
+                              (req.query?.operation === "setRetention") || (parsedBody?.operation === "setRetention");
+  
   const operation = shouldSave
     ? "save"
     : shouldCheckHash
       ? "checkHash"
       : shouldClearHash
         ? "clearHash"
-        : shouldFetchRemote
-          ? "remoteFile"
-          : req.method.toLowerCase() === "delete" ||
-              req.query.operation === "delete"
-            ? "delete"
-            : uri
-              ? DOC_EXTENSIONS.some((ext) => uri.toLowerCase().endsWith(ext))
-                ? "document_processing"
-                : "media_chunking"
-              : "upload";
+        : shouldSetRetention
+          ? "setRetention"
+          : shouldFetchRemote
+            ? "remoteFile"
+            : req.method.toLowerCase() === "delete" ||
+                (req.query?.operation === "delete") || (parsedBody?.operation === "delete")
+              ? "delete"
+              : uri
+                ? DOC_EXTENSIONS.some((ext) => uri.toLowerCase().endsWith(ext))
+                  ? "document_processing"
+                  : "media_chunking"
+                : "upload";
 
   context.log(
     `Processing ${req.method} request - ${requestId ? `requestId: ${requestId}, ` : ""}${uri ? `uri: ${uri}, ` : ""}${hash ? `hash: ${hash}, ` : ""}operation: ${operation}`,
@@ -159,13 +165,15 @@ async function CortexFileHandler(context, req) {
   if (operation === "delete") {
     // Check both query string and body params for delete parameters
     // Handle both req.body.params.hash and req.body.hash formats
+    // Note: container is already extracted from source above (line 82), same as checkHash
     const deleteRequestId = req.query.requestId || parsedBody?.params?.requestId || parsedBody?.requestId || requestId;
     const deleteHash = req.query.hash || parsedBody?.params?.hash || parsedBody?.hash || hash;
     
     // If only hash is provided, delete single file by hash
     if (deleteHash && !deleteRequestId) {
       try {
-        const deleted = await storageService.deleteFileByHash(deleteHash, container);
+        // Container parameter is ignored - always uses default container from env var
+        const deleted = await storageService.deleteFileByHash(deleteHash);
         context.res = {
           status: 200,
           body: { 
@@ -194,11 +202,10 @@ async function CortexFileHandler(context, req) {
 
     // First, get the hash from the map if it exists
     if (deleteHash) {
-      const scopedHash = getScopedHashKey(deleteHash, container);
-      const hashResult = await getFileStoreMap(scopedHash);
+      const hashResult = await getFileStoreMap(deleteHash);
       if (hashResult) {
-        context.log(`Found hash in map for deletion: ${deleteHash} (scoped key: ${scopedHash})`);
-        await removeFromFileStoreMap(scopedHash);
+        context.log(`Found hash in map for deletion: ${deleteHash}`);
+        await removeFromFileStoreMap(deleteHash);
       }
     }
 
@@ -210,6 +217,53 @@ async function CortexFileHandler(context, req) {
     return;
   }
 
+  // Set file retention (temporary or permanent)
+  if (operation === "setRetention") {
+    // Extract parameters from query string or body
+    const fileHash = req.query.hash || parsedBody?.params?.hash || parsedBody?.hash || hash;
+    const retention = req.query.retention || parsedBody?.params?.retention || parsedBody?.retention;
+    
+    if (!fileHash) {
+      context.res = {
+        status: 400,
+        body: "Missing hash parameter. Please provide hash in query string or request body.",
+      };
+      return;
+    }
+    
+    if (!retention) {
+      context.res = {
+        status: 400,
+        body: "Missing retention parameter. Please provide retention ('temporary' or 'permanent') in query string or request body.",
+      };
+      return;
+    }
+    
+    // Validate retention value
+    if (retention !== 'temporary' && retention !== 'permanent') {
+      context.res = {
+        status: 400,
+        body: "Invalid retention value. Must be 'temporary' or 'permanent'.",
+      };
+      return;
+    }
+
+    try {
+      const result = await storageService.setRetention(fileHash, retention, context);
+      context.res = {
+        status: 200,
+        body: result,
+      };
+      return;
+    } catch (error) {
+      context.res = {
+        status: error.message.includes("not found") ? 404 : 500,
+        body: error.message,
+      };
+      return;
+    }
+  }
+
   const remoteUrl = shouldFetchRemote;
   if (req.method.toLowerCase() === "get" && remoteUrl) {
     context.log(`Remote file: ${remoteUrl}`);
@@ -226,8 +280,7 @@ async function CortexFileHandler(context, req) {
       }
 
       // Check if file already exists (using hash or URL as the key)
-      // If hash is provided, scope it by container; otherwise use URL as-is
-      const cacheKey = hash ? getScopedHashKey(hash, container) : remoteUrl;
+      const cacheKey = hash || remoteUrl;
       const exists = await getFileStoreMap(cacheKey);
       if (exists) {
         context.res = {
@@ -250,9 +303,11 @@ async function CortexFileHandler(context, req) {
 
       // For remote files, we don't need a requestId folder structure since it's just a single file
       // Pass empty string to store the file directly in the root
-      const res = await storageService.uploadFile(context, filename, '', null, null, container);
+      // Container parameter is ignored - always uses default container from env var
+      const res = await storageService.uploadFile(context, filename, '', null, null);
 
       //Update Redis (using hash or URL as the key)
+      // Container parameter is ignored - always uses default container from env var
       await setFileStoreMap(cacheKey, res);
 
       // Return the file URL
@@ -281,10 +336,9 @@ async function CortexFileHandler(context, req) {
 
   if (hash && clearHash) {
     try {
-      const scopedHash = getScopedHashKey(hash, container);
-      const hashValue = await getFileStoreMap(scopedHash);
+      const hashValue = await getFileStoreMap(hash);
       if (hashValue) {
-        await removeFromFileStoreMap(scopedHash);
+        await removeFromFileStoreMap(hash);
         context.res = {
           status: 200,
           body: `Hash ${hash} removed`,
@@ -306,11 +360,10 @@ async function CortexFileHandler(context, req) {
   }
 
   if (hash && checkHash) {
-    const scopedHash = getScopedHashKey(hash, container);
-    let hashResult = await getFileStoreMap(scopedHash, true); // Skip lazy cleanup to handle it ourselves
+    let hashResult = await getFileStoreMap(hash, true); // Skip lazy cleanup to handle it ourselves
 
     if (hashResult) {
-      context.log(`File exists in map: ${hash} (scoped key: ${scopedHash})`);
+      context.log(`File exists in map: ${hash}`);
 
       // Log the URL retrieved from Redis before checking existence
       context.log(`Checking existence of URL from Redis: ${hashResult?.url}`);
@@ -329,7 +382,7 @@ async function CortexFileHandler(context, req) {
           context.log(
             `File not found in any storage. Removing from map: ${hash}`,
           );
-          await removeFromFileStoreMap(scopedHash);
+          await removeFromFileStoreMap(hash);
           context.res = {
             status: 404,
             body: `Hash ${hash} not found in storage`,
@@ -348,7 +401,7 @@ async function CortexFileHandler(context, req) {
           } catch (error) {
             context.log(`Error restoring to GCS: ${error}`);
             // If restoration fails, remove the hash from the map
-            await removeFromFileStoreMap(scopedHash);
+            await removeFromFileStoreMap(hash);
             context.res = {
               status: 404,
               body: `Hash ${hash} not found`,
@@ -380,13 +433,13 @@ async function CortexFileHandler(context, req) {
             await storageService.downloadFile(hashResult.gcs, downloadedFile);
 
             // Upload to primary storage
+            // Container parameter is ignored - always uses default container from env var
             const res = await storageService.uploadFile(
               context,
               downloadedFile,
               hash,
               null,
               null,
-              container,
             );
 
             // Update the hash result with the new primary storage URL
@@ -406,7 +459,7 @@ async function CortexFileHandler(context, req) {
           } catch (error) {
             console.error("Error restoring from GCS:", error);
             // If restoration fails, remove the hash from the map
-            await removeFromFileStoreMap(scopedHash);
+            await removeFromFileStoreMap(hash);
             context.res = {
               status: 404,
               body: `Hash ${hash} not found`,
@@ -424,7 +477,7 @@ async function CortexFileHandler(context, req) {
           : false;
         if (!finalPrimaryCheck && !finalGCSCheck) {
           context.log(`Failed to restore file. Removing from map: ${hash}`);
-          await removeFromFileStoreMap(scopedHash);
+          await removeFromFileStoreMap(hash);
           context.res = {
             status: 404,
             body: `Hash ${hash} not found`,
@@ -444,10 +497,10 @@ async function CortexFileHandler(context, req) {
 
         // Ensure converted version exists and is synced across storage providers
         try {
+          // Container parameter is ignored - always uses default container from env var
           hashResult = await conversionService.ensureConvertedVersion(
             hashResult,
             requestId,
-            container,
           );
         } catch (error) {
           context.log(`Error ensuring converted version: ${error}`);
@@ -466,6 +519,7 @@ async function CortexFileHandler(context, req) {
         const urlForShortLived = hashResult.converted?.url || hashResult.url;
         try {
           // Extract blob name from the URL to generate new SAS token
+          // Container parameter is ignored - always uses default container from env var
           let blobName;
           try {
             const url = new URL(urlForShortLived);
@@ -474,18 +528,19 @@ async function CortexFileHandler(context, req) {
             
             // For Azurite URLs, the path includes account name: devstoreaccount1/container/blob
             // For real Azure URLs, the path is: container/blob
-            const containerName = storageService.primaryProvider.containerName;
-            
-    // Check if this is an Azurite URL (contains devstoreaccount1)
-    if (path.startsWith(`${AZURITE_ACCOUNT_NAME}/`)) {
-      path = path.substring(`${AZURITE_ACCOUNT_NAME}/`.length); // Remove account prefix
+            // Check if this is an Azurite URL (contains devstoreaccount1)
+            if (path.startsWith(`${AZURITE_ACCOUNT_NAME}/`)) {
+              path = path.substring(`${AZURITE_ACCOUNT_NAME}/`.length); // Remove account prefix
             }
             
-            // Now remove container prefix if it exists
-            if (path.startsWith(containerName + '/')) {
-              blobName = path.substring(containerName.length + 1);
-            } else {
-              blobName = path;
+            // Extract blob name from path (skip container name, always use default container)
+            const pathSegments = path.split('/').filter(segment => segment.length > 0);
+            if (pathSegments.length >= 2) {
+              // Skip container name (first segment), get blob name (remaining segments)
+              blobName = pathSegments.slice(1).join('/');
+            } else if (pathSegments.length === 1) {
+              // Fallback: assume it's just the blob name in default container
+              blobName = pathSegments[0];
             }
             
           } catch (urlError) {
@@ -493,30 +548,42 @@ async function CortexFileHandler(context, req) {
           }
 
           // Generate short-lived SAS token
-          if (blobName && storageService.primaryProvider.generateShortLivedSASToken) {
-            const { containerClient } = await storageService.primaryProvider.getBlobClient();
-            const sasToken = storageService.primaryProvider.generateShortLivedSASToken(
-              containerClient, 
-              blobName, 
-              shortLivedDuration
-            );
-            
-            // Construct new URL with short-lived SAS token
-            const baseUrl = urlForShortLived.split('?')[0]; // Remove existing SAS token
-            const shortLivedUrl = `${baseUrl}?${sasToken}`;
+          // Container parameter is ignored - always uses default container from env var
+          if (blobName) {
+            const provider = storageService.primaryProvider;
             
-            // Add short-lived URL to response
-            response.shortLivedUrl = shortLivedUrl;
-            response.expiresInMinutes = shortLivedDuration;
-            
-            const urlType = hashResult.converted?.url ? 'converted' : 'original';
-            context.log(`Generated short-lived URL for hash: ${hash} using ${urlType} URL (expires in ${shortLivedDuration} minutes)`);
+            if (provider && provider.generateShortLivedSASToken) {
+              const blobClientResult = await provider.getBlobClient();
+              const containerClient = blobClientResult.containerClient;
+              
+              const sasToken = provider.generateShortLivedSASToken(
+                containerClient, 
+                blobName, 
+                shortLivedDuration
+              );
+              
+              // Construct new URL with short-lived SAS token
+              const baseUrl = urlForShortLived.split('?')[0]; // Remove existing SAS token
+              const shortLivedUrl = `${baseUrl}?${sasToken}`;
+              
+              // Add short-lived URL to response
+              response.shortLivedUrl = shortLivedUrl;
+              response.expiresInMinutes = shortLivedDuration;
+              
+              const urlType = hashResult.converted?.url ? 'converted' : 'original';
+              context.log(`Generated short-lived URL for hash: ${hash} using ${urlType} URL (expires in ${shortLivedDuration} minutes)`);
+            } else {
+              // Fallback for storage providers that don't support short-lived tokens
+              response.shortLivedUrl = urlForShortLived;
+              response.expiresInMinutes = shortLivedDuration;
+              const urlType = hashResult.converted?.url ? 'converted' : 'original';
+              context.log(`Storage provider doesn't support short-lived tokens, using ${urlType} URL`);
+            }
           } else {
-            // Fallback for storage providers that don't support short-lived tokens
+            // If we couldn't extract blob name, use original URL
             response.shortLivedUrl = urlForShortLived;
             response.expiresInMinutes = shortLivedDuration;
-            const urlType = hashResult.converted?.url ? 'converted' : 'original';
-            context.log(`Storage provider doesn't support short-lived tokens, using ${urlType} URL`);
+            context.log(`Could not extract blob name from URL, using original URL for short-lived`);
           }
         } catch (error) {
           context.log(`Error generating short-lived URL: ${error}`);
@@ -526,7 +593,7 @@ async function CortexFileHandler(context, req) {
         }
 
         //update redis timestamp with current time
-        await setFileStoreMap(scopedHash, hashResult);
+        await setFileStoreMap(hash, hashResult);
 
         context.res = {
           status: 200,
@@ -558,10 +625,10 @@ async function CortexFileHandler(context, req) {
       storageService.primaryProvider.constructor.name ===
       "LocalStorageProvider";
     // Use uploadBlob to handle multipart/form-data
-    const result = await uploadBlob(context, req, saveToLocal, null, hash, container);
+    // Container parameter is ignored - always uses default container from env var
+    const result = await uploadBlob(context, req, saveToLocal, null, hash);
     if (result?.hash && context?.res?.body) {
-      const scopedHash = getScopedHashKey(result.hash, result.container || container);
-      await setFileStoreMap(scopedHash, context.res.body);
+      await setFileStoreMap(result.hash, context.res.body);
     }
     return;
   }
@@ -623,12 +690,12 @@ async function CortexFileHandler(context, req) {
             }
 
             // Save the converted file
+            // Container parameter is ignored - always uses default container from env var
             const convertedSaveResult =
               await conversionService._saveConvertedFile(
                 conversion.convertedPath,
                 requestId,
                 null,
-                container,
               );
 
             // Return the converted file URL
@@ -641,11 +708,11 @@ async function CortexFileHandler(context, req) {
             };
           } else {
             // File doesn't need conversion, save the original file
+            // Container parameter is ignored - always uses default container from env var
             const saveResult = await conversionService._saveConvertedFile(
               downloadedFile,
               requestId,
               null,
-              container,
             );
 
             // Return the original file URL
@@ -721,13 +788,13 @@ async function CortexFileHandler(context, req) {
         const chunkPath = chunks[index];
         // Use the same base filename for all chunks to ensure consistency
         const chunkFilename = `chunk-${index + 1}-${chunkBaseName}`;
+        // Container parameter is ignored - always uses default container from env var
         const chunkResult = await storageService.uploadFile(
           context,
           chunkPath,
           requestId,
           null,
           chunkFilename,
-          container,
         );
 
         const chunkOffset = chunkOffsets[index];
diff --git a/helper-apps/cortex-file-handler/src/redis.js b/helper-apps/cortex-file-handler/src/redis.js
index 40a1ad9d..45fa44eb 100644
--- a/helper-apps/cortex-file-handler/src/redis.js
+++ b/helper-apps/cortex-file-handler/src/redis.js
@@ -1,32 +1,23 @@
 import redis from "ioredis";
-import { getDefaultContainerName } from "./constants.js";
 
 const connectionString = process.env["REDIS_CONNECTION_STRING"];
 
 /**
- * Generate a scoped hash key for Redis storage
- * Always includes the container name in the format hash:container
+ * Get hash key for Redis storage
+ * No scoping needed - single container only
  * @param {string} hash - The file hash
- * @param {string} containerName - The container name (optional, defaults to default container)
- * @returns {string} The scoped hash key
+ * @returns {string} The hash key (just the hash itself)
  */
-export const getScopedHashKey = (hash, containerName = null) => {
-  if (!hash) return hash;
-  
-  // Get the default container name at runtime to support dynamic env changes in tests
-  const defaultContainerName = getDefaultContainerName();
-  
-  // Use default container if not provided
-  const container = containerName || defaultContainerName;
-  
-  // Always scope by container
-  return `${hash}:${container}`;
+export const getScopedHashKey = (hash) => {
+  // No scoping - just return the hash directly
+  return hash;
 };
 
 // Create a mock client for test environment when Redis is not configured
 const createMockClient = () => {
   const store = new Map();
   const hashMap = new Map();
+  const locks = new Map(); // For lock simulation
   
   return {
     connected: false,
@@ -55,6 +46,28 @@ const createMockClient = () => {
       }
       return 0;
     },
+    async set(key, value, ...options) {
+      // Handle SET with NX (only set if not exists) and EX (expiration)
+      if (options.includes('NX')) {
+        if (locks.has(key)) {
+          return null; // Lock already exists
+        }
+        locks.set(key, Date.now());
+        // Handle expiration if EX is provided
+        const exIndex = options.indexOf('EX');
+        if (exIndex !== -1 && options[exIndex + 1]) {
+          const ttl = options[exIndex + 1] * 1000; // Convert to milliseconds
+          setTimeout(() => locks.delete(key), ttl);
+        }
+        return 'OK';
+      }
+      locks.set(key, Date.now());
+      return 'OK';
+    },
+    async del(key) {
+      locks.delete(key);
+      return 1;
+    },
     async eval(script, numKeys, ...args) {
       // Mock implementation for atomic get-and-delete operation
       if (script.includes('hget') && script.includes('hdel')) {
@@ -315,6 +328,43 @@ const cleanupRedisFileStoreMapAge = async (
   return cleaned;
 };
 
+/**
+ * Acquire a distributed lock for a given key
+ * Uses Redis SETNX with expiration to ensure atomic lock acquisition
+ * @param {string} lockKey - The key to lock
+ * @param {number} ttlSeconds - Time to live in seconds (default: 300 = 5 minutes)
+ * @returns {Promise<boolean>} True if lock was acquired, false if already locked
+ */
+const acquireLock = async (lockKey, ttlSeconds = 300) => {
+  try {
+    const lockName = `lock:${lockKey}`;
+    // Use SET with NX (only set if not exists) and EX (expiration)
+    // Returns 'OK' if lock was acquired, null if already locked
+    const result = await client.set(lockName, "1", "EX", ttlSeconds, "NX");
+    return result === "OK";
+  } catch (error) {
+    console.error(`Error acquiring lock for ${lockKey}:`, error);
+    // In case of error, allow operation to proceed (fail open)
+    // This prevents Redis issues from blocking operations
+    return true;
+  }
+};
+
+/**
+ * Release a distributed lock for a given key
+ * @param {string} lockKey - The key to unlock
+ * @returns {Promise<void>}
+ */
+const releaseLock = async (lockKey) => {
+  try {
+    const lockName = `lock:${lockKey}`;
+    await client.del(lockName);
+  } catch (error) {
+    console.error(`Error releasing lock for ${lockKey}:`, error);
+    // Ignore errors - lock will expire naturally
+  }
+};
+
 export {
   publishRequestProgress,
   connectClient,
@@ -323,5 +373,7 @@ export {
   removeFromFileStoreMap,
   cleanupRedisFileStoreMap,
   cleanupRedisFileStoreMapAge,
+  acquireLock,
+  releaseLock,
   client,
 };
diff --git a/helper-apps/cortex-file-handler/src/services/ConversionService.js b/helper-apps/cortex-file-handler/src/services/ConversionService.js
index ac5c4218..f138b00d 100644
--- a/helper-apps/cortex-file-handler/src/services/ConversionService.js
+++ b/helper-apps/cortex-file-handler/src/services/ConversionService.js
@@ -103,10 +103,9 @@ export class ConversionService {
    * Ensures a file has both original and converted versions
    * @param {Object} fileInfo - Information about the file
    * @param {string} requestId - Request ID for storage
-   * @param {string} containerName - Optional container name for storage
    * @returns {Promise<Object>} - Updated file info with conversion if needed
    */
-  async ensureConvertedVersion(fileInfo, requestId, containerName = null) {
+  async ensureConvertedVersion(fileInfo, requestId) {
     const { url, gcs } = fileInfo;
     // Remove any query parameters before extension check
     const extension = path.extname(url.split("?")[0]).toLowerCase();
@@ -163,11 +162,11 @@ export class ConversionService {
         }
 
         // Save converted file to primary storage
+        // Container parameter is ignored - always uses default container from env var
         const convertedSaveResult = await this._saveConvertedFile(
           conversion.convertedPath,
           requestId,
           null,
-          containerName,
         );
         if (!convertedSaveResult) {
           throw new Error("Failed to save converted file to primary storage");
@@ -378,7 +377,7 @@ export class ConversionService {
     throw new Error("Method _downloadFile must be implemented");
   }
 
-  async _saveConvertedFile(filePath, requestId, filename = null, containerName = null) {
+  async _saveConvertedFile(filePath, requestId, filename = null) {
     throw new Error("Method _saveConvertedFile must be implemented");
   }
 
diff --git a/helper-apps/cortex-file-handler/src/services/FileConversionService.js b/helper-apps/cortex-file-handler/src/services/FileConversionService.js
index 424308b4..a941f857 100644
--- a/helper-apps/cortex-file-handler/src/services/FileConversionService.js
+++ b/helper-apps/cortex-file-handler/src/services/FileConversionService.js
@@ -34,13 +34,14 @@ export class FileConversionService extends ConversionService {
     return downloadFile(url, destination);
   }
 
-  async _saveConvertedFile(filePath, requestId, filename = null, containerName = null) {
+  async _saveConvertedFile(filePath, requestId, filename = null) {
     // Generate a fallback requestId if none supplied (e.g. during checkHash calls)
     const reqId = requestId || uuidv4();
 
     let fileUrl;
     if (this.useAzure) {
-      const provider = await this.storageFactory.getAzureProvider(containerName);
+      // Container parameter is ignored - always uses default container from env var
+      const provider = await this.storageFactory.getAzureProvider();
       const result = await provider.uploadFile({}, filePath, reqId, null, filename);
       fileUrl = result.url;
     } else {
diff --git a/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js b/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js
index b9e904db..e93c9ba3 100644
--- a/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js
+++ b/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js
@@ -107,7 +107,7 @@ export class AzureStorageProvider extends StorageProvider {
     return this.generateSASToken(containerClient, blobName, { minutes });
   }
 
-  async uploadFile(context, filePath, requestId, hash = null, filename = null) {
+  async uploadFile(context, filePath, requestId, hash = null, filename = null, retention = 'temporary') {
     const { containerClient } = await this.getBlobClient();
 
     // Use provided filename or generate LLM-friendly naming
@@ -151,13 +151,18 @@ export class AzureStorageProvider extends StorageProvider {
         ...(contentEncoding ? { blobContentEncoding: contentEncoding } : {}),
         blobCacheControl: 'public, max-age=2592000, immutable',
       },
+      tags: {
+        retention: retention
+      },
     };
     await blockBlobClient.uploadStream(fileStream, undefined, undefined, uploadOptions);
 
     // Generate SAS token after successful upload
     const sasToken = this.generateSASToken(containerClient, blobName);
+    const shortLivedSasToken = this.generateShortLivedSASToken(containerClient, blobName, 5);
 
     const url = `${blockBlobClient.url}?${sasToken}`;
+    const shortLivedUrl = `${blockBlobClient.url}?${shortLivedSasToken}`;
     
     // Validate that the URL contains a blob name (not just container)
     // Azure blob URLs should be: https://account.blob.core.windows.net/container/blobname
@@ -171,11 +176,12 @@ export class AzureStorageProvider extends StorageProvider {
 
     return {
       url: url,
+      shortLivedUrl: shortLivedUrl,
       blobName: blobName,
     };
   }
 
-  async uploadStream(context, encodedFilename, stream, providedContentType = null) {
+  async uploadStream(context, encodedFilename, stream, providedContentType = null, retention = 'temporary') {
     const { containerClient } = await this.getBlobClient();
     let contentType = providedContentType || mime.lookup(encodedFilename);
 
@@ -205,6 +211,9 @@ export class AzureStorageProvider extends StorageProvider {
         ...(contentEncoding ? { blobContentEncoding: contentEncoding } : {}),
         blobCacheControl: 'public, max-age=2592000, immutable',
       },
+      tags: {
+        retention: retention
+      },
       maxConcurrency: 50,
       blockSize: 8 * 1024 * 1024,
     };
@@ -218,8 +227,10 @@ export class AzureStorageProvider extends StorageProvider {
     await blockBlobClient.uploadStream(stream, undefined, undefined, options);
     
     const sasToken = this.generateSASToken(containerClient, blobName);
+    const shortLivedSasToken = this.generateShortLivedSASToken(containerClient, blobName, 5);
     
     const url = `${blockBlobClient.url}?${sasToken}`;
+    const shortLivedUrl = `${blockBlobClient.url}?${shortLivedSasToken}`;
     
     // Validate that the URL contains a blob name (not just container)
     const urlObj = new URL(url);
@@ -228,7 +239,7 @@ export class AzureStorageProvider extends StorageProvider {
       throw new Error(`Generated invalid Azure URL (container-only) from uploadStream: ${url}, blobName: ${blobName}`);
     }
     
-    return url;
+    return { url, shortLivedUrl };
   }
 
   // Use shared utility for MIME type checking
@@ -412,4 +423,36 @@ export class AzureStorageProvider extends StorageProvider {
       return null;
     }
   }
+
+  /**
+   * Update blob index tags (specifically the retention tag)
+   * @param {string} blobName - The blob name
+   * @param {string} retention - The retention value ('temporary' or 'permanent')
+   * @returns {Promise<void>}
+   */
+  async updateBlobTags(blobName, retention) {
+    const { containerClient } = await this.getBlobClient();
+    const blockBlobClient = containerClient.getBlockBlobClient(blobName);
+    
+    // Get current tags first (may return empty object if no tags exist)
+    let currentTags = {};
+    try {
+      const tagsResponse = await blockBlobClient.getTags();
+      // Tags response might be an object with a tags property or a plain object
+      if (tagsResponse && typeof tagsResponse === 'object') {
+        currentTags = tagsResponse.tags || tagsResponse;
+      }
+    } catch (error) {
+      // If getTags fails (e.g., no tags exist), start with empty object
+      currentTags = {};
+    }
+    
+    // Update retention tag
+    const updatedTags = {
+      ...currentTags,
+      retention: retention
+    };
+    
+    await blockBlobClient.setTags(updatedTags);
+  }
 }
diff --git a/helper-apps/cortex-file-handler/src/services/storage/LocalStorageProvider.js b/helper-apps/cortex-file-handler/src/services/storage/LocalStorageProvider.js
index 2133aac0..d6b6eb7a 100644
--- a/helper-apps/cortex-file-handler/src/services/storage/LocalStorageProvider.js
+++ b/helper-apps/cortex-file-handler/src/services/storage/LocalStorageProvider.js
@@ -52,6 +52,7 @@ export class LocalStorageProvider extends StorageProvider {
 
     return {
       url,
+      shortLivedUrl: url, // For local storage, shortLivedUrl is the same as url
       blobName: path.join(requestId, uniqueFileName),
     };
   }
@@ -76,7 +77,11 @@ export class LocalStorageProvider extends StorageProvider {
     // Generate full URL
     const url = `http://${ipAddress}:${port}/files/${requestId}/${sanitizedFilename}`;
 
-    return url;
+    // Return object with url and shortLivedUrl for consistency
+    return {
+      url,
+      shortLivedUrl: url // For local storage, shortLivedUrl is the same as url
+    };
   }
 
   async deleteFiles(requestId) {
diff --git a/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js b/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js
index 51e53a85..f3687641 100644
--- a/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js
+++ b/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js
@@ -1,7 +1,7 @@
 import { AzureStorageProvider } from "./AzureStorageProvider.js";
 import { GCSStorageProvider } from "./GCSStorageProvider.js";
 import { LocalStorageProvider } from "./LocalStorageProvider.js";
-import { getCurrentContainerNames, GCS_BUCKETNAME } from "../../constants.js";
+import { getContainerName, GCS_BUCKETNAME } from "../../constants.js";
 import path from "path";
 import { fileURLToPath } from "url";
 
@@ -32,32 +32,23 @@ export class StorageFactory {
     storageFactoryInstance = null;
   }
 
-  async getPrimaryProvider(containerName = null) {
+  async getPrimaryProvider() {
     if (process.env.AZURE_STORAGE_CONNECTION_STRING) {
-      return await this.getAzureProvider(containerName);
+      return await this.getAzureProvider();
     }
     return this.getLocalProvider();
   }
 
-  async getAzureProvider(containerName = null) {
-    // Read container names from environment directly to get current values
-    const azureStorageContainerNames = getCurrentContainerNames();
-    const defaultAzureStorageContainerName = azureStorageContainerNames[0];
+  async getAzureProvider() {
+    // Always use single container from env var
+    const containerName = getContainerName();
     
-    // Use provided container name or default to first in whitelist
-    const finalContainerName = containerName || defaultAzureStorageContainerName;
-    
-    // Validate container name
-    if (!azureStorageContainerNames.includes(finalContainerName)) {
-      throw new Error(`Invalid container name '${finalContainerName}'. Allowed containers: ${azureStorageContainerNames.join(', ')}`);
-    }
-    
-    // Create unique key for each container
-    const key = `azure-${finalContainerName}`;
+    // Create unique key for caching
+    const key = `azure-${containerName}`;
     if (!this.providers.has(key)) {
       const provider = new AzureStorageProvider(
         process.env.AZURE_STORAGE_CONNECTION_STRING,
-        finalContainerName,
+        containerName,
       );
       this.providers.set(key, provider);
     }
diff --git a/helper-apps/cortex-file-handler/src/services/storage/StorageService.js b/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
index e31d7b7f..4a35ac4a 100644
--- a/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
+++ b/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
@@ -2,6 +2,7 @@ import { StorageFactory } from "./StorageFactory.js";
 import path from "path";
 import os from "os";
 import fs from "fs";
+import { v4 as uuidv4 } from "uuid";
 import { generateShortId } from "../../utils/filenameUtils.js";
 
 export class StorageService {
@@ -36,7 +37,7 @@ export class StorageService {
     /*
             Supported call shapes:
             1) uploadFile(buffer, filename)
-            2) uploadFile(context, filePath, requestId, hash?, filename?, containerName?) – legacy internal use
+            2) uploadFile(context, filePath, requestId, hash?, filename?) – legacy internal use
         */
 
     await this._initialize();
@@ -52,12 +53,21 @@ export class StorageService {
       const tempFile = path.join(os.tmpdir(), `${Date.now()}_${filename}`);
       await fs.promises.writeFile(tempFile, buffer);
       try {
-        const { url } = await this.primaryProvider.uploadFile(
+        const result = await this.primaryProvider.uploadFile(
           {},
           tempFile,
           filename,
+          null, // hash
+          null, // filename (will use provided filename)
+          'temporary' // retention
         );
-        return { url };
+        // Ensure shortLivedUrl is included
+        const response = {
+          url: result.url,
+          shortLivedUrl: result.shortLivedUrl || result.url,
+          blobName: result.blobName
+        };
+        return response;
       } finally {
         if (fs.existsSync(tempFile)) {
           await fs.promises.unlink(tempFile).catch(() => {});
@@ -65,9 +75,10 @@ export class StorageService {
       }
     }
 
-    // Fallback to legacy (context, filePath, requestId, hash?, filename?, containerName?)
-    const [context, filePath, requestId, hash, filename, containerName] = args;
-    return this.uploadFileWithProviders(context, filePath, requestId, hash, filename, containerName);
+    // Fallback to legacy (context, filePath, requestId, hash?, filename?)
+    // Container parameter is ignored - always uses default container from env var
+    const [context, filePath, requestId, hash, filename] = args;
+    return this.uploadFileWithProviders(context, filePath, requestId, hash, filename);
   }
 
   async uploadFileToBackup(fileOrBuffer, filename) {
@@ -140,11 +151,14 @@ export class StorageService {
   async deleteFile(url) {
     await this._initialize();
     
-    if (typeof this.primaryProvider.deleteFile === "function") {
-      return await this.primaryProvider.deleteFile(url);
+    // Always use primary provider - single container only
+    const provider = this.primaryProvider;
+    
+    if (typeof provider.deleteFile === "function") {
+      return await provider.deleteFile(url);
     }
     // Fallback for providers that only have deleteFiles
-    return await this.primaryProvider.deleteFiles([url]);
+    return await provider.deleteFiles([url]);
   }
 
   async deleteFileFromBackup(url) {
@@ -163,10 +177,9 @@ export class StorageService {
   /**
    * Delete a single file by its hash from both primary and backup storage
    * @param {string} hash - The hash of the file to delete
-   * @param {string} containerName - Optional container name for scoping the hash
    * @returns {Promise<Object>} Object containing deletion results and file info
    */
-  async deleteFileByHash(hash, containerName = null) {
+  async deleteFileByHash(hash) {
     await this._initialize();
     
     if (!hash) {
@@ -175,28 +188,13 @@ export class StorageService {
 
     const results = [];
 
-    // Get and remove file information from Redis map (non-atomic operations)
-    const { getFileStoreMap, removeFromFileStoreMap, getScopedHashKey, getDefaultContainerName } = await import("../../redis.js");
-    const { getDefaultContainerName: getDefaultContainerNameFromConstants } = await import("../../constants.js");
-    const scopedHash = getScopedHashKey(hash, containerName);
-    const hashResult = await getFileStoreMap(scopedHash);
+    // Get and remove file information from Redis map
+    const { getFileStoreMap, removeFromFileStoreMap } = await import("../../redis.js");
+    const hashResult = await getFileStoreMap(hash);
     
     if (hashResult) {
-      // Remove from scoped key
-      await removeFromFileStoreMap(scopedHash);
-      
-      // Also check and remove legacy key (unscoped) if this is the default container
-      // This handles backwards compatibility with old entries stored without container scoping
-      const defaultContainerName = getDefaultContainerNameFromConstants();
-      const effectiveContainer = containerName || defaultContainerName;
-      if (effectiveContainer === defaultContainerName && scopedHash.includes(':')) {
-        const [legacyHash] = scopedHash.split(':', 2);
-        // Try to remove legacy key - only attempt if it exists to avoid unnecessary "does not exist" logs
-        const legacyExists = await getFileStoreMap(legacyHash);
-        if (legacyExists) {
-          await removeFromFileStoreMap(legacyHash);
-        }
-      }
+      // Remove from Redis
+      await removeFromFileStoreMap(hash);
     }
     
     if (!hashResult) {
@@ -209,12 +207,16 @@ export class StorageService {
         // Log the URL being deleted for debugging (redact SAS token for security)
         const { redactSasToken } = await import('../../utils/logSecurity.js');
         console.log(`Deleting file from primary storage - hash: ${hash}, url: ${redactSasToken(hashResult.url)}`);
-        const primaryResult = await this.deleteFile(hashResult.url);
+        
+        // Always use primary provider - single container only
+        const provider = this.primaryProvider;
+        
+        const primaryResult = await provider.deleteFile(hashResult.url);
         if (primaryResult) {
           console.log(`Successfully deleted from primary storage - hash: ${hash}, result: ${primaryResult}`);
           results.push({ provider: 'primary', result: primaryResult });
         } else {
-          // deleteFile returned null, which means the URL was invalid
+          // deleteFile returned null, which means the URL was invalid or blob not found
           console.warn(`Invalid or empty URL for hash ${hash}: ${redactSasToken(hashResult.url)}`);
           results.push({ provider: 'primary', error: 'Invalid URL (container-only or empty blob name)' });
         }
@@ -258,7 +260,119 @@ export class StorageService {
     };
   }
 
-  async uploadFileWithProviders(context, filePath, requestId, hash = null, filename = null, containerName = null) {
+  /**
+   * Set the retention tag for a file (temporary or permanent)
+   * This is a simple tag update operation - no file copying occurs
+   * @param {string} hash - The hash of the file
+   * @param {string} retention - The retention value ('temporary' or 'permanent')
+   * @param {Object} context - Context object for logging
+   * @returns {Promise<Object>} Object containing updated file info
+   */
+  async setRetention(hash, retention, context = {}) {
+    await this._initialize();
+    
+    if (!hash) {
+      throw new Error("Missing hash parameter");
+    }
+    
+    if (retention !== 'temporary' && retention !== 'permanent') {
+      throw new Error("Retention must be 'temporary' or 'permanent'");
+    }
+
+    // Get Redis functions
+    const { getFileStoreMap, setFileStoreMap } = await import("../../redis.js");
+    const { getDefaultContainerName } = await import("../../constants.js");
+    
+    // Look up file by hash
+    const container = getDefaultContainerName();
+    const hashResult = await getFileStoreMap(hash);
+    
+    if (!hashResult) {
+      throw new Error(`File with hash ${hash} not found`);
+    }
+
+    context.log?.(`Setting retention tag for file ${hash} to ${retention}`);
+
+    // Extract blob name from URL
+    if (!hashResult.url) {
+      throw new Error(`File with hash ${hash} has no valid URL`);
+    }
+
+    // Get the Azure provider
+    const provider = await this.factory.getAzureProvider(container);
+    
+    // Extract blob name from URL
+    const blobName = provider.extractBlobNameFromUrl(hashResult.url);
+    if (!blobName) {
+      throw new Error(`Could not extract blob name from URL: ${hashResult.url}`);
+    }
+
+    // Update blob index tag
+    context.log?.(`Updating blob index tag for ${blobName} to ${retention}`);
+    await provider.updateBlobTags(blobName, retention);
+
+    // Generate new short-lived URL
+    const { containerClient } = await provider.getBlobClient();
+    const shortLivedSasToken = provider.generateShortLivedSASToken(containerClient, blobName, 5);
+    const urlObj = new URL(hashResult.url);
+    const baseUrl = `${urlObj.protocol}//${urlObj.host}${urlObj.pathname}`;
+    const shortLivedUrl = `${baseUrl}?${shortLivedSasToken}`;
+
+    // Handle converted file if it exists
+    let convertedResult = null;
+    if (hashResult.converted?.url) {
+      context.log?.(`Updating blob index tag for converted file to ${retention}`);
+      const convertedBlobName = provider.extractBlobNameFromUrl(hashResult.converted.url);
+      if (convertedBlobName) {
+        try {
+          await provider.updateBlobTags(convertedBlobName, retention);
+          const convertedUrlObj = new URL(hashResult.converted.url);
+          const convertedBaseUrl = `${convertedUrlObj.protocol}//${convertedUrlObj.host}${convertedUrlObj.pathname}`;
+          const convertedShortLivedSasToken = provider.generateShortLivedSASToken(containerClient, convertedBlobName, 5);
+          const convertedShortLivedUrl = `${convertedBaseUrl}?${convertedShortLivedSasToken}`;
+          convertedResult = {
+            url: hashResult.converted.url,
+            shortLivedUrl: convertedShortLivedUrl,
+            gcs: hashResult.converted.gcs
+          };
+        } catch (error) {
+          context.log?.(`Warning: Failed to update converted file tag: ${error.message}`);
+          convertedResult = hashResult.converted;
+        }
+      } else {
+        convertedResult = hashResult.converted;
+      }
+    }
+
+    // Update Redis with new information (including shortLivedUrl)
+    const newFileInfo = {
+      ...hashResult,
+      url: hashResult.url, // URL stays the same - same blob, just different tag
+      shortLivedUrl: shortLivedUrl,
+      gcs: hashResult.gcs,
+      timestamp: new Date().toISOString()
+    };
+    
+    if (convertedResult) {
+      newFileInfo.converted = convertedResult;
+    }
+
+    await setFileStoreMap(hash, newFileInfo);
+    context.log?.(`Updated Redis map for hash: ${hash}`);
+
+    return {
+      hash,
+      filename: hashResult.filename,
+      retention: retention,
+      url: hashResult.url,
+      shortLivedUrl: shortLivedUrl,
+      gcs: hashResult.gcs,
+      converted: convertedResult,
+      message: `File retention set to ${retention}`
+    };
+  }
+
+  async uploadFileWithProviders(context, filePath, requestId, hash = null, filename = null) {
     await this._initialize();
     
     // Use provided filename or generate one
@@ -268,17 +382,17 @@ export class StorageService {
       return `${shortId}${fileExtension}`;
     })();
 
-    // Get the appropriate provider for the container
-    const primaryProvider = containerName ? 
-      await this.factory.getAzureProvider(containerName) : 
-      this.primaryProvider;
+    // Always use the default provider (container parameter ignored)
+    const primaryProvider = this.primaryProvider;
 
+    // All files are uploaded with retention=temporary by default
     const primaryResult = await primaryProvider.uploadFile(
       context,
       filePath,
       requestId,
       hash,
       finalFilename,
+      'temporary' // retention tag
     );
 
     let gcsResult = null;
@@ -292,7 +406,31 @@ export class StorageService {
       );
     }
 
-    return { ...primaryResult, gcs: gcsResult?.url };
+    // Ensure shortLivedUrl is always included
+    const result = { ...primaryResult, gcs: gcsResult?.url };
+    if (!result.shortLivedUrl && result.url) {
+      // Fallback: generate short-lived URL if not provided
+      if (primaryProvider.generateShortLivedSASToken) {
+        try {
+          const { containerClient } = await primaryProvider.getBlobClient();
+          const blobName = primaryResult.blobName || primaryProvider.extractBlobNameFromUrl(result.url);
+          if (blobName) {
+            const shortLivedSasToken = primaryProvider.generateShortLivedSASToken(containerClient, blobName, 5);
+            const urlObj = new URL(result.url);
+            const baseUrl = `${urlObj.protocol}//${urlObj.host}${urlObj.pathname}`;
+            result.shortLivedUrl = `${baseUrl}?${shortLivedSasToken}`;
+          }
+        } catch (error) {
+          context.log?.(`Warning: Could not generate shortLivedUrl: ${error.message}`);
+        }
+      }
+      // If still no shortLivedUrl, use the regular URL as fallback
+      if (!result.shortLivedUrl) {
+        result.shortLivedUrl = result.url;
+      }
+    }
+
+    return result;
   }
 
   async deleteFiles(requestId) {
@@ -406,12 +544,15 @@ export class StorageService {
     }
 
     // Download from primary storage
-    const tempFile = path.join(os.tmpdir(), path.basename(existingFile.url));
+    // Extract filename from URL (remove query parameters first)
+    const urlWithoutQuery = existingFile.url.split('?')[0];
+    const filename = path.basename(urlWithoutQuery) || `restore-${uuidv4()}`;
+    const tempFile = path.join(os.tmpdir(), filename);
     try {
       await this.primaryProvider.downloadFile(existingFile.url, tempFile);
 
-      // Upload to GCS
-      const requestId = path.dirname(existingFile.blobName) || "restore";
+      // Upload to GCS - extract requestId from blobName if available, otherwise use empty string
+      const requestId = existingFile.blobName ? path.dirname(existingFile.blobName) : "";
       const gcsResult = await this.backupProvider.uploadFile(
         context,
         tempFile,
diff --git a/helper-apps/cortex-file-handler/src/start.js b/helper-apps/cortex-file-handler/src/start.js
index 0624871b..91e03d59 100644
--- a/helper-apps/cortex-file-handler/src/start.js
+++ b/helper-apps/cortex-file-handler/src/start.js
@@ -6,7 +6,7 @@ import cors from "cors";
 import { readFileSync } from "fs";
 
 import { publicIpv4 } from "public-ip";
-import { AZURE_STORAGE_CONTAINER_NAMES, getDefaultContainerName } from "./blobHandler.js";
+import { AZURE_STORAGE_CONTAINER_NAME, getDefaultContainerName } from "./blobHandler.js";
 import { sanitizeForLogging } from "./utils/logSecurity.js";
 
 // When running under tests we want all generated URLs to resolve to the
@@ -102,9 +102,8 @@ if (import.meta.url === `file://${process.argv[1]}`) {
         `Cortex File Handler v${version} running on port ${port} (includes legacy MediaFileChunker endpoint)`,
       );
       
-      // Debug: Show configured container names
-      console.log(`Configured container names: ${AZURE_STORAGE_CONTAINER_NAMES.join(', ')}`);
-      console.log(`Default container name: ${getDefaultContainerName()}`);
+      // Debug: Show configured container name
+      console.log(`Configured container name: ${AZURE_STORAGE_CONTAINER_NAME}`);
     });
   });
 }
diff --git a/helper-apps/cortex-file-handler/tests/blobHandler.test.js b/helper-apps/cortex-file-handler/tests/blobHandler.test.js
index 1b35ea41..83110fb4 100644
--- a/helper-apps/cortex-file-handler/tests/blobHandler.test.js
+++ b/helper-apps/cortex-file-handler/tests/blobHandler.test.js
@@ -11,9 +11,8 @@ import {
   gcsUrlExists,
   deleteGCS,
   getBlobClient,
-  AZURE_STORAGE_CONTAINER_NAMES,
+  AZURE_STORAGE_CONTAINER_NAME,
   getDefaultContainerName,
-  isValidContainerName,
 } from "../src/blobHandler.js";
 import { urlExists } from "../src/helper.js";
 import CortexFileHandler from "../src/index.js";
@@ -319,40 +318,14 @@ test("test hash check returns 404 when both storages are empty", async (t) => {
 });
 
 // Container name parsing and validation tests
-test("AZURE_STORAGE_CONTAINER_NAMES should be an array with at least one container", (t) => {
-  t.true(Array.isArray(AZURE_STORAGE_CONTAINER_NAMES), "Should be an array");
-  t.true(AZURE_STORAGE_CONTAINER_NAMES.length > 0, "Should have at least one container");
-  
-  // All items should be non-empty strings
-  AZURE_STORAGE_CONTAINER_NAMES.forEach((name, index) => {
-    t.is(typeof name, 'string', `Container at index ${index} should be a string`);
-    t.true(name.length > 0, `Container at index ${index} should not be empty`);
-  });
+test("AZURE_STORAGE_CONTAINER_NAME should be a string", (t) => {
+  t.is(typeof AZURE_STORAGE_CONTAINER_NAME, 'string', "Should be a string");
+  t.true(AZURE_STORAGE_CONTAINER_NAME.length > 0, "Should not be empty");
 });
 
-test("getDefaultContainerName should return the first container", (t) => {
+test("getDefaultContainerName should return the container name", (t) => {
   const defaultContainer = getDefaultContainerName();
-  t.is(defaultContainer, AZURE_STORAGE_CONTAINER_NAMES[0]);
+  t.is(defaultContainer, AZURE_STORAGE_CONTAINER_NAME);
   t.truthy(defaultContainer);
   t.is(typeof defaultContainer, 'string');
 });
-
-test("isValidContainerName should validate container names correctly", (t) => {
-  // All configured containers should be valid (getCurrentContainerNames reads from env at runtime)
-  const currentContainers = AZURE_STORAGE_CONTAINER_NAMES;
-  currentContainers.forEach(containerName => {
-    t.true(isValidContainerName(containerName), `${containerName} should be valid`);
-  });
-  
-  // Invalid containers should return false
-  t.false(isValidContainerName("invalid-container"));
-  t.false(isValidContainerName(""));
-  t.false(isValidContainerName(null));
-  t.false(isValidContainerName(undefined));
-  t.false(isValidContainerName("non-existent"));
-});
-
-test("container names should be unique", (t) => {
-  const uniqueNames = new Set(AZURE_STORAGE_CONTAINER_NAMES);
-  t.is(uniqueNames.size, AZURE_STORAGE_CONTAINER_NAMES.length, "All container names should be unique");
-});
diff --git a/helper-apps/cortex-file-handler/tests/containerConversionFlow.test.js b/helper-apps/cortex-file-handler/tests/containerConversionFlow.test.js
index 8f596126..9b26adc2 100644
--- a/helper-apps/cortex-file-handler/tests/containerConversionFlow.test.js
+++ b/helper-apps/cortex-file-handler/tests/containerConversionFlow.test.js
@@ -10,8 +10,7 @@ import XLSX from "xlsx";
 import { port } from "../src/start.js";
 import {
   uploadBlob,
-  isValidContainerName,
-  AZURE_STORAGE_CONTAINER_NAMES,
+  AZURE_STORAGE_CONTAINER_NAME,
   saveFileToBlob,
 } from "../src/blobHandler.js";
 import { FileConversionService } from "../src/services/FileConversionService.js";
@@ -107,63 +106,13 @@ test.after.always(async (t) => {
   }
 });
 
-// Test that FileConversionService._saveConvertedFile respects container parameter
-test("FileConversionService._saveConvertedFile should use specified container", async (t) => {
+// Test that file upload with conversion works
+test("File upload with conversion should work", async (t) => {
   if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
     t.pass("Skipping test - Azure not configured");
     return;
   }
 
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
-  try {
-    const service = new FileConversionService(mockContext, true); // useAzure = true
-    
-    // Create a test file to save
-    const testContent = "This is converted file content";
-    const testFile = await createTestFile(testContent, "txt", "converted-test.txt");
-    const requestId = uuidv4();
-    const targetContainer = "test2";
-
-    // Call _saveConvertedFile with container parameter
-    const result = await service._saveConvertedFile(
-      testFile,
-      requestId,
-      null, // filename
-      targetContainer
-    );
-
-    t.truthy(result);
-    t.truthy(result.url);
-
-    // Verify the URL indicates it was uploaded to the correct container
-    const containerFromUrl = getContainerFromUrl(result.url);
-    t.is(containerFromUrl, targetContainer, 
-      `File should be uploaded to container ${targetContainer}, but was uploaded to ${containerFromUrl}`);
-
-    // Cleanup
-    await cleanupHashAndFile(null, result.url, baseUrl);
-  } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
-});
-
-// Test that file upload with conversion respects container parameter
-test("File upload with conversion should upload both original and converted files to specified container", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
   try {
     // Create an Excel file that will need conversion
     const excelData = [
@@ -172,12 +121,10 @@ test("File upload with conversion should upload both original and converted file
       ["Jane", 25, "Boston"],
     ];
     const testFile = await createTestFile(excelData, "xlsx", "test-conversion.xlsx");
-    const targetContainer = "test3";
 
-    // Create form data with container parameter
+    // Create form data
     const form = new FormData();
     form.append("file", fs.createReadStream(testFile), "test-conversion.xlsx");
-    form.append("container", targetContainer);
 
     const response = await axios.post(baseUrl, form, {
       headers: {
@@ -191,16 +138,9 @@ test("File upload with conversion should upload both original and converted file
     t.is(response.status, 200);
     t.truthy(response.data.url);
 
-    // Check that the main uploaded file is in the correct container
-    const mainContainerFromUrl = getContainerFromUrl(response.data.url);
-    t.is(mainContainerFromUrl, targetContainer,
-      `Original file should be in container ${targetContainer}, but was in ${mainContainerFromUrl}`);
-
-    // If there's a converted file mentioned in the response, check its container too
+    // Check that conversion worked
     if (response.data.converted && response.data.converted.url) {
-      const convertedContainerFromUrl = getContainerFromUrl(response.data.converted.url);
-      t.is(convertedContainerFromUrl, targetContainer,
-        `Converted file should be in container ${targetContainer}, but was in ${convertedContainerFromUrl}`);
+      t.truthy(response.data.converted.url);
     }
 
     // Cleanup
@@ -209,35 +149,25 @@ test("File upload with conversion should upload both original and converted file
       await cleanupHashAndFile(null, response.data.converted.url, baseUrl);
     }
   } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
+    // No env changes needed
   }
 });
 
-// Test document processing with save=true and container parameter
-test("Document processing with save=true should save converted file to specified container", async (t) => {
+// Test document processing with save=true
+test("Document processing with save=true should work", async (t) => {
   if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
     t.pass("Skipping test - Azure not configured");
     return;
   }
 
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
   try {
     // First upload a document file to get a URI
     const docContent = "This is a test document content for processing.";
     const testFile = await createTestFile(docContent, "txt", "test-doc.txt");
-    const targetContainer = "test1";
 
     // Upload the file first
     const uploadForm = new FormData();
     uploadForm.append("file", fs.createReadStream(testFile), "test-doc.txt");
-    uploadForm.append("container", targetContainer);
 
     const uploadResponse = await axios.post(baseUrl, uploadForm, {
       headers: {
@@ -251,13 +181,12 @@ test("Document processing with save=true should save converted file to specified
     t.is(uploadResponse.status, 200);
     const documentUri = uploadResponse.data.url;
 
-    // Now process the document with save=true and container parameter
+    // Now process the document with save=true
     const processResponse = await axios.get(baseUrl, {
       params: {
         uri: documentUri,
         requestId: uuidv4(),
         save: true,
-        container: targetContainer,
       },
       validateStatus: (status) => true,
       timeout: 60000,
@@ -266,34 +195,21 @@ test("Document processing with save=true should save converted file to specified
     t.is(processResponse.status, 200);
     t.truthy(processResponse.data.url);
 
-    // Check that the saved file is in the correct container
-    const savedContainerFromUrl = getContainerFromUrl(processResponse.data.url);
-    t.is(savedContainerFromUrl, targetContainer,
-      `Saved processed file should be in container ${targetContainer}, but was in ${savedContainerFromUrl}`);
-
     // Cleanup
     await cleanupHashAndFile(null, documentUri, baseUrl);
     await cleanupHashAndFile(null, processResponse.data.url, baseUrl);
   } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
+    // No env changes needed
   }
 });
 
-// Test checkHash operation preserves container for converted files
-test("checkHash operation should respect container parameter for converted files", async (t) => {
+// Test checkHash operation with conversion
+test("checkHash operation should work with converted files", async (t) => {
   if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
     t.pass("Skipping test - Azure not configured");
     return;
   }
 
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
   try {
     // Create an Excel file that will need conversion
     const excelData = [
@@ -302,14 +218,12 @@ test("checkHash operation should respect container parameter for converted files
       ["Gadget", 15.50],
     ];
     const testFile = await createTestFile(excelData, "xlsx", "hash-test.xlsx");
-    const targetContainer = "test2";
     const testHash = uuidv4();
 
-    // Upload the file with a hash and container parameter
+    // Upload the file with a hash
     const form = new FormData();
     form.append("file", fs.createReadStream(testFile), "hash-test.xlsx");
     form.append("hash", testHash);
-    form.append("container", targetContainer);
 
     const uploadResponse = await axios.post(baseUrl, form, {
       headers: {
@@ -322,12 +236,11 @@ test("checkHash operation should respect container parameter for converted files
 
     t.is(uploadResponse.status, 200);
 
-    // Now check the hash with container parameter
+    // Now check the hash
     const checkResponse = await axios.get(baseUrl, {
       params: {
         hash: testHash,
         checkHash: true,
-        container: targetContainer,
       },
       validateStatus: (status) => true,
       timeout: 60000,
@@ -336,16 +249,9 @@ test("checkHash operation should respect container parameter for converted files
     t.is(checkResponse.status, 200);
     t.truthy(checkResponse.data.url);
 
-    // Check that the original file is in the correct container
-    const originalContainerFromUrl = getContainerFromUrl(checkResponse.data.url);
-    t.is(originalContainerFromUrl, targetContainer,
-      `Original file should be in container ${targetContainer}, but was in ${originalContainerFromUrl}`);
-
-    // If there's a converted file, check its container too
+    // Check that conversion worked
     if (checkResponse.data.converted && checkResponse.data.converted.url) {
-      const convertedContainerFromUrl = getContainerFromUrl(checkResponse.data.converted.url);
-      t.is(convertedContainerFromUrl, targetContainer,
-        `Converted file should be in container ${targetContainer}, but was in ${convertedContainerFromUrl}`);
+      t.truthy(checkResponse.data.converted.url);
     }
 
     // Cleanup
@@ -354,101 +260,6 @@ test("checkHash operation should respect container parameter for converted files
       await cleanupHashAndFile(null, checkResponse.data.converted.url, baseUrl);
     }
   } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
+    // No env changes needed
   }
 });
-
-// Test that default container is used when no container specified for conversions
-test("Conversion should use default container when no container specified", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
-  try {
-    const service = new FileConversionService(mockContext, true);
-    
-    // Create a test file to save
-    const testContent = "This is converted file content for default container test";
-    const testFile = await createTestFile(testContent, "txt", "default-container-test.txt");
-    const requestId = uuidv4();
-
-    // Call _saveConvertedFile without container parameter (should use default)
-    const result = await service._saveConvertedFile(
-      testFile,
-      requestId,
-      null, // filename
-      null  // container - should use default
-    );
-
-    t.truthy(result);
-    t.truthy(result.url);
-
-    // Verify the URL indicates it was uploaded to the default container
-    const containerFromUrl = getContainerFromUrl(result.url);
-    // Read current default from environment (not the cached module value)
-    const currentContainerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
-    const currentDefaultContainer = currentContainerStr.split(',').map(name => name.trim())[0];
-    t.is(containerFromUrl, currentDefaultContainer,
-      `File should be uploaded to default container ${currentDefaultContainer}, but was uploaded to ${containerFromUrl}`);
-
-    // Cleanup
-    await cleanupHashAndFile(null, result.url, baseUrl);
-  } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
-});
-
-// Test saveFileToBlob function directly with container parameter
-test("saveFileToBlob should respect container parameter", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
-  try {
-    // Create a test file
-    const testContent = "This is a test for saveFileToBlob with container parameter";
-    const testFile = await createTestFile(testContent, "txt", "save-blob-test.txt");
-    const requestId = uuidv4();
-    const targetContainer = "test3";
-
-    // Call saveFileToBlob directly with container parameter
-    const result = await saveFileToBlob(testFile, requestId, null, targetContainer);
-
-    t.truthy(result);
-    t.truthy(result.url);
-    t.truthy(result.blobName);
-
-    // Verify the URL indicates it was uploaded to the correct container
-    const containerFromUrl = getContainerFromUrl(result.url);
-    t.is(containerFromUrl, targetContainer,
-      `File should be uploaded to container ${targetContainer}, but was uploaded to ${containerFromUrl}`);
-
-    // Cleanup
-    await cleanupHashAndFile(null, result.url, baseUrl);
-  } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
-});
\ No newline at end of file
diff --git a/helper-apps/cortex-file-handler/tests/containerNameParsing.test.js b/helper-apps/cortex-file-handler/tests/containerNameParsing.test.js
index 90b3e361..efd0b0a3 100644
--- a/helper-apps/cortex-file-handler/tests/containerNameParsing.test.js
+++ b/helper-apps/cortex-file-handler/tests/containerNameParsing.test.js
@@ -1,9 +1,7 @@
 import test from "ava";
 import { 
-  AZURE_STORAGE_CONTAINER_NAMES, 
+  AZURE_STORAGE_CONTAINER_NAME, 
   getDefaultContainerName,
-  isValidContainerName,
-  getCurrentContainerNames
 } from "../src/blobHandler.js";
 
 // Mock environment variables for testing
@@ -19,219 +17,29 @@ test.afterEach(() => {
   process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
 });
 
-test("parseContainerNames should handle single container name", (t) => {
-  // Set environment variable for single container
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "single-container";
-  
-  // We need to reload the module to pick up the new environment variable
-  // Since we can't easily reload ES modules, we'll test the logic directly
-  const parseContainerNames = () => {
-    const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
-    return containerStr.split(',').map(name => name.trim());
-  };
-  
-  const result = parseContainerNames();
-  
-  t.is(result.length, 1);
-  t.is(result[0], "single-container");
-});
-
-test("parseContainerNames should handle comma-separated container names", (t) => {
-  // Set environment variable for multiple containers
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "container1,container2,container3";
-  
-  const parseContainerNames = () => {
-    const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
-    return containerStr.split(',').map(name => name.trim());
-  };
-  
-  const result = parseContainerNames();
-  
-  t.is(result.length, 3);
-  t.is(result[0], "container1");
-  t.is(result[1], "container2");
-  t.is(result[2], "container3");
-});
-
-test("parseContainerNames should handle comma-separated names with whitespace", (t) => {
-  // Set environment variable with spaces around commas
-  process.env.AZURE_STORAGE_CONTAINER_NAME = " container1 , container2 , container3 ";
-  
-  const parseContainerNames = () => {
-    const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
-    return containerStr.split(',').map(name => name.trim());
-  };
-  
-  const result = parseContainerNames();
-  
-  t.is(result.length, 3);
-  t.is(result[0], "container1");
-  t.is(result[1], "container2");
-  t.is(result[2], "container3");
-});
-
-test("parseContainerNames should default to cortextempfiles when env var is not set", (t) => {
-  // Unset environment variable
-  delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-  
-  const parseContainerNames = () => {
-    const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
-    return containerStr.split(',').map(name => name.trim());
-  };
-  
-  const result = parseContainerNames();
-  
-  t.is(result.length, 1);
-  t.is(result[0], "cortextempfiles");
-});
-
-test("parseContainerNames should handle empty string", (t) => {
-  // Set environment variable to empty string
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "";
-  
-  const parseContainerNames = () => {
-    const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
-    return containerStr.split(',').map(name => name.trim());
-  };
-  
-  const result = parseContainerNames();
-  
-  t.is(result.length, 1);
-  t.is(result[0], "cortextempfiles");
-});
-
-test("parseContainerNames should handle only commas", (t) => {
-  // Set environment variable to only commas
-  process.env.AZURE_STORAGE_CONTAINER_NAME = ",,,";
-  
-  const parseContainerNames = () => {
-    const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
-    return containerStr.split(',').map(name => name.trim());
-  };
-  
-  const result = parseContainerNames();
-  
-  // Should result in 4 empty strings after trimming
-  t.is(result.length, 4);
-  t.is(result[0], "");
-  t.is(result[1], "");
-  t.is(result[2], "");
-  t.is(result[3], "");
-});
-
-test("getDefaultContainerName should return the first container in the list", (t) => {
-  // Test with current module exports (these are loaded at import time)
-  // The default should be the first item in the array
+test("getDefaultContainerName should return the container name", (t) => {
   const defaultContainer = getDefaultContainerName();
-  t.is(defaultContainer, getCurrentContainerNames()[0]);
-  
-  // Additional validation that it's a non-empty string
+  t.is(defaultContainer, AZURE_STORAGE_CONTAINER_NAME);
   t.truthy(defaultContainer);
   t.is(typeof defaultContainer, 'string');
   t.true(defaultContainer.length > 0);
 });
 
-test("isValidContainerName should return true for valid container names", (t) => {
-  // Get current container names at runtime (not cached)
-  const currentContainers = getCurrentContainerNames();
-  
-  // Test with each container name from the current configuration
-  currentContainers.forEach(containerName => {
-    t.true(isValidContainerName(containerName), `Container name '${containerName}' should be valid`);
-  });
+test("AZURE_STORAGE_CONTAINER_NAME should be a non-empty string", (t) => {
+  t.is(typeof AZURE_STORAGE_CONTAINER_NAME, 'string');
+  t.true(AZURE_STORAGE_CONTAINER_NAME.length > 0);
+  t.true(AZURE_STORAGE_CONTAINER_NAME.trim().length > 0);
 });
 
-test("isValidContainerName should return false for invalid container names", (t) => {
-  const invalidNames = [
-    "invalid-container",
-    "not-in-list",
-    "",
-    null,
-    undefined,
-    "container-that-does-not-exist"
-  ];
+test("container name should default to cortextempfiles when env var is not set", (t) => {
+  // Unset environment variable
+  delete process.env.AZURE_STORAGE_CONTAINER_NAME;
   
-  invalidNames.forEach(containerName => {
-    t.false(isValidContainerName(containerName), `Container name '${containerName}' should be invalid`);
-  });
-});
-
-test("isValidContainerName should handle edge cases", (t) => {
-  // Test with various edge cases
-  t.false(isValidContainerName(null));
-  t.false(isValidContainerName(undefined));
-  t.false(isValidContainerName(""));
-  t.false(isValidContainerName("   ")); // whitespace only
-  t.false(isValidContainerName(123)); // number
-  t.false(isValidContainerName({})); // object
-  t.false(isValidContainerName([])); // array
-});
-
-test("container configuration should have at least one container", (t) => {
-  const currentContainers = getCurrentContainerNames();
-  t.true(currentContainers.length > 0, "Should have at least one container configured");
-  t.truthy(currentContainers[0], "First container should not be empty");
-});
-
-test("all configured container names should be non-empty strings", (t) => {
-  const currentContainers = getCurrentContainerNames();
-  currentContainers.forEach((containerName, index) => {
-    t.is(typeof containerName, 'string', `Container at index ${index} should be a string`);
-    t.true(containerName.length > 0, `Container at index ${index} should not be empty`);
-    t.true(containerName.trim().length > 0, `Container at index ${index} should not be only whitespace`);
-  });
-});
-
-test("container names should not contain duplicates", (t) => {
-  const currentContainers = getCurrentContainerNames();
-  const uniqueNames = new Set(currentContainers);
-  t.is(uniqueNames.size, currentContainers.length, "Container names should be unique");
-});
-
-// Integration test with actual environment simulation
-test("complete container parsing workflow", (t) => {
-  const testCases = [
-    {
-      env: "single",
-      expected: ["single"],
-      description: "single container name"
-    },
-    {
-      env: "first,second",
-      expected: ["first", "second"],
-      description: "two container names"
-    },
-    {
-      env: "one, two, three",
-      expected: ["one", "two", "three"],
-      description: "three container names with spaces"
-    },
-    {
-      env: "  leading  ,  trailing  ",
-      expected: ["leading", "trailing"],
-      description: "names with leading/trailing spaces"
-    }
-  ];
+  // The constant is loaded at import time, so we test the getter function
+  const getContainerName = () => {
+    return process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
+  };
   
-  testCases.forEach(({ env, expected, description }) => {
-    const parseContainerNames = () => {
-      return env.split(',').map(name => name.trim());
-    };
-    
-    const containerNames = parseContainerNames();
-    const defaultContainer = containerNames[0];
-    
-    // Test parsing
-    t.deepEqual(containerNames, expected, `Parsing should work for ${description}`);
-    
-    // Test default is first
-    t.is(defaultContainer, expected[0], `Default should be first container for ${description}`);
-    
-    // Test validation
-    expected.forEach(name => {
-      t.true(containerNames.includes(name), `${name} should be valid for ${description}`);
-    });
-    
-    t.false(containerNames.includes("invalid"), `Invalid name should not be valid for ${description}`);
-  });
-});
\ No newline at end of file
+  const result = getContainerName();
+  t.is(result, "cortextempfiles");
+});
diff --git a/helper-apps/cortex-file-handler/tests/containerParameterFlow.test.js b/helper-apps/cortex-file-handler/tests/containerParameterFlow.test.js
deleted file mode 100644
index bab036b4..00000000
--- a/helper-apps/cortex-file-handler/tests/containerParameterFlow.test.js
+++ /dev/null
@@ -1,396 +0,0 @@
-import test from "ava";
-import fs from "fs";
-import path from "path";
-import os from "os";
-import { fileURLToPath } from "url";
-import { v4 as uuidv4 } from "uuid";
-import axios from "axios";
-import FormData from "form-data";
-import { port } from "../src/start.js";
-import {
-  uploadBlob,
-  isValidContainerName,
-  AZURE_STORAGE_CONTAINER_NAMES,
-  getDefaultContainerName,
-} from "../src/blobHandler.js";
-import CortexFileHandler from "../src/index.js";
-import {
-  startTestServer,
-  stopTestServer,
-  setupTestDirectory,
-  cleanupHashAndFile,
-  getFolderNameFromUrl,
-} from "./testUtils.helper.js";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-const baseUrl = `http://localhost:${port}/api/CortexFileHandler`;
-
-// Helper function to create test files
-async function createTestFile(content, extension) {
-  const testDir = path.join(__dirname, "test-files");
-  if (!fs.existsSync(testDir)) {
-    fs.mkdirSync(testDir, { recursive: true });
-  }
-  const filename = path.join(testDir, `${uuidv4()}.${extension}`);
-  fs.writeFileSync(filename, content);
-  return filename;
-}
-
-// Mock context for testing
-const mockContext = {
-  log: (message) => console.log(`[TEST] ${message}`),
-  res: null,
-};
-
-// Setup: Create test directory and start server
-test.before(async (t) => {
-  await startTestServer();
-  await setupTestDirectory(t);
-});
-
-// Cleanup
-test.after.always(async (t) => {
-  await stopTestServer();
-
-  // Clean up test directory
-  if (t.context?.testDir) {
-    await fs.promises.rm(t.context.testDir, { recursive: true, force: true });
-  }
-
-  // Clean up any remaining files in the test-files directory
-  const testFilesDir = path.join(__dirname, "test-files");
-  if (fs.existsSync(testFilesDir)) {
-    try {
-      await fs.promises.rm(testFilesDir, { recursive: true, force: true });
-    } catch (error) {
-      console.log("Error cleaning test files:", error);
-    }
-  }
-});
-
-// Test container parameter validation
-test("should validate container names correctly", (t) => {
-  // Get current container names
-  const currentContainers = AZURE_STORAGE_CONTAINER_NAMES;
-  
-  // Test with valid container names from configuration
-  currentContainers.forEach(containerName => {
-    t.true(isValidContainerName(containerName), `${containerName} should be valid`);
-  });
-  
-  // Test with invalid container names
-  const invalidNames = ["invalid-container", "", null, undefined, "nonexistent"];
-  invalidNames.forEach(name => {
-    t.false(isValidContainerName(name), `${name} should be invalid`);
-  });
-});
-
-// Test uploadBlob function with container parameter
-test("uploadBlob should accept and use container parameter from function parameter", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  // Create a test file
-  const testFile = await createTestFile("test content", "txt");
-  const testStream = fs.createReadStream(testFile);
-  
-  // Mock request with container parameter in function call
-  const mockReq = {
-    headers: { "content-type": "application/octet-stream" },
-  };
-
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
-  try {
-    // Call uploadBlob with container parameter
-    const result = await uploadBlob(
-      mockContext,
-      mockReq,
-      false, // saveToLocal
-      testFile, // filePath
-      null, // hash
-      "test2" // container parameter
-    );
-
-    t.truthy(result);
-    t.truthy(result.url || mockContext.res?.body?.url);
-
-    // Cleanup
-    const uploadedUrl = result.url || mockContext.res?.body?.url;
-    if (uploadedUrl) {
-      const folderName = getFolderNameFromUrl(uploadedUrl);
-      await cleanupHashAndFile(null, uploadedUrl, baseUrl);
-    }
-  } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-    
-    // Cleanup test file
-    if (fs.existsSync(testFile)) {
-      fs.unlinkSync(testFile);
-    }
-  }
-});
-
-// Test uploadBlob function with container parameter in form data
-test("uploadBlob should accept container parameter from form data", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
-  try {
-    // Create a test file
-    const testContent = "test content for form data";
-    const testFile = await createTestFile(testContent, "txt");
-
-    // Create form data with container parameter
-    const form = new FormData();
-    form.append("file", fs.createReadStream(testFile), "test.txt");
-    form.append("container", "test3");
-
-    const response = await axios.post(baseUrl, form, {
-      headers: {
-        ...form.getHeaders(),
-        "Content-Type": "multipart/form-data",
-      },
-      validateStatus: (status) => true,
-      timeout: 30000,
-    });
-
-    t.is(response.status, 200);
-    t.truthy(response.data.url);
-
-    // Cleanup
-    await cleanupHashAndFile(null, response.data.url, baseUrl);
-  } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
-});
-
-// Test invalid container name in form data
-test("uploadBlob should reject invalid container names in form data", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  // Create a test file
-  const testContent = "test content";
-  const testFile = await createTestFile(testContent, "txt");
-
-  try {
-    // Create form data with invalid container parameter
-    const form = new FormData();
-    form.append("file", fs.createReadStream(testFile), "test.txt");
-    form.append("container", "invalid-container-name");
-
-    const response = await axios.post(baseUrl, form, {
-      headers: {
-        ...form.getHeaders(),
-        "Content-Type": "multipart/form-data",
-      },
-      validateStatus: (status) => true,
-      timeout: 30000,
-    });
-
-    t.is(response.status, 400);
-    t.truthy(response.data);
-    t.true(
-      response.data.includes("Invalid container name")
-    );
-  } finally {
-    // Cleanup test file
-    if (fs.existsSync(testFile)) {
-      fs.unlinkSync(testFile);
-    }
-  }
-});
-
-// Test container parameter flow through index.js
-test("CortexFileHandler should pass container parameter for remote file downloads", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
-  try {
-    // Create a test file URL (this would typically be a remote URL)
-    const testUrl = "https://example.com/test.txt";
-    
-    // Mock request for remote file with container parameter
-    const mockReq = {
-      query: {
-        fetch: testUrl,
-        container: "test2"
-      },
-      method: "GET"
-    };
-
-    // Since we can't easily test real remote downloads in unit tests,
-    // we'll test the parameter extraction and validation
-    const { container } = mockReq.query;
-    
-    t.is(container, "test2");
-    t.true(isValidContainerName(container));
-  } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
-});
-
-// Test container parameter flow for document processing
-test("CortexFileHandler should pass container parameter for document processing", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
-  try {
-    // Mock request for document processing with container parameter
-    const mockReq = {
-      body: {
-        params: {
-          uri: "https://example.com/test.pdf",
-          requestId: "test-request",
-          container: "test3"
-        }
-      },
-      query: {},
-      method: "GET"
-    };
-
-    // Extract parameters like index.js does
-    const {
-      uri,
-      requestId,
-      container,
-    } = mockReq.body?.params || mockReq.query;
-
-    t.is(uri, "https://example.com/test.pdf");
-    t.is(requestId, "test-request");
-    t.is(container, "test3");
-    t.true(isValidContainerName(container));
-  } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
-});
-
-// Test default container behavior
-test("should use default container when no container specified", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  // Test that default container is first in the list
-  const defaultContainer = getDefaultContainerName();
-  t.is(defaultContainer, AZURE_STORAGE_CONTAINER_NAMES[0]);
-  t.true(isValidContainerName(defaultContainer));
-
-  // Create a test file
-  const testContent = "test content for default container";
-  const testFile = await createTestFile(testContent, "txt");
-
-  try {
-    // Create form data without container parameter (should use default)
-    const form = new FormData();
-    form.append("file", fs.createReadStream(testFile), "test.txt");
-
-    const response = await axios.post(baseUrl, form, {
-      headers: {
-        ...form.getHeaders(),
-        "Content-Type": "multipart/form-data",
-      },
-      validateStatus: (status) => true,
-      timeout: 30000,
-    });
-
-    t.is(response.status, 200);
-    t.truthy(response.data.url);
-
-    // Cleanup
-    await cleanupHashAndFile(null, response.data.url, baseUrl);
-  } finally {
-    // Cleanup test file
-    if (fs.existsSync(testFile)) {
-      fs.unlinkSync(testFile);
-    }
-  }
-});
-
-// Test container parameter with media chunking
-test("should pass container parameter for media file chunking", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
-  try {
-    // Mock request for media file processing with container parameter
-    const mockReq = {
-      body: {
-        params: {
-          uri: "https://example.com/test.mp3",
-          requestId: "test-media-request",
-          container: "test1"
-        }
-      },
-      query: {},
-      method: "GET"
-    };
-
-    // Extract parameters like index.js does
-    const {
-      uri,
-      requestId,
-      container,
-    } = mockReq.body?.params || mockReq.query;
-
-    t.is(uri, "https://example.com/test.mp3");
-    t.is(requestId, "test-media-request");
-    t.is(container, "test1");
-    t.true(isValidContainerName(container));
-  } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
-});
\ No newline at end of file
diff --git a/helper-apps/cortex-file-handler/tests/deleteOperations.test.js b/helper-apps/cortex-file-handler/tests/deleteOperations.test.js
index 101202bb..16dbe110 100644
--- a/helper-apps/cortex-file-handler/tests/deleteOperations.test.js
+++ b/helper-apps/cortex-file-handler/tests/deleteOperations.test.js
@@ -581,24 +581,18 @@ test.serial("should not log 'does not exist' when legacy key doesn't exist", asy
     uploadResponse = await uploadFile(filePath, null, testHash);
     t.is(uploadResponse.status, 200, "Upload should succeed");
 
-    // Verify only scoped key exists
-    const { getFileStoreMap, getScopedHashKey } = await import("../src/redis.js");
-    const { getDefaultContainerName } = await import("../src/constants.js");
-    const defaultContainer = getDefaultContainerName();
-    const scopedHash = getScopedHashKey(testHash, defaultContainer);
-    const scopedExists = await getFileStoreMap(scopedHash);
-    const legacyExists = await getFileStoreMap(testHash);
-    t.truthy(scopedExists, "Scoped key should exist");
-    t.falsy(legacyExists, "Legacy key should not exist");
+    // Verify hash exists (no scoping - just hash directly)
+    const { getFileStoreMap } = await import("../src/redis.js");
+    const hashExists = await getFileStoreMap(testHash);
+    t.truthy(hashExists, "Hash should exist");
 
-    // Delete file - should not try to remove non-existent legacy key
-    // (This test verifies the fix doesn't log "does not exist" unnecessarily)
+    // Delete file
     const deleteResponse = await deleteFileByHash(testHash);
     t.is(deleteResponse.status, 200, "Delete should succeed");
 
-    // Verify scoped key is removed
-    const scopedAfter = await getFileStoreMap(scopedHash);
-    t.falsy(scopedAfter, "Scoped key should be removed");
+    // Verify hash is removed
+    const hashAfter = await getFileStoreMap(testHash);
+    t.falsy(hashAfter, "Hash should be removed");
 
   } finally {
     fs.unlinkSync(filePath);
diff --git a/helper-apps/cortex-file-handler/tests/hashContainerScoping.test.js b/helper-apps/cortex-file-handler/tests/hashContainerScoping.test.js
deleted file mode 100644
index e7a5c9c1..00000000
--- a/helper-apps/cortex-file-handler/tests/hashContainerScoping.test.js
+++ /dev/null
@@ -1,415 +0,0 @@
-import test from "ava";
-import axios from "axios";
-import FormData from "form-data";
-import fs from "fs";
-import os from "os";
-import path from "path";
-import { v4 as uuidv4 } from "uuid";
-import { port } from "../src/start.js";
-import { startTestServer } from "./testUtils.helper.js";
-
-// Test server setup
-let baseUrl;
-let server;
-
-// Start test server before running tests
-test.before(async (t) => {
-  baseUrl = `http://localhost:${port}/api/CortexFileHandler`;
-
-  // Start the test server
-  server = await startTestServer();
-});
-
-// Clean up server after tests
-test.after.always(async (t) => {
-  if (server) {
-    await new Promise((resolve, reject) => {
-      server.close((err) => {
-        if (err) reject(err);
-        else resolve();
-      });
-    });
-  }
-});
-
-// Helper to create a test file
-async function createTestFile(content, extension = "txt", filename = null) {
-  const tempDir = os.tmpdir();
-  const actualFilename = filename || `test-${uuidv4()}.${extension}`;
-  const filePath = path.join(tempDir, actualFilename);
-  
-  if (extension === "txt") {
-    fs.writeFileSync(filePath, content);
-  } else {
-    throw new Error(`Unsupported file extension: ${extension}`);
-  }
-  
-  return filePath;
-}
-
-// Helper to upload a file with hash and container
-async function uploadFileWithHashAndContainer(filePath, hash, containerName) {
-  const form = new FormData();
-  // Append hash and container BEFORE file so they're processed first
-  form.append("hash", hash);
-  if (containerName) {
-    form.append("container", containerName);
-  }
-  form.append("file", fs.createReadStream(filePath));
-
-  const response = await axios.post(baseUrl, form, {
-    headers: {
-      ...form.getHeaders(),
-      "Content-Type": "multipart/form-data",
-    },
-    validateStatus: (status) => true,
-    timeout: 10000,
-  });
-
-  return response;
-}
-
-// Helper to check if hash exists with optional container
-async function checkHashExists(hash, containerName = null) {
-  const params = {
-    hash,
-    checkHash: true,
-  };
-  
-  if (containerName) {
-    params.container = containerName;
-  }
-
-  const response = await axios.get(baseUrl, {
-    params,
-    validateStatus: (status) => true,
-    timeout: 10000,
-  });
-
-  return response;
-}
-
-// Helper to cleanup hash
-async function cleanupHash(hash, containerName = null) {
-  const params = {
-    hash,
-    clearHash: true,
-  };
-  
-  if (containerName) {
-    params.container = containerName;
-  }
-
-  try {
-    await axios.get(baseUrl, {
-      params,
-      validateStatus: (status) => true,
-      timeout: 5000,
-    });
-  } catch (error) {
-    // Ignore cleanup errors
-  }
-}
-
-// Main test: Hash scoping across containers
-test.serial("should scope hash by container - same hash different containers should be independent", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
-  try {
-    const testHash = `hash-scope-test-${uuidv4()}`;
-    const contentA = "Content for container A";
-    const contentB = "Content for container B";
-    const fileA = await createTestFile(contentA, "txt", "fileA.txt");
-    const fileB = await createTestFile(contentB, "txt", "fileB.txt");
-
-    // Upload file to container test1 with hash
-    const uploadA = await uploadFileWithHashAndContainer(fileA, testHash, "test1");
-    t.is(uploadA.status, 200, "Upload to test1 should succeed");
-    t.truthy(uploadA.data.url, "Upload A should have URL");
-
-    // Wait for Redis to update
-    await new Promise((resolve) => setTimeout(resolve, 1000));
-
-    // Upload file to container test2 with SAME hash
-    const uploadB = await uploadFileWithHashAndContainer(fileB, testHash, "test2");
-    t.is(uploadB.status, 200, "Upload to test2 should succeed");
-    t.truthy(uploadB.data.url, "Upload B should have URL");
-
-    // Wait for Redis to update
-    await new Promise((resolve) => setTimeout(resolve, 1000));
-
-    // Check hash in container test1 - should return file A
-    const checkA = await checkHashExists(testHash, "test1");
-    t.is(checkA.status, 200, "Hash should exist in test1");
-    t.is(checkA.data.url, uploadA.data.url, "Should return URL from container test1");
-
-    // Check hash in container test2 - should return file B
-    const checkB = await checkHashExists(testHash, "test2");
-    t.is(checkB.status, 200, "Hash should exist in test2");
-    t.is(checkB.data.url, uploadB.data.url, "Should return URL from container test2");
-
-    // Verify the URLs are different
-    t.not(checkA.data.url, checkB.data.url, "URLs should be different for same hash in different containers");
-
-    // Verify the file contents are different
-    const fileResponseA = await axios.get(uploadA.data.url, {
-      validateStatus: (status) => true,
-      timeout: 5000,
-    });
-    const fileResponseB = await axios.get(uploadB.data.url, {
-      validateStatus: (status) => true,
-      timeout: 5000,
-    });
-    
-    t.is(fileResponseA.data, contentA, "File A should have correct content");
-    t.is(fileResponseB.data, contentB, "File B should have correct content");
-
-    // Cleanup
-    fs.unlinkSync(fileA);
-    fs.unlinkSync(fileB);
-    await cleanupHash(testHash, "test1");
-    await cleanupHash(testHash, "test2");
-
-    // Delete the actual files
-    await axios.delete(baseUrl, {
-      params: {
-        hash: testHash,
-        container: "test1",
-      },
-      validateStatus: (status) => true,
-    });
-    await axios.delete(baseUrl, {
-      params: {
-        hash: testHash,
-        container: "test2",
-      },
-      validateStatus: (status) => true,
-    });
-  } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
-});
-
-// Test: Hash in default container should be scoped with container name
-test.serial("should scope hash for default container with container name", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  const originalDefaultEnv = process.env.DEFAULT_AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-  // Ensure test1 is the default container
-  delete process.env.DEFAULT_AZURE_STORAGE_CONTAINER_NAME;
-
-  try {
-    const testHash = `hash-default-test-${uuidv4()}`;
-    const content = "Content for default container";
-    const file = await createTestFile(content, "txt", "fileDefault.txt");
-
-    // Upload file to default container (test1) with hash
-    // We upload WITHOUT specifying container, so it should use default
-    // Now it will be stored as hash:test1 (always scoped)
-    const uploadDefault = await uploadFileWithHashAndContainer(file, testHash, null);
-    t.is(uploadDefault.status, 200, "Upload to default should succeed");
-    t.truthy(uploadDefault.data.url, "Upload should have URL");
-
-    // Wait for Redis to update
-    await new Promise((resolve) => setTimeout(resolve, 1000));
-
-    // Check hash without container parameter - should work for default container
-    const checkWithoutContainer = await checkHashExists(testHash, null);
-    t.is(checkWithoutContainer.status, 200, "Hash should exist without container param");
-    t.is(checkWithoutContainer.data.url, uploadDefault.data.url, "Should return URL from default container");
-
-    // Check hash with explicit default container parameter - should also work
-    const checkWithDefaultContainer = await checkHashExists(testHash, "test1");
-    t.is(checkWithDefaultContainer.status, 200, "Hash should exist with default container param");
-    t.is(checkWithDefaultContainer.data.url, uploadDefault.data.url, "Should return same URL with default container param");
-
-    // Cleanup
-    fs.unlinkSync(file);
-    await cleanupHash(testHash, null);
-
-    // Delete the actual file
-    await axios.delete(baseUrl, {
-      params: {
-        hash: testHash,
-      },
-      validateStatus: (status) => true,
-    });
-  } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-    if (originalDefaultEnv) {
-      process.env.DEFAULT_AZURE_STORAGE_CONTAINER_NAME = originalDefaultEnv;
-    } else {
-      delete process.env.DEFAULT_AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
-});
-
-// Test: Backwards compatibility - legacy hash without container should be found for default container
-test.serial("should support backwards compatibility for legacy hashes in default container", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  const originalDefaultEnv = process.env.DEFAULT_AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-  // Ensure test1 is the default container for backwards compatibility logic
-  delete process.env.DEFAULT_AZURE_STORAGE_CONTAINER_NAME;
-
-  try {
-    const testHash = `hash-legacy-test-${uuidv4()}`;
-    const content = "Content for legacy test";
-    const file = await createTestFile(content, "txt", "fileLegacy.txt");
-
-    // Upload file to default container (test1) with hash - this creates the scoped entry
-    const upload = await uploadFileWithHashAndContainer(file, testHash, "test1");
-    t.is(upload.status, 200, "Upload to test1 should succeed");
-    t.truthy(upload.data.url, "Upload should have URL");
-
-    // Wait for Redis to update
-    await new Promise((resolve) => setTimeout(resolve, 1000));
-
-    // Now simulate a legacy entry by also storing the hash WITHOUT container scope
-    // This mimics the old behavior before container scoping was added
-    const { client } = await import("../src/redis.js");
-    const legacyData = {
-      url: upload.data.url, // Use the real uploaded URL
-      blobName: upload.data.blobName || upload.data.filename, // Include blobName for proper restoration
-      filename: upload.data.filename,
-      timestamp: new Date().toISOString(),
-    };
-    await client.hset("FileStoreMap", testHash, JSON.stringify(legacyData));
-
-    // Wait for Redis to update
-    await new Promise((resolve) => setTimeout(resolve, 500));
-
-    // Delete the scoped key to simulate only having the legacy entry
-    const { getScopedHashKey } = await import("../src/redis.js");
-    const scopedKey = getScopedHashKey(testHash, "test1");
-    await client.hdel("FileStoreMap", scopedKey);
-
-    // Wait a bit for Redis
-    await new Promise((resolve) => setTimeout(resolve, 500));
-
-    // Check hash with default container parameter - should find the legacy entry
-    const checkWithDefaultContainer = await checkHashExists(testHash, "test1");
-    t.is(checkWithDefaultContainer.status, 200, "Legacy hash should be found with default container param");
-    t.is(checkWithDefaultContainer.data.url, legacyData.url, "Should return URL from legacy entry");
-
-    // Check hash without container parameter - should also find the legacy entry
-    const checkWithoutContainer = await checkHashExists(testHash, null);
-    t.is(checkWithoutContainer.status, 200, "Legacy hash should be found without container param");
-    t.is(checkWithoutContainer.data.url, legacyData.url, "Should return URL from legacy entry");
-
-    // After migration, the new scoped key should exist
-    const { getFileStoreMap } = await import("../src/redis.js");
-    const migratedValue = await getFileStoreMap(scopedKey, true); // Skip lazy cleanup
-    t.truthy(migratedValue, "Migrated value should exist with new scoped key");
-    t.is(migratedValue.url, legacyData.url, "Migrated value should have same URL");
-
-    // Cleanup
-    fs.unlinkSync(file);
-    await cleanupHash(testHash, "test1");
-    await cleanupHash(testHash, null);
-
-    // Delete the actual file
-    await axios.delete(baseUrl, {
-      params: {
-        hash: testHash,
-        container: "test1",
-      },
-      validateStatus: (status) => true,
-    });
-  } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-    if (originalDefaultEnv) {
-      process.env.DEFAULT_AZURE_STORAGE_CONTAINER_NAME = originalDefaultEnv;
-    } else {
-      delete process.env.DEFAULT_AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
-});
-
-// Test: Hash check with wrong container should return 404
-test.serial("should return 404 when checking hash with wrong container", async (t) => {
-  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
-    t.pass("Skipping test - Azure not configured");
-    return;
-  }
-
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
-
-  try {
-    const testHash = `hash-wrong-container-test-${uuidv4()}`;
-    const content = "Content for specific container";
-    const file = await createTestFile(content, "txt", "fileWrong.txt");
-
-    // Upload file to container test1 with hash
-    const upload = await uploadFileWithHashAndContainer(file, testHash, "test1");
-    t.is(upload.status, 200, "Upload to test1 should succeed");
-
-    // Wait for Redis to update
-    await new Promise((resolve) => setTimeout(resolve, 1000));
-
-    // Check hash in container test2 (wrong container) - should return 404
-    const checkWrong = await checkHashExists(testHash, "test2");
-    t.is(checkWrong.status, 404, "Hash should not exist in test2");
-
-    // Check hash in container test3 (also wrong) - should return 404
-    const checkWrong2 = await checkHashExists(testHash, "test3");
-    t.is(checkWrong2.status, 404, "Hash should not exist in test3");
-
-    // Check hash in container test1 (correct container) - should return 200
-    const checkCorrect = await checkHashExists(testHash, "test1");
-    t.is(checkCorrect.status, 200, "Hash should exist in test1");
-
-    // Cleanup
-    fs.unlinkSync(file);
-    await cleanupHash(testHash, "test1");
-
-    // Delete the actual file
-    await axios.delete(baseUrl, {
-      params: {
-        hash: testHash,
-        container: "test1",
-      },
-      validateStatus: (status) => true,
-    });
-  } finally {
-    // Restore environment
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
-});
-
diff --git a/helper-apps/cortex-file-handler/tests/setRetention.test.js b/helper-apps/cortex-file-handler/tests/setRetention.test.js
new file mode 100644
index 00000000..e0492657
--- /dev/null
+++ b/helper-apps/cortex-file-handler/tests/setRetention.test.js
@@ -0,0 +1,533 @@
+import test from "ava";
+import axios from "axios";
+import FormData from "form-data";
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { fileURLToPath } from "url";
+import { v4 as uuidv4 } from "uuid";
+
+import { port } from "../src/start.js";
+import { 
+  startTestServer, 
+  stopTestServer, 
+  setupTestDirectory 
+} from "./testUtils.helper.js";
+import { 
+  getFileStoreMap, 
+  removeFromFileStoreMap,
+  getScopedHashKey
+} from "../src/redis.js";
+import { getDefaultContainerName } from "../src/constants.js";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const baseUrl = `http://localhost:${port}/api/CortexFileHandler`;
+
+// Helper function to create test files
+async function createTestFile(content, extension) {
+  const testDir = path.join(__dirname, "test-files");
+  if (!fs.existsSync(testDir)) {
+    fs.mkdirSync(testDir, { recursive: true });
+  }
+  const filename = path.join(
+    testDir,
+    `test-retention-${uuidv4().slice(0, 8)}.${extension}`,
+  );
+  fs.writeFileSync(filename, content);
+  return filename;
+}
+
+// Helper function to upload file with hash and container
+async function uploadFile(filePath, hash = null, containerName = null) {
+  const form = new FormData();
+  form.append("file", fs.createReadStream(filePath));
+  if (hash) form.append("hash", hash);
+  if (containerName) form.append("container", containerName);
+
+  return await axios.post(baseUrl, form, {
+    headers: form.getHeaders(),
+    validateStatus: (status) => true,
+    timeout: 15000,
+  });
+}
+
+// Helper function to check if hash exists
+async function checkHashExists(hash, containerName = null) {
+  const params = { hash, checkHash: true };
+  if (containerName) {
+    params.container = containerName;
+  }
+  return await axios.get(baseUrl, {
+    params,
+    validateStatus: (status) => true,
+    timeout: 10000,
+  });
+}
+
+// Helper function to set retention
+async function setRetention(hash, retention, useBody = false) {
+  if (useBody) {
+    const body = { hash, retention, setRetention: true };
+    return await axios.post(baseUrl, body, {
+      validateStatus: (status) => true,
+      timeout: 30000,
+    });
+  } else {
+    const params = { hash, retention, setRetention: true };
+    return await axios.post(baseUrl, null, {
+      params,
+      validateStatus: (status) => true,
+      timeout: 30000,
+    });
+  }
+}
+
+// Test setup
+test.before(async (t) => {
+  await setupTestDirectory(t, "test-files");
+  await startTestServer();
+});
+
+test.after(async (t) => {
+  await stopTestServer();
+});
+
+// Basic retention tests
+test.serial("should set file retention to permanent", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const testContent = "test content for retention operation";
+  const testHash = `test-retention-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file (defaults to temporary)
+    uploadResponse = await uploadFile(filePath, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+    t.truthy(uploadResponse.data.url, "Should have file URL");
+    const originalUrl = uploadResponse.data.url;
+
+    // Wait for Redis to update
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Set retention to permanent
+    const retentionResponse = await setRetention(testHash, "permanent");
+    t.is(retentionResponse.status, 200, "Set retention should succeed");
+    t.is(retentionResponse.data.retention, "permanent", "Should have retention set to permanent");
+    t.is(retentionResponse.data.url, originalUrl, "URL should remain the same");
+    t.truthy(retentionResponse.data.shortLivedUrl, "Should have shortLivedUrl");
+    t.truthy(retentionResponse.data.message, "Should have success message");
+
+    // Wait for operations to complete
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Verify file still exists and is accessible
+    const checkAfter = await checkHashExists(testHash);
+    t.is(checkAfter.status, 200, "File should still exist after setting retention");
+    t.is(checkAfter.data.url, originalUrl, "URL should still match");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    // Cleanup
+    try {
+      const { getScopedHashKey } = await import("../src/redis.js");
+      const container = getDefaultContainerName();
+      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should set file retention to temporary", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const testContent = "test content for temporary retention";
+  const testHash = `test-retention-temp-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file (defaults to temporary)
+    uploadResponse = await uploadFile(filePath, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+    const originalUrl = uploadResponse.data.url;
+
+    // Wait for Redis to update
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // First set to permanent
+    await setRetention(testHash, "permanent");
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Then set back to temporary
+    const retentionResponse = await setRetention(testHash, "temporary");
+    t.is(retentionResponse.status, 200, "Set retention should succeed");
+    t.is(retentionResponse.data.retention, "temporary", "Should have retention set to temporary");
+    t.is(retentionResponse.data.url, originalUrl, "URL should remain the same");
+    t.truthy(retentionResponse.data.shortLivedUrl, "Should have shortLivedUrl");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    // Cleanup
+    try {
+      const { getScopedHashKey } = await import("../src/redis.js");
+      const container = getDefaultContainerName();
+      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should set retention using request body parameters", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const testContent = "test content for body retention";
+  const testHash = `test-retention-body-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file
+    uploadResponse = await uploadFile(filePath, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+
+    // Wait for Redis to update
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Set retention using body parameters
+    const retentionResponse = await setRetention(testHash, "permanent", true);
+    t.is(retentionResponse.status, 200, "Set retention should succeed");
+    t.is(retentionResponse.data.retention, "permanent", "Should have retention set to permanent");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    // Cleanup
+    try {
+      const { getScopedHashKey } = await import("../src/redis.js");
+      const container = getDefaultContainerName();
+      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should return 400 when hash is missing", async (t) => {
+  const retentionResponse = await axios.post(baseUrl, {
+    retention: "permanent",
+    setRetention: true,
+  }, {
+    validateStatus: (status) => true,
+    timeout: 10000,
+  });
+  t.is(retentionResponse.status, 400, "Should return 400 for missing hash");
+  t.truthy(retentionResponse.data.includes("hash"), "Error message should mention hash");
+});
+
+test.serial("should return 400 when retention is missing", async (t) => {
+  const testHash = `test-retention-${uuidv4()}`;
+  const retentionResponse = await axios.post(baseUrl, {
+    hash: testHash,
+    setRetention: true,
+  }, {
+    validateStatus: (status) => true,
+    timeout: 10000,
+  });
+  t.is(retentionResponse.status, 400, "Should return 400 for missing retention");
+  t.truthy(retentionResponse.data.includes("retention"), "Error message should mention retention");
+});
+
+test.serial("should return 400 when retention value is invalid", async (t) => {
+  const testHash = `test-retention-${uuidv4()}`;
+  const retentionResponse = await axios.post(baseUrl, {
+    hash: testHash,
+    retention: "invalid",
+    setRetention: true,
+  }, {
+    validateStatus: (status) => true,
+    timeout: 10000,
+  });
+  t.is(retentionResponse.status, 400, "Should return 400 for invalid retention");
+  t.truthy(
+    retentionResponse.data.includes("temporary") || retentionResponse.data.includes("permanent"),
+    "Error message should mention valid retention values"
+  );
+});
+
+test.serial("should return 404 when file hash not found", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const nonExistentHash = `non-existent-${uuidv4()}`;
+  const retentionResponse = await setRetention(nonExistentHash, "permanent");
+  t.is(retentionResponse.status, 404, "Should return 404 for non-existent hash");
+  t.truthy(retentionResponse.data.includes("not found"), "Error message should indicate file not found");
+});
+
+test.serial("should update Redis map with retention information", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const testContent = "test content for Redis map update";
+  const testHash = `test-retention-redis-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file
+    uploadResponse = await uploadFile(filePath, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+
+    // Wait for Redis to update
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Verify Redis entry exists
+    const container = getDefaultContainerName();
+    const scopedHash = getScopedHashKey(testHash, container);
+    const oldEntry = await getFileStoreMap(scopedHash);
+    t.truthy(oldEntry, "Redis entry should exist before setting retention");
+
+    // Set retention
+    const retentionResponse = await setRetention(testHash, "permanent");
+    t.is(retentionResponse.status, 200, "Set retention should succeed");
+
+    // Wait for operations to complete
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Verify Redis entry is updated
+    const newEntry = await getFileStoreMap(scopedHash);
+    t.truthy(newEntry, "Redis entry should still exist after setting retention");
+    t.is(newEntry.url, retentionResponse.data.url, "Entry should have correct URL");
+    t.truthy(newEntry.shortLivedUrl, "Entry should have shortLivedUrl");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    // Cleanup
+    try {
+      const { getScopedHashKey } = await import("../src/redis.js");
+      const container = getDefaultContainerName();
+      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should preserve file metadata after setting retention", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const testContent = "test content for metadata preservation";
+  const testHash = `test-retention-metadata-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file
+    uploadResponse = await uploadFile(filePath, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+    const originalFilename = uploadResponse.data.filename;
+    const originalUrl = uploadResponse.data.url;
+
+    // Wait for Redis to update
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Set retention
+    const retentionResponse = await setRetention(testHash, "permanent");
+    t.is(retentionResponse.status, 200, "Set retention should succeed");
+    t.is(retentionResponse.data.hash, testHash, "Hash should be preserved");
+    t.is(retentionResponse.data.filename, originalFilename, "Filename should be preserved");
+    t.is(retentionResponse.data.url, originalUrl, "URL should remain the same");
+
+    // Wait for operations to complete
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Verify metadata is preserved
+    const checkAfter = await checkHashExists(testHash);
+    t.is(checkAfter.status, 200, "File should still exist");
+    t.is(checkAfter.data.hash, testHash, "Hash should match");
+    t.is(checkAfter.data.filename, originalFilename, "Filename should match");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    // Cleanup
+    try {
+      const { getScopedHashKey } = await import("../src/redis.js");
+      const container = getDefaultContainerName();
+      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should support operation=setRetention query parameter", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const testContent = "test content for operation parameter";
+  const testHash = `test-retention-operation-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file
+    uploadResponse = await uploadFile(filePath, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+
+    // Wait for Redis to update
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Set retention using operation=setRetention query parameter
+    const retentionResponse = await axios.post(baseUrl, null, {
+      params: {
+        operation: "setRetention",
+        hash: testHash,
+        retention: "permanent",
+      },
+      validateStatus: (status) => true,
+      timeout: 30000,
+    });
+    t.is(retentionResponse.status, 200, "Set retention should succeed");
+    t.is(retentionResponse.data.retention, "permanent", "Should have retention set to permanent");
+
+    // Wait for operations to complete
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Verify file still exists
+    const checkAfter = await checkHashExists(testHash);
+    t.is(checkAfter.status, 200, "File should still exist after setting retention");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    // Cleanup
+    try {
+      const { getScopedHashKey } = await import("../src/redis.js");
+      const container = getDefaultContainerName();
+      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should preserve GCS URL when setting retention", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  // Skip if GCS is not configured
+  if (!process.env.GCP_SERVICE_ACCOUNT_KEY && !process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64) {
+    t.pass("Skipping test - GCS not configured");
+    return;
+  }
+
+  const testContent = "test content for GCS preservation";
+  const testHash = `test-retention-gcs-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file
+    uploadResponse = await uploadFile(filePath, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+    t.truthy(uploadResponse.data.url, "Should have Azure URL");
+    t.truthy(uploadResponse.data.gcs, "Should have GCS URL");
+    
+    const originalGcsUrl = uploadResponse.data.gcs;
+    const originalAzureUrl = uploadResponse.data.url;
+
+    // Wait for Redis to update
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Set retention
+    const retentionResponse = await setRetention(testHash, "permanent");
+    t.is(retentionResponse.status, 200, "Set retention should succeed");
+    
+    // Verify GCS URL is preserved
+    t.is(retentionResponse.data.gcs, originalGcsUrl, "GCS URL should be preserved");
+    
+    // Verify Azure URL remains the same (no container change)
+    t.is(retentionResponse.data.url, originalAzureUrl, "Azure URL should remain the same");
+
+    // Wait for operations to complete
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Verify GCS URL is still preserved in checkHash response
+    const checkAfter = await checkHashExists(testHash);
+    t.is(checkAfter.status, 200, "File should still exist");
+    t.is(checkAfter.data.gcs, originalGcsUrl, "GCS URL should still be preserved");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    // Cleanup
+    try {
+      const { getScopedHashKey } = await import("../src/redis.js");
+      const container = getDefaultContainerName();
+      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should always include shortLivedUrl in response", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const testContent = "test content for shortLivedUrl";
+  const testHash = `test-retention-shortlived-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file
+    uploadResponse = await uploadFile(filePath, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+    t.truthy(uploadResponse.data.shortLivedUrl, "Upload response should include shortLivedUrl");
+
+    // Wait for Redis to update
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Set retention
+    const retentionResponse = await setRetention(testHash, "permanent");
+    t.is(retentionResponse.status, 200, "Set retention should succeed");
+    t.truthy(retentionResponse.data.shortLivedUrl, "Retention response should include shortLivedUrl");
+    t.truthy(retentionResponse.data.url, "Should have regular URL");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    // Cleanup
+    try {
+      const { getScopedHashKey } = await import("../src/redis.js");
+      const container = getDefaultContainerName();
+      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
diff --git a/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js b/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js
index 70987add..8228d5e6 100644
--- a/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js
+++ b/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js
@@ -117,36 +117,23 @@ test("should get azure provider with default container when no container specifi
   t.truthy(provider.containerName);
 });
 
-test("should get azure provider with specific container name", async (t) => {
+test("should get azure provider (container parameter ignored)", async (t) => {
   if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
     t.pass("Skipping test - Azure not configured");
     return;
   }
 
-  // Save original env value
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
+  const factory = new StorageFactory();
   
-  try {
-    // Set test container names in environment
-    process.env.AZURE_STORAGE_CONTAINER_NAME = "container1,container2,container3";
-    
-    const factory = new StorageFactory();
-    
-    // Test with valid container name
-    const provider = await factory.getAzureProvider("container2");
-    t.truthy(provider);
-    t.is(provider.containerName, "container2");
-  } finally {
-    // Restore original env
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
+  // Container parameter is ignored - always uses default container from env
+  const provider = await factory.getAzureProvider("any-container-name");
+  t.truthy(provider);
+  // Should use the default container from env, not the parameter
+  const { getContainerName } = await import("../../src/constants.js");
+  t.is(provider.containerName, getContainerName());
 });
 
-test("should throw error for invalid container name", async (t) => {
+test("should ignore container parameter and use default container", async (t) => {
   if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
     t.pass("Skipping test - Azure not configured");
     return;
@@ -154,44 +141,28 @@ test("should throw error for invalid container name", async (t) => {
 
   const factory = new StorageFactory();
   
-  // Test with invalid container name
-  await t.throwsAsync(
-    () => factory.getAzureProvider("invalid-container"),
-    { message: /Invalid container name/ }
-  );
+  // Container parameter is ignored - always uses default container
+  const provider1 = await factory.getAzureProvider("invalid-container");
+  const provider2 = await factory.getAzureProvider();
+  
+  // Both should return the same provider instance (same default container)
+  t.is(provider1, provider2);
 });
 
-test("should cache providers by container name", async (t) => {
+test("should cache provider instance (single container)", async (t) => {
   if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
     t.pass("Skipping test - Azure not configured");
     return;
   }
 
-  // Save original env value
-  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
+  const factory = new StorageFactory();
   
-  try {
-    // Set test container names in environment
-    process.env.AZURE_STORAGE_CONTAINER_NAME = "container1,container2,container3";
-    
-    const factory = new StorageFactory();
-    
-    const provider1 = await factory.getAzureProvider("container1");
-    const provider2 = await factory.getAzureProvider("container1");
-    const provider3 = await factory.getAzureProvider("container2");
-    
-    // Same container should return same instance
-    t.is(provider1, provider2);
-    // Different container should return different instance
-    t.not(provider1, provider3);
-    t.is(provider1.containerName, "container1");
-    t.is(provider3.containerName, "container2");
-  } finally {
-    // Restore original env
-    if (originalEnv) {
-      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-    } else {
-      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-    }
-  }
+  // All calls should return the same provider instance (single container)
+  const provider1 = await factory.getAzureProvider();
+  const provider2 = await factory.getAzureProvider();
+  const provider3 = await factory.getAzureProvider("ignored-container");
+  
+  // All should return the same instance
+  t.is(provider1, provider2);
+  t.is(provider1, provider3);
 });
diff --git a/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js b/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js
index f9dadd58..6f1e9e4a 100644
--- a/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js
+++ b/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js
@@ -348,8 +348,8 @@ test("should handle delete file by hash with empty URL in Redis", async (t) => {
   }
 });
 
-// Container-specific tests
-test("should upload file with specific container name", async (t) => {
+// Container-specific tests - now using single container only
+test("should upload file using default container", async (t) => {
   if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
     t.pass("Skipping test - Azure not configured");
     return;
@@ -364,33 +364,20 @@ test("should upload file with specific container name", async (t) => {
   fs.writeFileSync(testFile, "test content");
   
   try {
-    // Mock environment to have multiple containers
-    const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-    process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
+    // Test upload - container parameter is ignored, always uses default
+    const result = await service.uploadFileWithProviders(
+      { log: () => {} }, // mock context
+      testFile,
+      "test-request",
+      null,
+      null
+    );
     
-    try {
-      // Test upload with specific container
-      const result = await service.uploadFileWithProviders(
-        { log: () => {} }, // mock context
-        testFile,
-        "test-request",
-        null,
-        "test2"
-      );
-      
-      t.truthy(result.url);
-      t.truthy(result.url.includes("test2") || result.url.includes("/test2/"));
-      
-      // Cleanup
-      await service.deleteFiles("test-request");
-    } finally {
-      // Restore original env
-      if (originalEnv) {
-        process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-      } else {
-        delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-      }
-    }
+    t.truthy(result.url);
+    t.truthy(result.shortLivedUrl);
+    
+    // Cleanup
+    await service.deleteFiles("test-request");
   } finally {
     // Cleanup temp file
     fs.rmSync(tempDir, { recursive: true, force: true });
@@ -431,7 +418,7 @@ test("should use default container when no container specified", async (t) => {
   }
 });
 
-test("should pass container parameter through uploadFile method", async (t) => {
+test("should upload file using uploadFile method (container parameter ignored)", async (t) => {
   if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
     t.pass("Skipping test - Azure not configured");
     return;
@@ -446,32 +433,20 @@ test("should pass container parameter through uploadFile method", async (t) => {
   fs.writeFileSync(testFile, "test content");
   
   try {
-    // Mock environment to have multiple containers
-    const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
-    process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
+    // Test upload using the uploadFile method - container parameter is ignored
+    const result = await service.uploadFile(
+      { log: () => {} }, // context
+      testFile,         // filePath
+      "test-request",   // requestId
+      null,             // hash
+      null              // filename (containerName parameter removed)
+    );
     
-    try {
-      // Test upload using the uploadFile method with container parameter
-      const result = await service.uploadFile(
-        { log: () => {} }, // context
-        testFile,         // filePath
-        "test-request",   // requestId
-        null,             // hash
-        "test3"           // containerName
-      );
-      
-      t.truthy(result.url);
-      
-      // Cleanup
-      await service.deleteFiles("test-request");
-    } finally {
-      // Restore original env
-      if (originalEnv) {
-        process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
-      } else {
-        delete process.env.AZURE_STORAGE_CONTAINER_NAME;
-      }
-    }
+    t.truthy(result.url);
+    t.truthy(result.shortLivedUrl);
+    
+    // Cleanup
+    await service.deleteFiles("test-request");
   } finally {
     // Cleanup temp file
     fs.rmSync(tempDir, { recursive: true, force: true });

From 4fa307d3537be30de8fac1c18afbf2eb6463319c Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Thu, 11 Dec 2025 20:59:13 -0700
Subject: [PATCH 02/27] refactor: enhance Redis client implementation with
 improved error handling

- Updated Redis client initialization to use `ioredis` for better error management and connection handling.
- Implemented a retry strategy with exponential backoff for connection attempts.
- Added event listeners for connection status updates and error logging to prevent process crashes.
- Maintained compatibility with existing mock client functionality.
---
 helper-apps/cortex-file-handler/src/redis.js | 76 ++++++++++++++++++--
 1 file changed, 69 insertions(+), 7 deletions(-)

diff --git a/helper-apps/cortex-file-handler/src/redis.js b/helper-apps/cortex-file-handler/src/redis.js
index 40a1ad9d..125804a1 100644
--- a/helper-apps/cortex-file-handler/src/redis.js
+++ b/helper-apps/cortex-file-handler/src/redis.js
@@ -1,4 +1,4 @@
-import redis from "ioredis";
+import Redis from "ioredis";
 import { getDefaultContainerName } from "./constants.js";
 
 const connectionString = process.env["REDIS_CONNECTION_STRING"];
@@ -76,7 +76,55 @@ const createMockClient = () => {
 // Only create real Redis client if connection string is provided
 let client;
 if (connectionString && process.env.NODE_ENV !== 'test') {
-  client = redis.createClient(connectionString);
+  // ioredis client with explicit error handling to avoid:
+  //   [ioredis] Unhandled error event: Error: read ETIMEDOUT
+  //
+  // This Redis usage is a cache / coordination layer for the file-handler.
+  // It should degrade gracefully when Redis is unavailable.
+  const retryStrategy = (times) => {
+    // Exponential backoff: 100ms, 200ms, 400ms... up to 30s
+    const delay = Math.min(100 * Math.pow(2, times), 30000);
+    // After ~10 attempts, stop retrying (prevents tight reconnect loops forever).
+    if (times > 10) {
+      console.error(
+        `[redis] Connection failed after ${times} attempts. Stopping retries.`,
+      );
+      return null;
+    }
+    console.warn(
+      `[redis] Connection retry attempt ${times}, waiting ${delay}ms`,
+    );
+    return delay;
+  };
+
+  client = new Redis(connectionString, {
+    retryStrategy,
+    enableReadyCheck: true,
+    connectTimeout: 10000,
+    // If Redis is down, don't indefinitely queue cache operations in memory.
+    // We'll catch and log failures at call sites instead.
+    enableOfflineQueue: false,
+    // Fail fast on commands during connection issues.
+    maxRetriesPerRequest: 1,
+  });
+
+  // IMPORTANT: prevent process crashes on connection errors
+  client.on("error", (error) => {
+    const code = error?.code ? ` (${error.code})` : "";
+    console.error(`[redis] Client error${code}: ${error?.message || error}`);
+  });
+  client.on("connect", () => {
+    console.log("[redis] Connected");
+  });
+  client.on("ready", () => {
+    console.log("[redis] Ready");
+  });
+  client.on("close", () => {
+    console.warn("[redis] Connection closed");
+  });
+  client.on("reconnecting", (delay) => {
+    console.warn(`[redis] Reconnecting in ${delay}ms`);
+  });
 } else {
   console.log('Using mock Redis client for tests or missing connection string');
   client = createMockClient();
@@ -85,13 +133,27 @@ if (connectionString && process.env.NODE_ENV !== 'test') {
 const channel = "requestProgress";
 
 const connectClient = async () => {
-  if (!client.connected) {
-    try {
-      await client.connect();
-    } catch (error) {
-      console.error(`Error reconnecting to Redis: ${error}`);
+  // ioredis connects automatically; this function is kept for backwards
+  // compatibility and for the mock client.
+  try {
+    // Mock client uses `connected`; ioredis uses `status`.
+    if (typeof client?.connected === "boolean") {
+      if (!client.connected && typeof client.connect === "function") {
+        await client.connect();
+      }
       return;
     }
+
+    // ioredis states: "wait" | "connecting" | "connect" | "ready" | "close" | "end"
+    if (client?.status && client.status !== "ready") {
+      // If the caller explicitly wants to ensure connectivity, we can ping.
+      // If Redis is down, ping will throw and we handle it.
+      await client.ping();
+    }
+  } catch (error) {
+    console.error(
+      `[redis] Not ready (status=${client?.status || "unknown"}): ${error?.message || error}`,
+    );
   }
 };
 

From 3a062b251e8533b05a8d8c5916f0bdbcd3647395 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Thu, 11 Dec 2025 21:03:47 -0700
Subject: [PATCH 03/27] refactor: enhance file handling and model selection
 capabilities

- Updated the README to clarify model selection mechanisms, detailing static model selection and dynamic runtime overrides.
- Improved file handling functions to support optional container parameters for file deletion and uploads, allowing for better organization in cloud storage.
- Introduced a new tool for step-by-step planning, enhancing the system's ability to manage complex tasks.
- Added functionality for short-lived URLs in file handling, ensuring efficient access to files while maintaining security.
- Removed deprecated reasoning tool and streamlined image viewing capabilities, improving overall system performance and usability.
- Enhanced tests for file handling and short-lived URL functionality, ensuring robust error handling and accurate responses.
---
 README.md                                     | 140 +++++++++-
 lib/entityConstants.js                        |  89 ++----
 lib/fileUtils.js                              | 168 ++++++++----
 lib/pathwayTools.js                           |   8 +
 lib/redisSubscription.js                      |  45 ++-
 lib/requestExecutor.js                        |  38 ++-
 pathways/system/entity/sys_entity_agent.js    |  86 ++++--
 .../entity/tools/sys_tool_file_collection.js  |  23 +-
 .../system/entity/tools/sys_tool_planner.js   |  59 ++++
 .../system/entity/tools/sys_tool_reasoning.js |  80 ------
 .../entity/tools/sys_tool_view_image.js       | 115 ++++++++
 server/modelExecutor.js                       |   4 +
 .../plugins/gemini3ReasoningVisionPlugin.js   | 192 +++++++++++++
 .../features/tools/fileCollection.test.js     |   2 +
 tests/unit/core/shortLivedUrl.test.js         | 257 ++++++++++++++++++
 tests/unit/core/util.test.js                  |   2 +
 16 files changed, 1085 insertions(+), 223 deletions(-)
 create mode 100644 pathways/system/entity/tools/sys_tool_planner.js
 delete mode 100644 pathways/system/entity/tools/sys_tool_reasoning.js
 create mode 100644 pathways/system/entity/tools/sys_tool_view_image.js
 create mode 100644 server/plugins/gemini3ReasoningVisionPlugin.js
 create mode 100644 tests/unit/core/shortLivedUrl.test.js

diff --git a/README.md b/README.md
index 252faff9..cb9bce74 100644
--- a/README.md
+++ b/README.md
@@ -427,10 +427,144 @@ Each pathway can define the following properties (with defaults from basePathway
 - `json`: Require valid JSON response from model. Default: false
 - `manageTokenLength`: Manage input token length for model. Default: true
   
-#### Dynamic model override
+#### Model Overrides
 
-- `model`: In many cases, specifying the model as an input parameter will tell the pathway which model to use when setting up the pathway for execution.
-- `modelOverride`: In some cases, you need even more dynamic model selection. At runtime, a pathway can optionally specify `modelOverride` in request args to switch the model used for execution without restarting the server. Cortex will attempt a hot swap and continue execution; errors are logged gracefully if the model is invalid.
+Cortex provides two mechanisms for specifying which model to use: static model selection (via `model`) and dynamic runtime model override (via `modelOverride`).
+
+##### Static Model Selection (`model`)
+
+The `model` parameter can be specified in multiple ways, and Cortex follows this order of precedence when selecting a model at pathway initialization:
+
+1. `pathway.model` - The model specified directly in the pathway definition
+2. `args.model` - The model passed in the request arguments
+3. `pathway.inputParameters.model` - The model specified in the pathway's input parameters
+4. `config.get('defaultModelName')` - The default model specified in the configuration
+
+The first valid model found in this order will be used. If none of these models are found in the configured endpoints, Cortex will log a warning and use the default model defined in the configuration.
+
+**Example:**
+```js
+export default {
+    model: 'oai-gpt4o',  // Static model for this pathway
+    prompt: '{{text}}',
+    // ...
+};
+```
+
+##### Runtime Model Override (`modelOverride`)
+
+The `modelOverride` parameter enables dynamic model switching at runtime, after the pathway has been initialized. This is useful when:
+
+- You need to switch models based on runtime conditions
+- Different parts of a pathway should use different models
+- You want to implement model fallback strategies
+- You need to test different models without restarting the server
+
+**How it works:**
+
+1. The pathway is initialized with a model using the static selection precedence above
+2. During execution, if `modelOverride` is specified in the request args and differs from the current model, Cortex performs a "hot swap"
+3. The `swapModel()` method updates the model reference, creates a new `ModelExecutor` instance, and recalculates token limits
+4. Execution continues with the new model
+5. If the override model is invalid, an error is logged gracefully and execution continues with the original model
+
+**Implementation details:**
+
+The model swap occurs in the `promptAndParse()` method of `PathwayResolver`:
+
+```649:666:server/pathwayResolver.js
+    swapModel(newModelName) {
+        // Validate that the new model exists in endpoints
+        if (!this.endpoints[newModelName]) {
+            throw new Error(`Model ${newModelName} not found in config`);
+        }
+
+        // Update model references
+        this.modelName = newModelName;
+        this.model = this.endpoints[newModelName];
+
+        // Create new ModelExecutor with the new model
+        this.modelExecutor = new ModelExecutor(this.pathway, this.model);
+
+        // Recalculate chunk max token length as it depends on the model
+        this.chunkMaxTokenLength = this.getChunkMaxTokenLength();
+
+        this.logWarning(`Model swapped to ${newModelName}`);
+    }
+```
+
+**Usage examples:**
+
+1. **In a pathway's `executePathway` function:**
+```js
+export default {
+    model: 'oai-gpt4o',
+    executePathway: async ({args, runAllPrompts}) => {
+        // Switch to a different model based on input length
+        if (args.text && args.text.length > 10000) {
+            args.modelOverride = 'oai-gpt4-turbo';  // Use faster model for long text
+        }
+        return await runAllPrompts();
+    }
+};
+```
+
+2. **In a pathway that calls other pathways:**
+```js
+export default {
+    executePathway: async ({args, runAllPrompts}) => {
+        // First pass with one model
+        const initialResult = await runAllPrompts();
+        
+        // Second pass with a different model
+        args.modelOverride = 'oai-gpt4o';
+        args.text = initialResult;
+        return await runAllPrompts();
+    }
+};
+```
+
+3. **Conditional model selection:**
+```js
+export default {
+    executePathway: async ({args, runAllPrompts}) => {
+        // Select model based on language or complexity
+        if (args.language === 'ja' || args.complexity === 'high') {
+            args.modelOverride = 'oai-gpt4o';
+        } else {
+            args.modelOverride = 'oai-gpt4-turbo';
+        }
+        return await runAllPrompts();
+    }
+};
+```
+
+**Error handling:**
+
+If `modelOverride` specifies a model that doesn't exist in the configured endpoints, Cortex will:
+- Log an error message: `Failed to swap model to {modelName}: {error message}`
+- Continue execution with the originally selected model
+- Not throw an exception that would stop pathway execution
+
+**When to use `model` vs `modelOverride`:**
+
+- Use `model` when:
+  - The model selection is known at pathway definition time
+  - The pathway always uses the same model
+  - You want the model to be part of the pathway's configuration
+
+- Use `modelOverride` when:
+  - The model needs to change based on runtime conditions
+  - Different parts of execution need different models
+  - You're implementing model fallback or A/B testing
+  - The model selection depends on input characteristics (length, language, complexity, etc.)
+
+**Important notes:**
+
+- `modelOverride` only takes effect if it differs from the currently selected model
+- The swap happens before prompt execution, so all subsequent prompts in the pathway will use the new model
+- Token limits are automatically recalculated after a model swap to account for different model capabilities
+- Model swaps are logged as warnings for debugging purposes
 
 ## Core (Default) Pathways
 
diff --git a/lib/entityConstants.js b/lib/entityConstants.js
index a7482359..f6e1eacf 100644
--- a/lib/entityConstants.js
+++ b/lib/entityConstants.js
@@ -23,63 +23,37 @@ Your responses should be in {{language}} unless the user has expressed another p
 
     AI_TOOLS: `# Tool Instructions
 
-- You have an extensive toolkit. Each time you call tool(s) you will get the result(s), evaluate, decide what's next, and chain as many steps as needed.
-- Your tools work most efficiently when called in parallel so if you know you will need multiple tool calls and you know what the parameters are, call them in parallel.
+- Your tools work most efficiently when called in parallel so if you know you will need multiple tool calls try to call them in parallel where possible.
 - Always honor user requests to use specific tools.
-- For data processing requests (e.g. tell me how many articles were published in the last 30 days), or deep file analysis (chart the trends in this spreadsheet, etc.), you should call your code execution tool to perform the task - especially if the task requires a lot of data, deep analysis, complex filtering, or precision calculations. For simpler things (e.g. make me a chart of the population of the world in the last 100 years) you might find it faster to search for the data and then just call your charting tool to generate the chart.
 - You must always search if you are being asked questions about current events, news, fact-checking, or information requiring citation.
-- Do not make up information - if information cannot be confirmed with rigorous logic or reliable sources, do not include it in your response.
-- Start searches broad and consult multiple sources, running all searches in parallel to save time.
-- Consult all available sources and cross-reference with specific searches before responding.
+- Do not make up, hallucinate, or fabricate information - if information cannot be confirmed with rigorous logic or direct sources, do not include it in your response.
 - If a tool fails or has a technical difficulty, try to fix the problem or call a different or backup tool before giving up or reporting the error.
-- Don't settle for the first plausible answer—dig until the response is complete, corroborated, and clear.
+- Don't settle for the first plausible answer — dig until the data is complete, corroborated, and clear.
 - Deliver concise, well-structured responses with complete citations.
 - Double-check accuracy, coherence, and alignment with the user request.
-- Charts and Diagrams - you can generate most charts using your charting tool. Always use a tool to generate charts and diagrams rather than trying to do it yourself as the tools do validation for you. If you need to generate a more complex chart or do data analysis or visualization work, you should call your code execution tool to generate the chart.
+- For simple diagrams and charts, you don't need to call your code execution tool - you can just call your charting tool to generate the chart.
+- For data processing requests (e.g. tell me how many articles were published in the last 30 days), or deep file analysis (chart the trends in this spreadsheet, etc.), you should call your code execution tool to perform the task - especially if the task requires a lot of data, deep analysis, complex filtering, or precision calculations.
+- For research problems or multi-step tasks that require careful planning and sequencing of multiple tool calls, use the CreatePlan tool to develop an optimal step-by-step plan before executing.
 `,            
 
-    AI_SEARCH_RULES: `# News Search Protocol
-When searching for news, you must complete the following steps:
-
-1. Triangulate
-   - Run multiple, parallel queries across all applicable sources.
-   - Request at least double the number of results you want to share, then select the best results.
-   - Confirm that multiple sources tell the same story.
-
-2. Check Freshness
-   - Confirm the publication date.
-   - Apply date filters to surface the most recent credible material.
-
-# Internet Search Protocol
-
-Before you share online information with the user, you MUST complete all of the steps below:
-
-1. Triangulate  
-   - Run multiple, parallel queries across reputable outlets.  
-   - Confirm that independent sources tell the same story.
-
-2. Verify  
-   - Treat social / monetized platforms (YouTube, X, TikTok, Instagram, Reddit, etc.) as unverified tips only unless there is strong evidence that the information is credible and reliable.  
-   - Corroborate every claim from those platforms with at least one authoritative source.
-
-3. Check Freshness  
-   - Confirm the publication date.  
-   - Apply date filters to surface the most recent credible material.
-
-4. Read, don't skim  
-   - For high-stakes, complex, or time-sensitive topics, use your tools toopen and read the full article or document.  
-   - Never rely solely on snippets, headlines, or auto-generated summaries.
+    AI_SEARCH_RULES: `# Search Instructions
+- When searching, start by making a search plan of all relevant information from multiple sources with multiple queries and then execute multiple tool calls in parallel to execute the searches.
+- Keep searching until you have all the information you need - adjust the plan as needed at every step.
+- If you don't get good results from one query or source, vary the query terms and try different approaches - e.g. broadening the date range or searching for a related set of terms.
+- Confirm that multiple sources tell the same story.
+- Search the same sources multiple times with different terms to get a complete picture.
+- Confirm the publication date.
+- Apply date filters to surface the most recent credible material.
+- If the results are relevant, but not complete, try a different search with different terms.
+
+# Web / Internet / Social searches
+- for news: include explicit date/timeframe and geography for targeted, current coverage (“US news headlines August 20 2025”). Use “summary,” “overview,” “trends,” or “breaking/latest” to control breadth and recency
+- for non-news/company/tech: specify the aspect or attribute needed (“technology overview,” “funding history,” “competitor analysis”), add output preferences (“in bullet points,” “detailed review”), and include date/context for freshness (“2025,” “latest update”)
+- for social and monetized platforms (YouTube, TikTok, Instagram, Reddit, etc.) - try to corroborate the information with multiple posts or at least one authoritative source
+- for high-stakes, complex, or time-sensitive topics, never rely on snippets or summaries - always use your tools to open and read the full article or document
 `,
 
-    AI_SEARCH_SYNTAX: `# Internet Search Tool
-
-When using the internet search tool, always tailor your prompt for specificity and depth.  
-- For news: Include explicit date/timeframe and geography for targeted, current coverage (“US news headlines August 20 2025”). Use “summary,” “overview,” “trends,” or “breaking/latest” to control breadth and recency.  
-- For non-news/company/tech: Specify the aspect or attribute needed (“technology overview,” “funding history,” “competitor analysis”), add output preferences (“in bullet points,” “detailed review”), and include date/context for freshness (“2025,” “latest update”).  
-- For high-stakes queries: Run parallel, focused searches on different facets. Always disambiguate terms and clarify ambiguous subjects.  
-Avoid generic queries — precise, context-rich searches return the most relevant, accurate results.
-
-# AI Search Syntax
+    AI_SEARCH_SYNTAX: `# AI Search Syntax
 
 When creating a query string for your index-based search tools, you can use the following AI Search syntax. Important: these tools do not support AND, OR, or NOT strings as operators - you MUST use the syntax below.  E.g. you cannot use "term1 AND term2", you must use "term1 + term2".
 
@@ -103,24 +77,19 @@ term~N                 (Match terms similar to "term", edit distance N)
 
     AI_MEMORY_INSTRUCTIONS: `# Memory Instructions
 
-You have a memory system that contains important details, instructions, and context. Consult your memories when formulating a response to ensure your answers reflect previous learnings and context.
-
-The Preloaded Memories are not your complete memory system. If you don't see the information you need in there, or need more details, call your SearchMemory tool to search the rest of your memory system.
-
-It's critical that you never fabricate or miss existing memories. Everything that you say you remember must be backed by a Preloaded Memory or a SearchMemory result without exception.
-
-Your memories may also contain details about the user to help personalize responses. You do not need to include the user's name or personal information in every reply—only when relevant to the conversation.
-
-When sharing information from memory, state it naturally (e.g., 'I remember...'); never refer to the memory structure or technical details.
-
-Privacy is critical. If asked to forget or delete something, always comply affirmatively. If there is user information in your memories you have talked to this user before.
+- You have a memory system that contains important details, instructions, and context. Consult your memories when formulating a response to ensure your answers reflect previous learnings and context.
+- The Preloaded Memories are not your complete memory system. If you don't see the information you need in there, or need more details, call your SearchMemory tool to search the rest of your memory system.
+- It's critical that you never fabricate or miss existing memories. Everything that you say you remember must be backed by a Preloaded Memory or a SearchMemory result without exception.
+- Your memories may also contain details about the user to help personalize responses. You do not need to include the user's name or personal information in every reply—only when relevant to the conversation.
+- When sharing information from memory, state it naturally (e.g., 'I remember...'); never refer to the memory structure or technical details.
+- Privacy is critical. If asked to forget or delete something, always comply affirmatively. If there is user information in your memories you have talked to this user before.
 `,
 
     AI_MEMORY: "# Preloaded Memories\n\n## Self\n{{{memorySelf}}}\n\n## User\n{{{memoryUser}}}\n\n## Directives\n{{{memoryDirectives}}}\n\n## Topics\n{{{memoryTopics}}}",
 
     AI_MEMORY_CONTEXT: "## Contextual\n{{{memoryContext}}}",
 
-    AI_DATETIME: "# Time, Date, and Time Zone\n\nThe current time and date in GMT is {{now}}. {{#if userInfo}}The user's time information is: {{{userInfo}}}, so all references like like \"today\" or \"yesterday\" are relative to that time. {{/if}}Temporal relevance is critical - it's important to ground your thinking and responses in the current date and time - things may have changed since your training cutoff date - so always search for the most current information when you need to.",
+    AI_DATETIME: "# Time, Date, and Time Zone\n\nThe current time and date in GMT is {{now}}, but this may be different from the user's time zone. {{#if userInfo}}The user's time information is: {{{userInfo}}}, so all references like like \"today\" or \"yesterday\" are relative to that time. {{/if}}Temporal relevance is critical - it's important to ground your thinking and responses in the current date and time - things may have changed since your training cutoff date - so always search for the most current information when you need to.",
 
     AI_STYLE_OPENAI: "oai-gpt5-chat",
     AI_STYLE_OPENAI_RESEARCH: "oai-gpt5",
diff --git a/lib/fileUtils.js b/lib/fileUtils.js
index ab30564e..88c30897 100644
--- a/lib/fileUtils.js
+++ b/lib/fileUtils.js
@@ -180,9 +180,10 @@ async function markCompletedForCleanUp(requestId) {
  * Delete a file from cloud storage by hash
  * @param {string} hash - File hash to delete
  * @param {pathwayResolver} pathwayResolver - Optional pathway resolver for logging
+ * @param {string} container - Optional container name where the file is stored
  * @returns {Promise<boolean>} True if file was deleted, false if not found or error
  */
-async function deleteFileByHash(hash, pathwayResolver = null) {
+async function deleteFileByHash(hash, pathwayResolver = null, container = null) {
     if (!hash || typeof hash !== 'string') {
         logger.warn('deleteFileByHash: hash is required and must be a string');
         return false;
@@ -196,7 +197,12 @@ async function deleteFileByHash(hash, pathwayResolver = null) {
 
     try {
         const separator = fileHandlerUrl.includes('?') ? '&' : '?';
-        const deleteUrl = `${fileHandlerUrl}${separator}hash=${encodeURIComponent(hash)}`;
+        let deleteUrl = `${fileHandlerUrl}${separator}hash=${encodeURIComponent(hash)}`;
+        
+        // Add container parameter if provided
+        if (container) {
+            deleteUrl += `&container=${encodeURIComponent(container)}`;
+        }
         
         const response = await axios.delete(deleteUrl, {
             validateStatus: (status) => status >= 200 && status < 500, // Accept 200-499 as valid responses
@@ -595,11 +601,16 @@ async function modifyFileCollectionWithLock(contextId, contextKey, modifierCallb
  * @param {pathwayResolver} pathwayResolver - Optional pathway resolver for logging
  * @returns {Promise<Object>} File entry object with id
  */
-async function addFileToCollection(contextId, contextKey, url, gcs, filename, tags = [], notes = '', hash = null, fileUrl = null, pathwayResolver = null) {
+async function addFileToCollection(contextId, contextKey, url, gcs, filename, tags = [], notes = '', hash = null, fileUrl = null, pathwayResolver = null, permanent = false) {
     if (!contextId || !filename) {
         throw new Error("contextId and filename are required");
     }
 
+    // Determine container based on permanent flag
+    const containerName = permanent && process.env.CORTEX_MEDIA_PERMANENT_STORE_NAME 
+        ? process.env.CORTEX_MEDIA_PERMANENT_STORE_NAME 
+        : null;
+
     // If fileUrl is provided and url is not already a cloud URL, upload the file first
     let finalUrl = url;
     let finalGcs = gcs;
@@ -609,7 +620,7 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
         // Upload the file from the URL
         // uploadFileToCloud will download it, compute hash, check if it exists, and upload if needed
         // It uploads the local file stream, not the URL, to avoid triggering remoteFile fetch
-        const uploadResult = await uploadFileToCloud(fileUrl, null, filename, pathwayResolver);
+        const uploadResult = await uploadFileToCloud(fileUrl, null, filename, pathwayResolver, containerName);
         finalUrl = uploadResult.url;
         finalGcs = uploadResult.gcs;
         finalHash = uploadResult.hash || hash;
@@ -635,6 +646,7 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
         tags: Array.isArray(tags) ? tags : [],
         notes: notes || '',
         hash: finalHash || null,
+        permanent: permanent || false,
         addedDate: new Date().toISOString(),
         lastAccessed: new Date().toISOString()
     };
@@ -1109,12 +1121,15 @@ async function generateFileMessageContent(fileParam, contextId, contextKey = nul
         return null;
     }
 
+    // Resolve to short-lived URL if possible
+    const fileWithShortLivedUrl = await ensureShortLivedUrl(foundFile, MEDIA_API_URL);
+
     return {
         type: 'image_url',
-        url: foundFile.url,
-        gcs: foundFile.gcs || null,
-        originalFilename: foundFile.filename || null,
-        hash: foundFile.hash || null
+        url: fileWithShortLivedUrl.url,
+        gcs: fileWithShortLivedUrl.gcs || null,
+        originalFilename: fileWithShortLivedUrl.filename || null,
+        hash: fileWithShortLivedUrl.hash || null
     };
 
 }
@@ -1176,36 +1191,52 @@ function injectFileIntoChatHistory(chatHistory, fileContent) {
 
 /**
  * Check if a file exists by hash using the file handler
+ * Returns short-lived URL when available, with fallback to regular URL
  * @param {string} hash - File hash to check
  * @param {string} fileHandlerUrl - File handler service URL
  * @param {pathwayResolver} pathwayResolver - Optional pathway resolver for logging
+ * @param {string} container - Optional container name
+ * @param {number} shortLivedMinutes - Optional duration for short-lived URL (default: 5)
  * @returns {Promise<Object|null>} {url, gcs, hash} if file exists, null otherwise
+ *   url: shortLivedUrl if available (prefers converted), otherwise regular URL
+ *   gcs: GCS URL (prefers converted, no short-lived version for GCS)
  */
-async function checkHashExists(hash, fileHandlerUrl, pathwayResolver = null) {
+async function checkHashExists(hash, fileHandlerUrl, pathwayResolver = null, container = null, shortLivedMinutes = 5) {
     if (!hash || !fileHandlerUrl) {
         return null;
     }
     
     try {
         const separator = fileHandlerUrl.includes('?') ? '&' : '?';
-        const checkHashUrl = `${fileHandlerUrl}${separator}hash=${hash}&checkHash=true`;
+        let checkHashUrl = `${fileHandlerUrl}${separator}hash=${hash}&checkHash=true`;
+        
+        // Add container parameter if provided
+        if (container) {
+            checkHashUrl += `&container=${encodeURIComponent(container)}`;
+        }
+        
+        // Request short-lived URL
+        if (shortLivedMinutes) {
+            checkHashUrl += `&shortLivedMinutes=${shortLivedMinutes}`;
+        }
         
         const checkResponse = await axios.get(checkHashUrl, {
             timeout: 10000,
             validateStatus: (status) => status >= 200 && status < 500
         });
         
-        // If file exists (200), return existing URLs
-        // Use converted URLs if available (for converted files like XLSX->CSV, DOCX->TXT, etc.)
+        // If file exists (200), return URLs with short-lived URL preferred
         if (checkResponse.status === 200 && checkResponse.data && checkResponse.data.url) {
             const data = checkResponse.data;
-            // Prefer converted URLs if they exist, otherwise use original URLs
-            const url = data.converted?.url || data.url;
+            // shortLivedUrl automatically prefers converted URL if it exists
+            // Use shortLivedUrl if available, otherwise fall back to regular URL
+            // For GCS, always use the GCS URL from checkHash (no short-lived for GCS)
+            const url = data.shortLivedUrl || data.converted?.url || data.url;
             const gcs = data.converted?.gcs || data.gcs || null;
             
             return {
-                url: url,
-                gcs: gcs,
+                url: url, // shortLivedUrl if available (prefers converted), otherwise regular URL
+                gcs: gcs, // GCS URL (prefers converted, no short-lived version for GCS)
                 hash: data.hash || hash
             };
         }
@@ -1229,6 +1260,40 @@ async function checkHashExists(hash, fileHandlerUrl, pathwayResolver = null) {
     }
 }
 
+/**
+ * Central function to resolve a file object to use short-lived URL when available
+ * This is the single point of logic for ensuring files sent to LLMs use short-lived URLs
+ * @param {Object} fileObject - File object from collection (must have hash and url)
+ * @param {string} fileHandlerUrl - File handler service URL
+ * @param {number} shortLivedMinutes - Optional duration for short-lived URL (default: 5)
+ * @returns {Promise<Object>} File object with url set to shortLivedUrl (or original if not available)
+ */
+async function ensureShortLivedUrl(fileObject, fileHandlerUrl, shortLivedMinutes = 5) {
+    if (!fileObject || !fileObject.hash || !fileHandlerUrl) {
+        // No hash or no file handler - return original object
+        return fileObject;
+    }
+    
+    try {
+        const resolved = await checkHashExists(fileObject.hash, fileHandlerUrl, null, null, shortLivedMinutes);
+        if (resolved && resolved.url) {
+            // Return file object with url replaced by shortLivedUrl (or fallback to regular url)
+            // GCS URL comes from checkHash (no short-lived version for GCS)
+            return {
+                ...fileObject,
+                url: resolved.url, // shortLivedUrl (or fallback)
+                gcs: resolved.gcs || fileObject.gcs || null // GCS from checkHash
+            };
+        }
+    } catch (error) {
+        // If resolution fails, log but return original object
+        logger.warn(`Failed to resolve short-lived URL for file ${fileObject.hash}: ${error.message}`);
+    }
+    
+    // Fallback to original object if resolution fails
+    return fileObject;
+}
+
 /**
  * Generic function to upload a file to cloud storage
  * Handles both URLs (downloads then uploads) and base64 data
@@ -1239,7 +1304,7 @@ async function checkHashExists(hash, fileHandlerUrl, pathwayResolver = null) {
  * @param {pathwayResolver} pathwayResolver - Optional pathway resolver for logging
  * @returns {Promise<Object>} {url, gcs, hash}
  */
-async function uploadFileToCloud(fileInput, mimeType = null, filename = null, pathwayResolver = null) {
+async function uploadFileToCloud(fileInput, mimeType = null, filename = null, pathwayResolver = null, containerName = null) {
     let tempFilePath = null;
     let tempDir = null;
     let fileBuffer = null;
@@ -1308,8 +1373,8 @@ async function uploadFileToCloud(fileInput, mimeType = null, filename = null, pa
         if (fileBuffer) {
             fileHash = await computeBufferHash(fileBuffer);
             
-            // Check if file already exists using checkHash
-            const existingFile = await checkHashExists(fileHash, fileHandlerUrl, pathwayResolver);
+            // Check if file already exists using checkHash (with container if specified)
+            const existingFile = await checkHashExists(fileHash, fileHandlerUrl, pathwayResolver, containerName);
             if (existingFile) {
                 return existingFile;
             }
@@ -1354,6 +1419,10 @@ async function uploadFileToCloud(fileInput, mimeType = null, filename = null, pa
         if (fileHash) {
             formData.append('hash', fileHash);
         }
+        // Add container if specified
+        if (containerName) {
+            formData.append('container', containerName);
+        }
         
         // Append requestId parameter
         const separator = fileHandlerUrl.includes('?') ? '&' : '?';
@@ -1445,40 +1514,42 @@ async function resolveFileHashesToContent(fileHashes, config) {
             const fileHandlerUrl = config?.get?.('whisperMediaApiUrl');
             
             if (fileHandlerUrl && fileHandlerUrl !== 'null') {
-                // Use shared checkHashExists function
+                // Use shared checkHashExists function - it already returns shortLivedUrl in url field
                 const existingFile = await checkHashExists(hash, fileHandlerUrl);
                 if (existingFile) {
-                    const fileData = existingFile;
-                    const fileUrl = fileData.url;
-                    const convertedUrl = fileData.converted?.url;
-                    const convertedGcsUrl = fileData.converted?.gcs;
-                    
-                    return JSON.stringify({
-                        type: "image_url",
-                        url: convertedUrl || fileUrl,
-                        image_url: { url: convertedUrl || fileUrl },
-                        gcs: convertedGcsUrl || fileData.gcs, // Add GCS URL for Gemini models
-                        originalFilename: fileData.filename,
-                        hash: hash
-                    });
-                }
-                
-                // Fallback: try direct axios call for backward compatibility (in case checkHashExists doesn't work)
-                const response = await axios.get(fileHandlerUrl, { 
-                    params: { hash: hash, checkHash: true } 
-                });
-                if (response.status === 200) {
-                    const fileData = response.data;
-                    const fileUrl = fileData.shortLivedUrl || fileData.url;
-                    const convertedUrl = fileData.converted?.url;
-                    const convertedGcsUrl = fileData.converted?.gcs;
+                    // checkHashExists already returns shortLivedUrl (prefers converted) in url field
+                    // and GCS URL (prefers converted) in gcs field
+                    // We need filename from the checkHash response, so make a direct call
+                    try {
+                        const separator = fileHandlerUrl.includes('?') ? '&' : '?';
+                        const checkHashUrl = `${fileHandlerUrl}${separator}hash=${hash}&checkHash=true&shortLivedMinutes=5`;
+                        const response = await axios.get(checkHashUrl, {
+                            timeout: 10000,
+                            validateStatus: (status) => status >= 200 && status < 500
+                        });
+                        
+                        if (response.status === 200 && response.data) {
+                            const data = response.data;
+                            return JSON.stringify({
+                                type: "image_url",
+                                url: data.shortLivedUrl || data.converted?.url || data.url,
+                                image_url: { url: data.shortLivedUrl || data.converted?.url || data.url },
+                                gcs: data.converted?.gcs || data.gcs || null, // GCS from checkHash (no short-lived)
+                                originalFilename: data.filename,
+                                hash: hash
+                            });
+                        }
+                    } catch (error) {
+                        // Fallback to existingFile data if direct call fails
+                    }
                     
+                    // Fallback: use data from checkHashExists
                     return JSON.stringify({
                         type: "image_url",
-                        url: convertedUrl || fileUrl,
-                        image_url: { url: convertedUrl || fileUrl },
-                        gcs: convertedGcsUrl || fileData.gcs, // Add GCS URL for Gemini models
-                        originalFilename: fileData.filename,
+                        url: existingFile.url, // Already has shortLivedUrl
+                        image_url: { url: existingFile.url },
+                        gcs: existingFile.gcs || null, // GCS from checkHash
+                        originalFilename: null,
                         hash: hash
                     });
                 }
@@ -1625,6 +1696,7 @@ export {
     saveFileCollection,
     modifyFileCollectionWithLock,
     checkHashExists,
+    ensureShortLivedUrl,
     uploadFileToCloud,
     uploadImageToCloud,
     resolveFileHashesToContent,
diff --git a/lib/pathwayTools.js b/lib/pathwayTools.js
index 1fbde044..55ba68c9 100644
--- a/lib/pathwayTools.js
+++ b/lib/pathwayTools.js
@@ -136,6 +136,14 @@ const callTool = async (toolName, args, toolDefinitions, pathwayResolver) => {
                 pathwayResolver.searchResults = [];
             }
 
+            // Check if tool result has imageUrl or imageUrls field (for ViewImage/ViewImages tools)
+            if (parsedResult.imageUrl && typeof parsedResult.imageUrl === 'object') {
+                toolImages.push(parsedResult.imageUrl);
+            }
+            if (parsedResult.imageUrls && Array.isArray(parsedResult.imageUrls)) {
+                toolImages.push(...parsedResult.imageUrls);
+            }
+
             // Check if this is a search response
             if (parsedResult._type === "SearchResponse" && Array.isArray(parsedResult.value)) {
                 // Extract and add each search result
diff --git a/lib/redisSubscription.js b/lib/redisSubscription.js
index c8a64dee..4a476db1 100644
--- a/lib/redisSubscription.js
+++ b/lib/redisSubscription.js
@@ -14,21 +14,62 @@ let subscriptionClient;
 let publisherClient;
 
 if (connectionString) {
+    // Configure Redis with exponential backoff retry strategy
+    const retryStrategy = (times) => {
+        // Exponential backoff: 100ms, 200ms, 400ms, 800ms, 1600ms, 3200ms, 6400ms, 12800ms, 25600ms, 30000ms (max)
+        const delay = Math.min(100 * Math.pow(2, times), 30000);
+        // Stop retrying after 10 attempts (about 5 minutes total)
+        if (times > 10) {
+            logger.error(`Redis connection failed after ${times} attempts. Stopping retries.`);
+            return null;
+        }
+        logger.warn(`Redis connection retry attempt ${times}, waiting ${delay}ms before next attempt`);
+        return delay;
+    };
+
+    const redisOptions = {
+        retryStrategy,
+        maxRetriesPerRequest: null, // Allow unlimited retries for connection issues
+        enableReadyCheck: true,
+        lazyConnect: false,
+        connectTimeout: 10000, // 10 second connection timeout
+    };
+
     logger.info(`Using Redis subscription for channel(s) ${requestProgressChannel}, ${requestProgressSubscriptionsChannel}`);
     try {
-        subscriptionClient = connectionString && new Redis(connectionString);
+        subscriptionClient = connectionString && new Redis(connectionString, redisOptions);
+        if (subscriptionClient) {
+            subscriptionClient.on('connect', () => {
+                logger.info('Redis subscription client connected successfully');
+            });
+            subscriptionClient.on('ready', () => {
+                logger.info('Redis subscription client ready');
+            });
+            subscriptionClient.on('reconnecting', (delay) => {
+                logger.info(`Redis subscription client reconnecting in ${delay}ms`);
+            });
+        }
     } catch (error) {
         logger.error(`Redis connection error: ${error}`);
     }
 
     logger.info(`Using Redis publish for channel(s) ${requestProgressChannel}, ${requestProgressSubscriptionsChannel}`);
     try {
-        publisherClient = connectionString && new Redis(connectionString);
+        publisherClient = connectionString && new Redis(connectionString, redisOptions);
         // Handle Redis publisher client errors to prevent crashes
         if (publisherClient) {
             publisherClient.on('error', (error) => {
                 logger.error(`Redis publisherClient error: ${error}`);
             });
+            publisherClient.on('connect', () => {
+                logger.info('Redis publisher client connected successfully');
+            });
+            publisherClient.on('ready', () => {
+                logger.info('Redis publisher client ready');
+            });
+            publisherClient.on('reconnecting', (delay) => {
+                logger.info(`Redis publisher client reconnecting in ${delay}ms`);
+            });
         }
     } catch (error) {
         logger.error(`Redis connection error: ${error}`);
diff --git a/lib/requestExecutor.js b/lib/requestExecutor.js
index 99deac74..514aeac5 100644
--- a/lib/requestExecutor.js
+++ b/lib/requestExecutor.js
@@ -19,11 +19,47 @@ let client;
 
 if (connectionString) {
     try {
-        client = new Redis(connectionString);
+        // Configure Redis with exponential backoff retry strategy
+        const retryStrategy = (times) => {
+            // Exponential backoff: 100ms, 200ms, 400ms, 800ms, 1600ms, 3200ms, 6400ms, 12800ms, 25600ms, 30000ms (max)
+            const delay = Math.min(100 * Math.pow(2, times), 30000);
+            // Stop retrying after 10 attempts (about 5 minutes total)
+            if (times > 10) {
+                logger.error(`Redis connection failed after ${times} attempts. Stopping retries.`);
+                return null;
+            }
+            logger.warn(`Redis connection retry attempt ${times}, waiting ${delay}ms before next attempt`);
+            return delay;
+        };
+
+        client = new Redis(connectionString, {
+            retryStrategy,
+            maxRetriesPerRequest: null, // Allow unlimited retries for connection issues
+            enableReadyCheck: true,
+            lazyConnect: false,
+            connectTimeout: 10000, // 10 second connection timeout
+        });
+        
         // Handle Redis connection errors to prevent crashes
         client.on('error', (error) => {
             logger.error(`Redis client connection error: ${error}`);
         });
+        
+        client.on('connect', () => {
+            logger.info('Redis client connected successfully');
+        });
+        
+        client.on('ready', () => {
+            logger.info('Redis client ready');
+        });
+        
+        client.on('close', () => {
+            logger.warn('Redis client connection closed');
+        });
+        
+        client.on('reconnecting', (delay) => {
+            logger.info(`Redis client reconnecting in ${delay}ms`);
+        });
     } catch (error) {
         logger.error(`Redis connection error: ${error}`);
     }
diff --git a/pathways/system/entity/sys_entity_agent.js b/pathways/system/entity/sys_entity_agent.js
index 26bf4671..e51649a1 100644
--- a/pathways/system/entity/sys_entity_agent.js
+++ b/pathways/system/entity/sys_entity_agent.js
@@ -35,6 +35,27 @@ async function generateErrorResponse(error, args, pathwayResolver) {
     }
 }
 
+// Helper function to insert a system message, removing any existing ones first
+function insertSystemMessage(messages, text, requestId = null) {
+    // Create a unique marker to avoid collisions with legitimate content
+    const marker = requestId ? `[system message: ${requestId}]` : '[system message]';
+    
+    // Remove any existing challenge messages with this specific requestId to avoid spamming the model
+    const filteredMessages = messages.filter(msg => {
+        if (msg.role !== 'user') return true;
+        const content = typeof msg.content === 'string' ? msg.content : '';
+        return !content.startsWith(marker);
+    });
+    
+    // Insert the new system message
+    filteredMessages.push({
+        role: "user",
+        content: `${marker} ${text}`
+    });
+    
+    return filteredMessages;
+}
+
 export default {
     emulateOpenAIChatModel: 'cortex-agent',
     useInputChunking: false,
@@ -84,7 +105,7 @@ export default {
         pathwayResolver.toolCallCount = (pathwayResolver.toolCallCount || 0);
         
         const preToolCallMessages = JSON.parse(JSON.stringify(args.chatHistory || []));
-        const finalMessages = JSON.parse(JSON.stringify(preToolCallMessages));
+        let finalMessages = JSON.parse(JSON.stringify(preToolCallMessages));
 
         if (tool_calls && tool_calls.length > 0) {
             if (pathwayResolver.toolCallCount < MAX_TOOL_CALLS) {
@@ -161,7 +182,7 @@ export default {
                             content: toolResultContent
                         });
 
-                        // Add the screenshots using OpenAI image format
+                        // Add the screenshots/images using OpenAI image format
                         if (toolResult?.toolImages && toolResult.toolImages.length > 0) {
                             toolMessages.push({
                                 role: "user",
@@ -170,12 +191,35 @@ export default {
                                         type: "text",
                                         text: "The tool with id " + toolCall.id + " has also supplied you with these images."
                                     },
-                                    ...toolResult.toolImages.map(toolImage => ({
-                                        type: "image_url",
-                                        image_url: {
-                                            url: `data:image/png;base64,${toolImage}`
+                                    ...toolResult.toolImages.map(toolImage => {
+                                        // Handle both base64 strings (screenshots) and image_url objects (file collection images)
+                                        if (typeof toolImage === 'string') {
+                                            // Base64 string format (screenshots)
+                                            return {
+                                                type: "image_url",
+                                                image_url: {
+                                                    url: `data:image/png;base64,${toolImage}`
+                                                }
+                                            };
+                                        } else if (typeof toolImage === 'object' && toolImage.image_url) {
+                                            // Image URL object format (file collection images)
+                                            return {
+                                                type: "image_url",
+                                                url: toolImage.url,
+                                                gcs: toolImage.gcs,
+                                                image_url: toolImage.image_url,
+                                                originalFilename: toolImage.originalFilename
+                                            };
+                                        } else {
+                                            // Fallback for any other format
+                                            return {
+                                                type: "image_url",
+                                                image_url: {
+                                                    url: toolImage.url || toolImage
+                                                }
+                                            };
                                         }
-                                    }))
+                                    })
                                 ]
                             });
                         }
@@ -306,26 +350,22 @@ export default {
                     return toolDefinition?.handoff === true;
                 });
 
-                // Inject oversight message after tools are executed to encourage task completion
+                // Inject challenge message after tools are executed to encourage task completion
                 // Skip this check if a hand-off tool was used (async agents handle their own completion)
                 if (!hasHandoffTool) {
-                    finalMessages.push({
-                        role: "user",
-                        content: "[System: Task Completion Check] Please evaluate whether you have completed your task based on the tool results you just received. If the task is not yet complete, you should call additional tools as needed to finish the work. Only respond to the user when the task is fully complete or you have gathered all necessary information. If you need to read more of a file, search for more information, or perform additional operations, do so now before responding."
-                    });
+                    const requestId = pathwayResolver.rootRequestId || pathwayResolver.requestId;
+                    finalMessages = insertSystemMessage(finalMessages, 
+                        "Review the tool results above. If your task is incomplete or requires additional steps or information, call the necessary tools now. Adapt your approach and re-plan if needed. Only respond to the user once the task is complete and all required information has been gathered.",
+                        requestId
+                    );
                 }
 
             } else {
-                finalMessages.push({
-                    role: "user",
-                    content: [
-                        {
-                            type: "text",
-                            text: "[System: Tool Limit Reached] This agent has reached the maximum number of tool calls - no more tool calls will be executed."
-                        }
-                    ]
-                });
-                
+                const requestId = pathwayResolver.rootRequestId || pathwayResolver.requestId;
+                finalMessages = insertSystemMessage(finalMessages,
+                    "Maximum tool call limit reached - no more tool calls will be executed. Provide your response based on the information gathered so far.",
+                    requestId
+                );
             }
 
             args.chatHistory = finalMessages;
@@ -464,7 +504,7 @@ export default {
         const styleConfig = styleModelMap[aiStyle] || styleModelMap["OpenAI"]; // Default to OpenAI
         const styleModel = researchMode ? styleConfig.research : styleConfig.normal;
         // Use 'high' reasoning effort in research mode for thorough analysis, 'none' in normal mode for faster responses
-        const reasoningEffort = researchMode ? 'high' : 'none';
+        const reasoningEffort = researchMode ? 'high' : 'low';
 
         // Limit the chat history to 20 messages to speed up processing
         if (args.messages && args.messages.length > 0) {
diff --git a/pathways/system/entity/tools/sys_tool_file_collection.js b/pathways/system/entity/tools/sys_tool_file_collection.js
index f9811d89..304af65b 100644
--- a/pathways/system/entity/tools/sys_tool_file_collection.js
+++ b/pathways/system/entity/tools/sys_tool_file_collection.js
@@ -46,6 +46,10 @@ export default {
                             type: "string",
                             description: "Optional: File hash for deduplication and identification (usually computed automatically during upload)"
                         },
+                        permanent: {
+                            type: "boolean",
+                            description: "Optional: If true, the file will be stored indefinitely instead of being subject to the default 30 day storage limit. Default: false"
+                        },
                         userMessage: {
                             type: "string",
                             description: "A user-friendly message that describes what you're doing with this tool"
@@ -158,7 +162,7 @@ export default {
 
             if (isAdd) {
                 // Add file to collection
-                const { fileUrl, url, gcs, filename, tags = [], notes = '', hash = null } = args;
+                const { fileUrl, url, gcs, filename, tags = [], notes = '', hash = null, permanent = false } = args;
                 
                 if (!filename) {
                     throw new Error("filename is required");
@@ -179,7 +183,8 @@ export default {
                     notes,
                     hash,
                     fileUrl,
-                    resolver
+                    resolver,
+                    permanent
                 );
 
                 resolver.tool = JSON.stringify({ toolUsed: "AddFileToCollection" });
@@ -339,12 +344,13 @@ export default {
                 const fileIdsToRemove = new Set(filesToRemove.map(f => f.id));
                 const hashesToDelete = [];
                 const finalCollection = await modifyFileCollectionWithLock(contextId, contextKey, (collection) => {
-                    // Capture hashes of files that will be removed (at current lock time)
+                    // Capture hashes and container info of files that will be removed (at current lock time)
                     collection.forEach(file => {
                         if (fileIdsToRemove.has(file.id) && file.hash) {
                             hashesToDelete.push({
                                 hash: file.hash,
-                                filename: file.filename || 'unknown'
+                                filename: file.filename || 'unknown',
+                                permanent: file.permanent || false
                             });
                         }
                     });
@@ -357,10 +363,15 @@ export default {
                 // We do this after updating collection so user gets fast response and files are "gone" from UI immediately
                 // Use hashes captured inside the lock to ensure we delete the correct files
                 (async () => {
+                    const { config } = await import('../../../../config.js');
+                    const permanentContainerName = process.env.CORTEX_MEDIA_PERMANENT_STORE_NAME;
+                    
                     for (const fileInfo of hashesToDelete) {
                         try {
-                            logger.info(`Deleting file from cloud storage: ${fileInfo.filename} (hash: ${fileInfo.hash})`);
-                            await deleteFileByHash(fileInfo.hash, resolver);
+                            // Determine container based on permanent flag
+                            const container = fileInfo.permanent && permanentContainerName ? permanentContainerName : null;
+                            logger.info(`Deleting file from cloud storage: ${fileInfo.filename} (hash: ${fileInfo.hash}${container ? `, container: ${container}` : ''})`);
+                            await deleteFileByHash(fileInfo.hash, resolver, container);
                         } catch (error) {
                             logger.warn(`Failed to delete file ${fileInfo.filename} (hash: ${fileInfo.hash}) from cloud storage: ${error?.message || String(error)}`);
                         }
diff --git a/pathways/system/entity/tools/sys_tool_planner.js b/pathways/system/entity/tools/sys_tool_planner.js
new file mode 100644
index 00000000..d266fa35
--- /dev/null
+++ b/pathways/system/entity/tools/sys_tool_planner.js
@@ -0,0 +1,59 @@
+// sys_tool_planner.js
+// Entity tool that provides step-by-step planning capabilities using high reasoning mode
+
+import { Prompt } from '../../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `You are the part of an AI entity named {{aiName}} that provides optimal step-by-step planning capabilities. Your role is to analyze the task at hand and create a detailed, well-structured plan that breaks down the work into clear, actionable steps. Focus on efficiency, completeness, and optimal sequencing of operations.\n\nCreate a step-by-step plan that:\n- Identifies all required information and resources\n- Sequences steps in the most efficient order\n- Considers dependencies between steps\n- Anticipates potential issues and includes contingencies\n- Ensures the plan leads to complete task fulfillment\n\nProvide your plan in a clear, structured format that can be easily followed.\n{{renderTemplate AI_DATETIME}}`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    max_tokens: 100000,
+    model: 'oai-gpt51',
+    reasoningEffort: 'high',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    toolDefinition: {
+        type: "function",
+        enabled: false,
+        icon: "📋",
+        function: {
+            name: "CreatePlan",
+            description: "Create a detailed, step-by-step plan to optimally accomplish a complex task. Use this tool when approaching research problems, multi-step operations, or any task that requires careful planning and sequencing of multiple tool calls or operations.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "Detailed description of the task that needs to be planned, including any constraints, requirements, or context that should be considered in the plan"
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "userMessage"]
+            }
+        }
+    },
+    
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        if (args.detailedInstructions) {
+            args.chatHistory.push({role: "user", content: args.detailedInstructions});
+        }
+        let result = await runAllPrompts({ ...args, stream: false, reasoningEffort: 'high' });        
+        resolver.tool = JSON.stringify({ toolUsed: "planner" });          
+        return result;
+    }
+};
+
diff --git a/pathways/system/entity/tools/sys_tool_reasoning.js b/pathways/system/entity/tools/sys_tool_reasoning.js
deleted file mode 100644
index e43bf872..00000000
--- a/pathways/system/entity/tools/sys_tool_reasoning.js
+++ /dev/null
@@ -1,80 +0,0 @@
-// sys_tool_reasoning.js
-// Entity tool that provides advanced reasoning and planning capabilities
-
-import { Prompt } from '../../../../server/prompt.js';
-
-export default {
-    prompt:
-        [
-            new Prompt({ messages: [
-                {"role": "system", "content": `You are the part of an AI entity named {{aiName}} that provides advanced reasoning and planning capabilities. You excel at breaking down complex problems, creating detailed plans, and providing thorough analysis. Think carefully about the latest request and provide a detailed, well thought out, carefully reviewed response.\n{{renderTemplate AI_DATETIME}}`},
-                "{{chatHistory}}",
-            ]}),
-        ],
-    inputParameters: {
-        chatHistory: [{role: '', content: []}],
-        contextId: ``,
-        aiName: "Jarvis",
-        language: "English",
-    },
-    max_tokens: 100000,
-    model: 'oai-o3',
-    useInputChunking: false,
-    enableDuplicateRequests: false,
-    timeout: 600,
-    toolDefinition: [{
-        type: "function",
-        enabled: false,
-        icon: "🗺️",
-        function: {
-            name: "PlanMultiStepTask",
-            description: "Use specifically to create a thorough, well thought out, step by step plan to accomplish a task. You should always use this tool when you're planning to do something complex or something that might require multiple steps.",
-            parameters: {
-                type: "object",
-                properties: {
-                    detailedInstructions: {
-                        type: "string",
-                        description: "Detailed instructions about what you need the tool to do"
-                    },
-                    userMessage: {
-                        type: "string",
-                        description: "A user-friendly message that describes what you're doing with this tool"
-                    }
-                },
-                required: ["detailedInstructions", "userMessage"]
-            }
-        }
-    },
-    {
-        type: "function",
-        enabled: false,
-        icon: "🧠",
-        function: {
-            name: "ApplyAdvancedReasoning",
-            description: "Employ for advanced reasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices.",
-            parameters: {
-                type: "object",
-                properties: {
-                    detailedInstructions: {
-                        type: "string",
-                        description: "Detailed instructions about what you need the tool to do" 
-                    },
-                    userMessage: {
-                        type: "string",
-                        description: "A user-friendly message that describes what you're doing with this tool"
-                    }
-                },
-                required: ["detailedInstructions", "userMessage"]
-            }
-        }
-    }],
-    
-    executePathway: async ({args, runAllPrompts, resolver}) => {
-        if (args.detailedInstructions) {
-            args.chatHistory.push({role: "user", content: args.detailedInstructions});
-        }
-        let result = await runAllPrompts({ ...args, stream: false });        
-        resolver.tool = JSON.stringify({ toolUsed: "reasoning" });          
-        return result;
-    }
-} 
\ No newline at end of file
diff --git a/pathways/system/entity/tools/sys_tool_view_image.js b/pathways/system/entity/tools/sys_tool_view_image.js
new file mode 100644
index 00000000..5d42925a
--- /dev/null
+++ b/pathways/system/entity/tools/sys_tool_view_image.js
@@ -0,0 +1,115 @@
+// sys_tool_view_image.js
+// Tool pathway that allows agents to view image files from the file collection
+import logger from '../../../../lib/logger.js';
+import { loadFileCollection, findFileInCollection, ensureShortLivedUrl } from '../../../../lib/fileUtils.js';
+import { config } from '../../../../config.js';
+
+export default {
+    prompt: [],
+    timeout: 30,
+    toolDefinition: { 
+        type: "function",
+        icon: "👀",
+        function: {
+            name: "ViewImages",
+            description: "View one or more image files from your file collection. This injects the images into the conversation so you can see them. Use this when you need to look at image files that are in your collection but not currently visible in the conversation.",
+            parameters: {
+                type: "object",
+                properties: {
+                    files: {
+                        type: "array",
+                        items: {
+                            type: "string"
+                        },
+                        description: "Array of files to view (from ListFileCollection or SearchFileCollection): each can be the hash, the filename, the URL, or the GCS URL. You can find available files in the availableFiles section."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["files", "userMessage"]
+            }
+        }
+    },
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const { files, contextId, contextKey } = args;
+
+        if (!files || !Array.isArray(files) || files.length === 0) {
+            throw new Error("Files parameter is required and must be a non-empty array");
+        }
+
+        try {
+            // Load the file collection
+            const collection = await loadFileCollection(contextId, contextKey, true);
+            
+            const imageUrls = [];
+            const errors = [];
+            const foundFilenames = [];
+
+            // Process each file
+            for (const file of files) {
+                // Find the file in the collection
+                const foundFile = findFileInCollection(file, collection);
+                
+                if (!foundFile) {
+                    errors.push(`File not found: ${file}`);
+                    continue;
+                }
+
+                // Check if it's an image by MIME type
+                const mimeType = foundFile.mimeType || foundFile.contentType || '';
+                const isImage = mimeType.startsWith('image/') || 
+                              /\.(jpg|jpeg|png|gif|bmp|webp|svg)$/i.test(foundFile.filename || '');
+
+                if (!isImage) {
+                    errors.push(`File "${foundFile.filename || file}" is not an image file (MIME type: ${mimeType || 'unknown'})`);
+                    continue;
+                }
+
+                // Resolve to short-lived URL if possible
+                const fileHandlerUrl = config.get('whisperMediaApiUrl');
+                const fileWithShortLivedUrl = await ensureShortLivedUrl(foundFile, fileHandlerUrl);
+
+                // Add to imageUrls array
+                imageUrls.push({
+                    type: "image_url",
+                    url: fileWithShortLivedUrl.url,
+                    gcs: fileWithShortLivedUrl.gcs,
+                    image_url: { url: fileWithShortLivedUrl.url },
+                    originalFilename: fileWithShortLivedUrl.filename,
+                    hash: fileWithShortLivedUrl.hash
+                });
+
+                foundFilenames.push(foundFile.filename || file);
+            }
+
+            // If no images were found, return error
+            if (imageUrls.length === 0) {
+                return JSON.stringify({
+                    error: `No valid images found. ${errors.join('; ')}`
+                });
+            }
+
+            // Return the file info in a format that can be extracted as toolImages
+            // This will be picked up by pathwayTools.js and added to toolImages
+            resolver.tool = JSON.stringify({ toolUsed: "ViewImages" });
+            
+            const message = imageUrls.length === 1 
+                ? `Image "${foundFilenames[0]}" is now available for viewing.`
+                : `${imageUrls.length} image(s) (${foundFilenames.join(', ')}) are now available for viewing.`;
+
+            return JSON.stringify({
+                success: true,
+                message: message,
+                imageUrls: imageUrls,
+                errors: errors.length > 0 ? errors : undefined
+            });
+        } catch (e) {
+            logger.error(`Error in ViewImages tool: ${e}`);
+            throw e;
+        }
+    }
+};
+
diff --git a/server/modelExecutor.js b/server/modelExecutor.js
index e0e32e94..a3642d26 100644
--- a/server/modelExecutor.js
+++ b/server/modelExecutor.js
@@ -23,6 +23,7 @@ import Gemini15ChatPlugin from './plugins/gemini15ChatPlugin.js';
 import Gemini15VisionPlugin from './plugins/gemini15VisionPlugin.js';
 import Gemini25ImagePlugin from './plugins/gemini25ImagePlugin.js';
 import Gemini3ImagePlugin from './plugins/gemini3ImagePlugin.js';
+import Gemini3ReasoningVisionPlugin from './plugins/gemini3ReasoningVisionPlugin.js';
 import Claude3VertexPlugin from './plugins/claude3VertexPlugin.js';
 import Claude4VertexPlugin from './plugins/claude4VertexPlugin.js';
 import NeuralSpacePlugin from './plugins/neuralSpacePlugin.js';
@@ -111,6 +112,9 @@ class ModelExecutor {
             case 'GEMINI-3-IMAGE':
                 plugin = new Gemini3ImagePlugin(pathway, model);
                 break;
+            case 'GEMINI-3-REASONING-VISION':
+                plugin = new Gemini3ReasoningVisionPlugin(pathway, model);
+                break;
             case 'CLAUDE-3-VERTEX':
                 plugin = new Claude3VertexPlugin(pathway, model);
                 break;
diff --git a/server/plugins/gemini3ReasoningVisionPlugin.js b/server/plugins/gemini3ReasoningVisionPlugin.js
new file mode 100644
index 00000000..6913808f
--- /dev/null
+++ b/server/plugins/gemini3ReasoningVisionPlugin.js
@@ -0,0 +1,192 @@
+import Gemini3ImagePlugin from './gemini3ImagePlugin.js';
+import CortexResponse from '../../lib/cortexResponse.js';
+import logger from '../../lib/logger.js';
+
+class Gemini3ReasoningVisionPlugin extends Gemini3ImagePlugin {
+
+    constructor(pathway, model) {
+        super(pathway, model);
+    }
+
+    // Override getRequestParameters to add Gemini 3 thinking support
+    getRequestParameters(text, parameters, prompt, cortexRequest) {
+        const baseParameters = super.getRequestParameters(text, parameters, prompt, cortexRequest);
+        
+        // Add Gemini 3 thinking support
+        // Gemini 3 uses thinkingLevel: 'low' or 'high' (instead of thinkingBudget)
+        // includeThoughts: true to get thought summaries in response
+        let thinkingLevel = parameters?.thinkingLevel ?? parameters?.thinking_level;
+        let includeThoughts = parameters?.includeThoughts ?? parameters?.include_thoughts ?? false;
+
+        // Convert OpenAI reasoningEffort to Gemini 3 thinkingLevel
+        // OpenAI supports: 'high', 'medium', 'low', 'none'
+        // Gemini 3 supports: 'high' or 'low' (thinking cannot be disabled)
+        // Mapping: 'high' or 'medium' → 'high', 'low' or 'minimal' → 'low', 'none' → 'low'
+        const reasoningEffort = parameters?.reasoningEffort ?? this.promptParameters?.reasoningEffort;
+        if (reasoningEffort && thinkingLevel === undefined) {
+            const effort = typeof reasoningEffort === 'string' ? reasoningEffort.toLowerCase() : String(reasoningEffort).toLowerCase();
+            if (effort === 'high' || effort === 'medium') {
+                // High or medium reasoning effort → high thinking level
+                thinkingLevel = 'high';
+            } else {
+                // Low, minimal, or none → low thinking level (Gemini 3 doesn't support disabling thinking)
+                thinkingLevel = 'low';
+            }
+        }
+
+        // Also check pathway parameters
+        if (thinkingLevel === undefined && cortexRequest?.pathway?.thinkingLevel !== undefined) {
+            thinkingLevel = cortexRequest.pathway.thinkingLevel;
+        } else if (thinkingLevel === undefined && cortexRequest?.pathway?.thinking_level !== undefined) {
+            thinkingLevel = cortexRequest.pathway.thinking_level;
+        } else if (thinkingLevel === undefined && cortexRequest?.pathway?.reasoningEffort !== undefined) {
+            // Also check pathway for reasoningEffort
+            const pathwayEffort = typeof cortexRequest.pathway.reasoningEffort === 'string' 
+                ? cortexRequest.pathway.reasoningEffort.toLowerCase() 
+                : String(cortexRequest.pathway.reasoningEffort).toLowerCase();
+            if (pathwayEffort === 'high' || pathwayEffort === 'medium') {
+                thinkingLevel = 'high';
+            } else {
+                thinkingLevel = 'low';
+            }
+        }
+        
+        if (includeThoughts === false && cortexRequest?.pathway?.includeThoughts !== undefined) {
+            includeThoughts = cortexRequest.pathway.includeThoughts;
+        } else if (includeThoughts === false && cortexRequest?.pathway?.include_thoughts !== undefined) {
+            includeThoughts = cortexRequest.pathway.include_thoughts;
+        }
+
+        // Set up thinkingConfig in generationConfig if thinking is enabled
+        if (thinkingLevel !== undefined || includeThoughts) {
+            if (!baseParameters.generationConfig.thinkingConfig) {
+                baseParameters.generationConfig.thinkingConfig = {};
+            }
+            
+            // Gemini 3 uses thinkingLevel: 'low' or 'high'
+            if (thinkingLevel !== undefined) {
+                const level = typeof thinkingLevel === 'string' ? thinkingLevel.toLowerCase() : String(thinkingLevel).toLowerCase();
+                // Validate and set thinkingLevel (only 'low' or 'high' are valid)
+                if (level === 'low' || level === 'high') {
+                    baseParameters.generationConfig.thinkingConfig.thinkingLevel = level;
+                } else {
+                    // Default to 'low' if invalid value
+                    baseParameters.generationConfig.thinkingConfig.thinkingLevel = 'low';
+                }
+            }
+            
+            // includeThoughts: true to get thought summaries
+            if (includeThoughts !== undefined) {
+                baseParameters.generationConfig.thinkingConfig.includeThoughts = Boolean(includeThoughts);
+            }
+        }
+
+        return baseParameters;
+    }
+
+    // Override parseResponse to handle thought summaries
+    parseResponse(data) {
+        // First, let the parent handle the response
+        const baseResponse = super.parseResponse(data);
+        
+        // Check if we have thought summaries in the response
+        if (data?.candidates?.[0]?.content?.parts) {
+            const parts = data.candidates[0].content.parts;
+            let thoughtSummaries = [];
+            let hasThoughts = false;
+
+            // Extract thought summaries from parts
+            for (const part of parts) {
+                if (part.thought && part.text) {
+                    // This is a thought summary
+                    thoughtSummaries.push(part.text);
+                    hasThoughts = true;
+                }
+            }
+
+            // If we have thought summaries, add them to the response
+            if (hasThoughts) {
+                // If baseResponse is already a CortexResponse, add thoughts to it
+                if (baseResponse && typeof baseResponse === 'object' && baseResponse.constructor && baseResponse.constructor.name === 'CortexResponse') {
+                    baseResponse.thoughts = thoughtSummaries;
+                    return baseResponse;
+                } else {
+                    // Create new CortexResponse with thoughts
+                    // Preserve the baseResponse text if it's a string
+                    const outputText = typeof baseResponse === 'string' ? baseResponse : '';
+                    return new CortexResponse({
+                        output_text: outputText,
+                        thoughts: thoughtSummaries,
+                        finishReason: data?.candidates?.[0]?.finishReason === 'STOP' ? 'stop' : 'length',
+                        usage: data?.usageMetadata || null,
+                        metadata: { model: this.modelName }
+                    });
+                }
+            }
+        }
+
+        return baseResponse;
+    }
+
+    // Override processStreamEvent to handle thought summaries in streaming
+    processStreamEvent(event, requestProgress) {
+        const baseProgress = super.processStreamEvent(event, requestProgress);
+        
+        const eventData = JSON.parse(event.data);
+        
+        // Initialize thought summaries array if needed
+        if (!requestProgress.thoughts) {
+            requestProgress.thoughts = [];
+        }
+        
+        // Handle thought summaries in streaming
+        if (eventData.candidates?.[0]?.content?.parts) {
+            const parts = eventData.candidates[0].content.parts;
+            
+            for (const part of parts) {
+                if (part.thought && part.text) {
+                    // This is a thought summary chunk
+                    // Accumulate thought summaries
+                    if (!requestProgress.thoughts.includes(part.text)) {
+                        requestProgress.thoughts.push(part.text);
+                    }
+                    
+                    // Optionally, you could emit thought chunks separately
+                    // For now, we'll accumulate them and they'll be available in the final response
+                }
+            }
+        }
+
+        return baseProgress;
+    }
+
+    // Override logRequestData to include thought information
+    logRequestData(data, responseData, prompt) {
+        // Check if responseData is a CortexResponse object with thoughts
+        if (responseData && typeof responseData === 'object' && responseData.constructor && responseData.constructor.name === 'CortexResponse') {
+            const { length, units } = this.getLength(responseData.output_text || '');
+            logger.info(`[response received containing ${length} ${units}]`);
+            
+            if (responseData.thoughts && responseData.thoughts.length > 0) {
+                logger.info(`[response contains ${responseData.thoughts.length} thought summary(ies)]`);
+                responseData.thoughts.forEach((thought, index) => {
+                    logger.verbose(`[thought ${index + 1}]: ${this.shortenContent(thought)}`);
+                });
+            }
+            
+            if (responseData.artifacts && responseData.artifacts.length > 0) {
+                logger.info(`[response contains ${responseData.artifacts.length} image artifact(s)]`);
+            }
+            
+            logger.verbose(`${this.shortenContent(responseData.output_text || '')}`);
+            return;
+        }
+
+        // Fall back to parent implementation for non-CortexResponse objects
+        super.logRequestData(data, responseData, prompt);
+    }
+
+}
+
+export default Gemini3ReasoningVisionPlugin;
+
diff --git a/tests/integration/features/tools/fileCollection.test.js b/tests/integration/features/tools/fileCollection.test.js
index 3ffcc963..a4827ace 100644
--- a/tests/integration/features/tools/fileCollection.test.js
+++ b/tests/integration/features/tools/fileCollection.test.js
@@ -493,6 +493,8 @@ test('Memory system: memoryFiles ignored in memoryAll save', async t => {
 });
 
 // Test generateFileMessageContent function (integration tests)
+// Note: These tests verify basic functionality. If WHISPER_MEDIA_API_URL is configured,
+// generateFileMessageContent will automatically use short-lived URLs when file hashes are available.
 test('generateFileMessageContent should find file by ID', async t => {
     const contextId = createTestContext();
     
diff --git a/tests/unit/core/shortLivedUrl.test.js b/tests/unit/core/shortLivedUrl.test.js
new file mode 100644
index 00000000..3a1050e8
--- /dev/null
+++ b/tests/unit/core/shortLivedUrl.test.js
@@ -0,0 +1,257 @@
+// shortLivedUrl.test.js
+// Unit tests for short-lived URL functionality
+
+import test from 'ava';
+import sinon from 'sinon';
+import { checkHashExists, ensureShortLivedUrl } from '../../../lib/fileUtils.js';
+import { axios } from '../../../lib/requestExecutor.js';
+
+test.beforeEach(t => {
+    t.context.sandbox = sinon.createSandbox();
+});
+
+test.afterEach.always(t => {
+    t.context.sandbox.restore();
+});
+
+test('checkHashExists should return shortLivedUrl when available', async t => {
+    const hash = 'test-hash-123';
+    const fileHandlerUrl = 'https://file-handler.example.com';
+    const mockResponse = {
+        status: 200,
+        data: {
+            url: 'https://storage.example.com/file.pdf?sv=2023-11-03&se=2025-01-01T00:00:00Z&sig=long-lived',
+            shortLivedUrl: 'https://storage.example.com/file.pdf?sv=2023-11-03&se=2024-01-01T10:15:00Z&sig=short-lived',
+            gcs: 'gs://bucket/file.pdf',
+            hash: hash,
+            filename: 'file.pdf'
+        }
+    };
+
+    const axiosGetStub = t.context.sandbox.replace(axios, 'get', sinon.stub().resolves(mockResponse));
+
+    const result = await checkHashExists(hash, fileHandlerUrl);
+
+    t.truthy(result);
+    t.is(result.url, mockResponse.data.shortLivedUrl, 'Should return shortLivedUrl');
+    t.is(result.gcs, mockResponse.data.gcs, 'Should return GCS URL');
+    t.is(result.hash, hash, 'Should return hash');
+
+    // Verify axios was called with correct parameters
+    t.true(axiosGetStub.calledOnce);
+    const callArgs = axiosGetStub.getCall(0).args;
+    t.true(callArgs[0].includes('checkHash=true'));
+    t.true(callArgs[0].includes('shortLivedMinutes=5'));
+});
+
+test('checkHashExists should fallback to regular URL when shortLivedUrl not available', async t => {
+    const hash = 'test-hash-456';
+    const fileHandlerUrl = 'https://file-handler.example.com';
+    const mockResponse = {
+        status: 200,
+        data: {
+            url: 'https://storage.example.com/file.pdf?sv=2023-11-03&se=2025-01-01T00:00:00Z&sig=long-lived',
+            // No shortLivedUrl in response
+            gcs: 'gs://bucket/file.pdf',
+            hash: hash,
+            filename: 'file.pdf'
+        }
+    };
+
+    t.context.sandbox.replace(axios, 'get', sinon.stub().resolves(mockResponse));
+
+    const result = await checkHashExists(hash, fileHandlerUrl);
+
+    t.truthy(result);
+    t.is(result.url, mockResponse.data.url, 'Should fallback to regular URL');
+    t.is(result.gcs, mockResponse.data.gcs, 'Should return GCS URL');
+});
+
+test('checkHashExists should prefer converted URL in shortLivedUrl', async t => {
+    const hash = 'test-hash-789';
+    const fileHandlerUrl = 'https://file-handler.example.com';
+    const mockResponse = {
+        status: 200,
+        data: {
+            url: 'https://storage.example.com/file.xlsx?sv=2023-11-03&se=2025-01-01T00:00:00Z&sig=long-lived',
+            shortLivedUrl: 'https://storage.example.com/file.csv?sv=2023-11-03&se=2024-01-01T10:15:00Z&sig=short-lived',
+            converted: {
+                url: 'https://storage.example.com/file.csv?sv=2023-11-03&se=2025-01-01T00:00:00Z&sig=long-lived',
+                gcs: 'gs://bucket/file.csv'
+            },
+            gcs: 'gs://bucket/file.xlsx',
+            hash: hash
+        }
+    };
+
+    t.context.sandbox.replace(axios, 'get', sinon.stub().resolves(mockResponse));
+
+    const result = await checkHashExists(hash, fileHandlerUrl);
+
+    t.truthy(result);
+    // shortLivedUrl should be based on converted file
+    t.is(result.url, mockResponse.data.shortLivedUrl, 'Should use shortLivedUrl (which prefers converted)');
+    // GCS should prefer converted
+    t.is(result.gcs, mockResponse.data.converted.gcs, 'Should prefer converted GCS URL');
+});
+
+test('checkHashExists should return null when file not found', async t => {
+    const hash = 'non-existent-hash';
+    const fileHandlerUrl = 'https://file-handler.example.com';
+    const mockResponse = {
+        status: 404,
+        data: { message: 'File not found' }
+    };
+
+    t.context.sandbox.replace(axios, 'get', sinon.stub().resolves(mockResponse));
+
+    const result = await checkHashExists(hash, fileHandlerUrl);
+
+    t.is(result, null, 'Should return null when file not found');
+});
+
+test('checkHashExists should return null when hash or fileHandlerUrl missing', async t => {
+    t.is(await checkHashExists(null, 'https://file-handler.example.com'), null);
+    t.is(await checkHashExists('hash-123', null), null);
+    t.is(await checkHashExists('', 'https://file-handler.example.com'), null);
+});
+
+test('checkHashExists should handle errors gracefully', async t => {
+    const hash = 'test-hash-error';
+    const fileHandlerUrl = 'https://file-handler.example.com';
+
+    t.context.sandbox.replace(axios, 'get', sinon.stub().rejects(new Error('Network error')));
+
+    const result = await checkHashExists(hash, fileHandlerUrl);
+
+    t.is(result, null, 'Should return null on error');
+});
+
+test('ensureShortLivedUrl should resolve file to short-lived URL when hash available', async t => {
+    const fileObject = {
+        url: 'https://storage.example.com/file.pdf?sv=2023-11-03&se=2025-01-01T00:00:00Z&sig=long-lived',
+        gcs: 'gs://bucket/file.pdf',
+        hash: 'test-hash-123',
+        filename: 'file.pdf'
+    };
+    const fileHandlerUrl = 'https://file-handler.example.com';
+    const shortLivedUrl = 'https://storage.example.com/file.pdf?sv=2023-11-03&se=2024-01-01T10:15:00Z&sig=short-lived';
+
+    const mockResponse = {
+        status: 200,
+        data: {
+            url: fileObject.url,
+            shortLivedUrl: shortLivedUrl,
+            gcs: fileObject.gcs,
+            hash: fileObject.hash
+        }
+    };
+
+    t.context.sandbox.replace(axios, 'get', sinon.stub().resolves(mockResponse));
+
+    const result = await ensureShortLivedUrl(fileObject, fileHandlerUrl);
+
+    t.truthy(result);
+    t.is(result.url, shortLivedUrl, 'Should use short-lived URL');
+    t.is(result.gcs, fileObject.gcs, 'Should preserve GCS URL');
+    t.is(result.hash, fileObject.hash, 'Should preserve hash');
+    t.is(result.filename, fileObject.filename, 'Should preserve filename');
+});
+
+test('ensureShortLivedUrl should return original object when no hash', async t => {
+    const fileObject = {
+        url: 'https://storage.example.com/file.pdf',
+        filename: 'file.pdf'
+        // No hash
+    };
+    const fileHandlerUrl = 'https://file-handler.example.com';
+
+    const result = await ensureShortLivedUrl(fileObject, fileHandlerUrl);
+
+    t.deepEqual(result, fileObject, 'Should return original object when no hash');
+});
+
+test('ensureShortLivedUrl should return original object when no fileHandlerUrl', async t => {
+    const fileObject = {
+        url: 'https://storage.example.com/file.pdf',
+        hash: 'test-hash-123',
+        filename: 'file.pdf'
+    };
+
+    const result = await ensureShortLivedUrl(fileObject, null);
+
+    t.deepEqual(result, fileObject, 'Should return original object when no fileHandlerUrl');
+});
+
+test('ensureShortLivedUrl should fallback to original object on error', async t => {
+    const fileObject = {
+        url: 'https://storage.example.com/file.pdf',
+        hash: 'test-hash-error',
+        filename: 'file.pdf'
+    };
+    const fileHandlerUrl = 'https://file-handler.example.com';
+
+    t.context.sandbox.replace(axios, 'get', sinon.stub().rejects(new Error('Network error')));
+
+    const result = await ensureShortLivedUrl(fileObject, fileHandlerUrl);
+
+    t.deepEqual(result, fileObject, 'Should fallback to original object on error');
+});
+
+test('ensureShortLivedUrl should update GCS URL from checkHash response', async t => {
+    const fileObject = {
+        url: 'https://storage.example.com/file.xlsx',
+        gcs: 'gs://bucket/file.xlsx',
+        hash: 'test-hash-789',
+        filename: 'file.xlsx'
+    };
+    const fileHandlerUrl = 'https://file-handler.example.com';
+    const convertedGcs = 'gs://bucket/file.csv';
+
+    const mockResponse = {
+        status: 200,
+        data: {
+            url: fileObject.url,
+            shortLivedUrl: 'https://storage.example.com/file.csv?sv=2023-11-03&se=2024-01-01T10:15:00Z&sig=short-lived',
+            converted: {
+                gcs: convertedGcs
+            },
+            gcs: fileObject.gcs,
+            hash: fileObject.hash
+        }
+    };
+
+    t.context.sandbox.replace(axios, 'get', sinon.stub().resolves(mockResponse));
+
+    const result = await ensureShortLivedUrl(fileObject, fileHandlerUrl);
+
+    t.truthy(result);
+    t.is(result.gcs, convertedGcs, 'Should update GCS URL from converted');
+});
+
+test('ensureShortLivedUrl should respect shortLivedMinutes parameter', async t => {
+    const fileObject = {
+        url: 'https://storage.example.com/file.pdf',
+        hash: 'test-hash-123',
+        filename: 'file.pdf'
+    };
+    const fileHandlerUrl = 'https://file-handler.example.com';
+    const shortLivedMinutes = 10;
+
+    const mockResponse = {
+        status: 200,
+        data: {
+            url: fileObject.url,
+            shortLivedUrl: 'https://storage.example.com/file.pdf?sv=2023-11-03&se=2024-01-01T10:15:00Z&sig=short-lived',
+            hash: fileObject.hash
+        }
+    };
+
+    const axiosGetStub = t.context.sandbox.replace(axios, 'get', sinon.stub().resolves(mockResponse));
+
+    await ensureShortLivedUrl(fileObject, fileHandlerUrl, shortLivedMinutes);
+
+    // Verify axios was called with correct shortLivedMinutes
+    const callArgs = axiosGetStub.getCall(0).args;
+    t.true(callArgs[0].includes(`shortLivedMinutes=${shortLivedMinutes}`));
+});
diff --git a/tests/unit/core/util.test.js b/tests/unit/core/util.test.js
index d28443ef..b7b7a9a7 100644
--- a/tests/unit/core/util.test.js
+++ b/tests/unit/core/util.test.js
@@ -5,8 +5,10 @@ import test from 'ava';
 import fs from 'fs';
 import path from 'path';
 import os from 'os';
+import sinon from 'sinon';
 import { removeOldImageAndFileContent } from '../../../lib/util.js';
 import { computeFileHash, computeBufferHash, generateFileMessageContent, injectFileIntoChatHistory } from '../../../lib/fileUtils.js';
+import { axios } from '../../../lib/requestExecutor.js';
 
 // Test removeOldImageAndFileContent function
 

From e7692b34b858c851299aed4d25a6f93eb89ffe09 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Tue, 16 Dec 2025 08:17:31 -0700
Subject: [PATCH 04/27] refactor: implement context scoping for file handling
 and streamline container management

- Updated INTERFACE.md to include optional `contextId` for per-user/per-context file scoping, enhancing file isolation in multi-tenant applications.
- Modified file handling scripts to support context-scoped keys, ensuring secure access and management of files based on user context.
- Simplified container management by enforcing a single container approach, removing legacy support for multiple containers.
- Enhanced Redis key management to facilitate migration from legacy keys to the new context-scoped format, ensuring backward compatibility.
- Improved tests to validate context scoping functionality and ensure robust handling of legacy keys during migration.
- Updated environment variable validation to reflect the new single container requirement, improving clarity in configuration settings.
---
 helper-apps/cortex-file-handler/INTERFACE.md  |  77 ++-
 .../scripts/setup-test-containers.js          |   5 +-
 .../scripts/validate-env.js                   |  24 +-
 .../cortex-file-handler/src/constants.js      |  19 +-
 helper-apps/cortex-file-handler/src/index.js  |  43 +-
 helper-apps/cortex-file-handler/src/redis.js  | 106 +++-
 .../tests/containerNameParsing.test.js        |  75 +++
 .../tests/deleteOperations.test.js            |   4 +-
 .../tests/redisMigration.test.js              | 519 ++++++++++++++++++
 .../tests/setRetention.test.js                |  28 +-
 10 files changed, 817 insertions(+), 83 deletions(-)
 create mode 100644 helper-apps/cortex-file-handler/tests/redisMigration.test.js

diff --git a/helper-apps/cortex-file-handler/INTERFACE.md b/helper-apps/cortex-file-handler/INTERFACE.md
index 8983272c..d8851882 100644
--- a/helper-apps/cortex-file-handler/INTERFACE.md
+++ b/helper-apps/cortex-file-handler/INTERFACE.md
@@ -22,7 +22,8 @@ The file handler uses a unified storage approach with Azure Blob Storage:
 - **Content-Type**: `multipart/form-data`
 - **Parameters**:
   - `hash` (optional): Unique identifier for the file
-  - `requestId` (required): Unique identifier for the request
+  - `requestId` (optional): Unique identifier for the request (not required for simple uploads)
+  - `contextId` (optional): Context identifier for per-user/per-context file scoping
   - File content must be included in the form data
 - **Behavior**:
   - Uploads file to primary storage (Azure or Local)
@@ -34,6 +35,7 @@ The file handler uses a unified storage approach with Azure Blob Storage:
   - `shortLivedUrl`: Short-lived URL (5-minute expiration, always included)
   - `gcs`: GCS URL (if GCS is configured)
   - `hash`: Hash value (if provided)
+  - `contextId`: Context identifier (if provided)
   - `message`: Success message
   - `filename`: Original filename
 - **Note**: The `save` parameter is not supported in POST requests. To convert and save a document as text, use GET with the `save` parameter.
@@ -52,6 +54,7 @@ The file handler uses a unified storage approach with Azure Blob Storage:
     - Does not save to GCS
     - Original document is deleted from storage after text conversion
   - `hash` (optional): Unique identifier for the file
+  - `contextId` (optional): Context identifier for per-user/per-context file scoping
   - `checkHash` (optional): Check if hash exists
   - `clearHash` (optional): Remove hash from storage
   - `generateShortLived` (optional): Generate a short-lived URL for an existing hash
@@ -102,13 +105,15 @@ The file handler uses a unified storage approach with Azure Blob Storage:
 - **Parameters** (can be in query string or request body):
   - `requestId` (optional): Unique identifier for the request (for multi-file deletion)
   - `hash` (optional): Hash of the file to delete (for single-file deletion)
+  - `contextId` (optional): Context identifier for per-user/per-context file scoping
+  - `operation` (optional): Set to `"delete"` to explicitly trigger delete operation
 - **Behavior**:
   - Supports two deletion modes:
     1. **By requestId**: Deletes all files associated with a requestId
     2. **By hash**: Deletes a single file by its hash
   - Deletes file from primary storage (Azure or Local)
   - Deletes file from GCS if configured
-  - Removes file metadata from Redis
+  - Removes file metadata from Redis (including legacy keys if they exist)
   - Returns deletion result
 - **Response**: 
   - For requestId deletion: Array of deleted file URLs
@@ -120,6 +125,7 @@ The file handler uses a unified storage approach with Azure Blob Storage:
 - **Parameters** (can be in query string or request body):
   - `hash` (required): Hash of the file
   - `retention` (required): Retention value - either `'temporary'` or `'permanent'`
+  - `contextId` (optional): Context identifier for per-user/per-context file scoping
   - `setRetention` (optional): Set to `true` to trigger operation, or use `operation=setRetention` in query string
 - **Behavior**:
   - Updates the blob index tag to the specified retention value
@@ -222,7 +228,21 @@ The file handler uses a unified storage approach with Azure Blob Storage:
   - Used for caching remote file results
   - Tracks file access timestamps
   - Used for progress tracking
-  - Files are stored by hash directly (no container scoping)
+  - **Key Format**:
+    - Without context: `<hash>` (unscoped)
+    - With context: `<hash>:ctx:<contextId>` (context-scoped)
+    - Legacy keys (`<hash>:<containerName>`) are automatically migrated on read
+  - **Key Scoping**:
+    - Files can be scoped by `contextId` for per-user/per-context isolation
+    - When `contextId` is provided, files are stored with context-scoped keys
+    - When `contextId` is not provided, files use unscoped keys
+    - Context-scoped reads fall back to unscoped keys if not found
+    - Unscoped reads fall back to legacy container-scoped keys (if they exist) and migrate them automatically
+  - **Migration Behavior**:
+    - Legacy container-scoped keys are automatically migrated to unscoped keys on first read
+    - Migration copies data to new key format and deletes legacy key
+    - New writes never create legacy keys
+    - Deletes clean up both new and legacy keys
 - **Short-Lived URLs**:
   - All file operations now return a `shortLivedUrl` field
   - Short-lived URLs expire after 5 minutes (configurable via `shortLivedMinutes`)
@@ -242,6 +262,36 @@ The file handler uses a unified storage approach with Azure Blob Storage:
   - After successful processing
   - On error conditions
 
+## Context Scoping
+
+The file handler supports optional context scoping for per-user or per-context file isolation:
+
+- **Parameter**: `contextId`
+- **Usage**: Include in any request that uses `hash` parameter
+- **Behavior**:
+  - When provided, files are stored/retrieved using context-scoped Redis keys: `<hash>:ctx:<contextId>`
+  - When not provided, files use unscoped keys: `<hash>`
+  - Context-scoped reads fall back to unscoped keys if context-scoped key doesn't exist
+  - Unscoped reads fall back to legacy container-scoped keys and migrate them automatically
+- **Use Cases**:
+  - Multi-tenant applications where files should be isolated per user/tenant
+  - Per-session file management
+  - Per-workspace file organization
+
+**Example:**
+```bash
+# Upload with contextId
+POST /file-handler
+Content-Type: multipart/form-data
+hash=abc123&contextId=user-456&requestId=req-789
+
+# Check hash with contextId
+GET /file-handler?hash=abc123&checkHash=true&contextId=user-456
+
+# Delete with contextId
+DELETE /file-handler?hash=abc123&contextId=user-456
+```
+
 ## Usage Examples
 
 ### Check Hash (Always Returns Short-Lived URL)
@@ -273,6 +323,27 @@ GET /file-handler?hash=abc123&checkHash=true&shortLivedMinutes=10
 }
 ```
 
+### Context-Scoped File Operations
+
+```bash
+# Upload file with contextId
+POST /file-handler
+Content-Type: multipart/form-data
+hash=abc123&contextId=user-456&requestId=req-789
+[file content]
+
+# Check hash with contextId (falls back to unscoped if context-scoped not found)
+GET /file-handler?hash=abc123&checkHash=true&contextId=user-456
+
+# Delete file with contextId
+DELETE /file-handler?hash=abc123&contextId=user-456
+
+# Set retention with contextId
+POST /file-handler?hash=abc123&retention=permanent&contextId=user-456&setRetention=true
+```
+
+**Note**: When `contextId` is provided, files are stored in Redis with context-scoped keys. If a context-scoped key doesn't exist, the system falls back to unscoped keys, and if those don't exist, it falls back to legacy container-scoped keys (which are automatically migrated).
+
 ## Error Handling
 
 - **400 Bad Request**:
diff --git a/helper-apps/cortex-file-handler/scripts/setup-test-containers.js b/helper-apps/cortex-file-handler/scripts/setup-test-containers.js
index 230181dc..19aa6635 100644
--- a/helper-apps/cortex-file-handler/scripts/setup-test-containers.js
+++ b/helper-apps/cortex-file-handler/scripts/setup-test-containers.js
@@ -7,8 +7,9 @@ async function createAzureContainers() {
       "UseDevelopmentStorage=true",
     );
     
-    // Always create all possible test containers to support dynamic test environments
-    const allTestContainers = ["default", "test-container", "test1", "test2", "test3", "container1", "container2", "container3"];
+    // Create test containers that are actually used in test configurations
+    // Note: We only use a single container now (no container scoping)
+    const allTestContainers = ["default", "test-container"];
     
     console.log(`Creating Azure containers: ${allTestContainers.join(', ')}`);
     
diff --git a/helper-apps/cortex-file-handler/scripts/validate-env.js b/helper-apps/cortex-file-handler/scripts/validate-env.js
index 4d60a2ca..ce9efe39 100644
--- a/helper-apps/cortex-file-handler/scripts/validate-env.js
+++ b/helper-apps/cortex-file-handler/scripts/validate-env.js
@@ -26,7 +26,7 @@ const REQUIRED_ENV_VARS = {
   },
   AZURE_STORAGE_CONTAINER_NAME: {
     required: true,
-    description: 'Must specify container names (comma-separated for multiple)'
+    description: 'Must specify a single container name'
   },
   REDIS_CONNECTION_STRING: {
     required: false,
@@ -68,19 +68,17 @@ function validateEnvironment() {
     }
   }
 
-  // Validate container names format
-  const containerNames = process.env.AZURE_STORAGE_CONTAINER_NAME;
-  if (containerNames) {
-    const containers = containerNames.split(',').map(name => name.trim());
-    console.log(`✅ Container names: ${containers.join(', ')}`);
+  // Validate container name
+  const containerName = process.env.AZURE_STORAGE_CONTAINER_NAME;
+  if (containerName) {
+    // Handle legacy comma-separated values (take the last one)
+    const containers = containerName.split(',').map(name => name.trim()).filter(name => name.length > 0);
+    const actualContainer = containers[containers.length - 1];
+    console.log(`✅ Container name: ${actualContainer}`);
     
-    // Check for common test containers that might be missing
-    const commonTestContainers = ['test1', 'test2', 'test3', 'container1', 'container2', 'container3', 'test-container'];
-    const missingContainers = commonTestContainers.filter(container => !containers.includes(container));
-    
-    if (missingContainers.length > 0) {
-      warnings.push(`⚠️  Some test containers might be missing: ${missingContainers.join(', ')}`);
-      warnings.push(`   Consider adding them to AZURE_STORAGE_CONTAINER_NAME if tests fail`);
+    // Warn if comma-separated (legacy format)
+    if (containers.length > 1) {
+      warnings.push(`⚠️  AZURE_STORAGE_CONTAINER_NAME contains comma-separated values (legacy format). Using: "${actualContainer}"`);
     }
   }
 
diff --git a/helper-apps/cortex-file-handler/src/constants.js b/helper-apps/cortex-file-handler/src/constants.js
index 586ca97e..c1d14f0b 100644
--- a/helper-apps/cortex-file-handler/src/constants.js
+++ b/helper-apps/cortex-file-handler/src/constants.js
@@ -136,7 +136,24 @@ export const AZURITE_ACCOUNT_NAME = "devstoreaccount1";
 // Get single container name from environment variable
 // CFH operates on a single Azure container and single GCS bucket
 export const getContainerName = () => {
-  return process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
+  const envValue = process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
+  
+  // Handle legacy comma-separated values (take the last one)
+  if (envValue.includes(",")) {
+    const containers = envValue.split(",").map(c => c.trim()).filter(c => c.length > 0);
+    if (containers.length > 0) {
+      const containerName = containers[containers.length - 1];
+      console.warn(
+        `[WARNING] AZURE_STORAGE_CONTAINER_NAME contains comma-separated values (legacy format). ` +
+        `Using last container: "${containerName}". ` +
+        `Full value: "${envValue}". ` +
+        `Please update to use a single container name.`
+      );
+      return containerName;
+    }
+  }
+  
+  return envValue;
 };
 
 // Helper function to get current container name at runtime
diff --git a/helper-apps/cortex-file-handler/src/index.js b/helper-apps/cortex-file-handler/src/index.js
index b6cff2ed..d37890fd 100644
--- a/helper-apps/cortex-file-handler/src/index.js
+++ b/helper-apps/cortex-file-handler/src/index.js
@@ -10,6 +10,7 @@ import { ensureEncoded, ensureFileExtension, urlExists } from "./helper.js";
 import {
   cleanupRedisFileStoreMap,
   getFileStoreMap,
+  getScopedHashKey,
   publishRequestProgress,
   removeFromFileStoreMap,
   setFileStoreMap,
@@ -80,8 +81,10 @@ async function CortexFileHandler(context, req) {
     restore,
     setRetention,
     retention,
+    contextId,
   } = source;
   // Container parameter is ignored - always uses default container from env var
+  const resolvedContextId = contextId || null;
 
   // Normalize boolean parameters
   const shouldSave = save === true || save === "true";
@@ -115,7 +118,7 @@ async function CortexFileHandler(context, req) {
                 : "upload";
 
   context.log(
-    `Processing ${req.method} request - ${requestId ? `requestId: ${requestId}, ` : ""}${uri ? `uri: ${uri}, ` : ""}${hash ? `hash: ${hash}, ` : ""}operation: ${operation}`,
+    `Processing ${req.method} request - ${requestId ? `requestId: ${requestId}, ` : ""}${uri ? `uri: ${uri}, ` : ""}${hash ? `hash: ${hash}, ` : ""}${resolvedContextId ? `contextId: ${resolvedContextId}, ` : ""}operation: ${operation}`,
   );
 
   // Trigger lightweight age-based cleanup (runs every 100 requests)
@@ -202,10 +205,11 @@ async function CortexFileHandler(context, req) {
 
     // First, get the hash from the map if it exists
     if (deleteHash) {
-      const hashResult = await getFileStoreMap(deleteHash);
+      const deleteKey = getScopedHashKey(deleteHash, resolvedContextId);
+      const hashResult = await getFileStoreMap(deleteKey);
       if (hashResult) {
-        context.log(`Found hash in map for deletion: ${deleteHash}`);
-        await removeFromFileStoreMap(deleteHash);
+        context.log(`Found hash in map for deletion: ${deleteHash}${resolvedContextId ? ` (contextId: ${resolvedContextId})` : ""}`);
+        await removeFromFileStoreMap(deleteKey);
       }
     }
 
@@ -280,7 +284,7 @@ async function CortexFileHandler(context, req) {
       }
 
       // Check if file already exists (using hash or URL as the key)
-      const cacheKey = hash || remoteUrl;
+      const cacheKey = hash ? getScopedHashKey(hash, resolvedContextId) : remoteUrl;
       const exists = await getFileStoreMap(cacheKey);
       if (exists) {
         context.res = {
@@ -336,9 +340,10 @@ async function CortexFileHandler(context, req) {
 
   if (hash && clearHash) {
     try {
-      const hashValue = await getFileStoreMap(hash);
+      const hashKey = getScopedHashKey(hash, resolvedContextId);
+      const hashValue = await getFileStoreMap(hashKey);
       if (hashValue) {
-        await removeFromFileStoreMap(hash);
+        await removeFromFileStoreMap(hashKey);
         context.res = {
           status: 200,
           body: `Hash ${hash} removed`,
@@ -360,10 +365,11 @@ async function CortexFileHandler(context, req) {
   }
 
   if (hash && checkHash) {
-    let hashResult = await getFileStoreMap(hash, true); // Skip lazy cleanup to handle it ourselves
+    const hashKey = getScopedHashKey(hash, resolvedContextId);
+    let hashResult = await getFileStoreMap(hashKey, true); // Skip lazy cleanup to handle it ourselves
 
     if (hashResult) {
-      context.log(`File exists in map: ${hash}`);
+      context.log(`File exists in map: ${hash}${resolvedContextId ? ` (contextId: ${resolvedContextId})` : ""}`);
 
       // Log the URL retrieved from Redis before checking existence
       context.log(`Checking existence of URL from Redis: ${hashResult?.url}`);
@@ -382,7 +388,7 @@ async function CortexFileHandler(context, req) {
           context.log(
             `File not found in any storage. Removing from map: ${hash}`,
           );
-          await removeFromFileStoreMap(hash);
+          await removeFromFileStoreMap(hashKey);
           context.res = {
             status: 404,
             body: `Hash ${hash} not found in storage`,
@@ -401,7 +407,7 @@ async function CortexFileHandler(context, req) {
           } catch (error) {
             context.log(`Error restoring to GCS: ${error}`);
             // If restoration fails, remove the hash from the map
-            await removeFromFileStoreMap(hash);
+            await removeFromFileStoreMap(hashKey);
             context.res = {
               status: 404,
               body: `Hash ${hash} not found`,
@@ -459,7 +465,7 @@ async function CortexFileHandler(context, req) {
           } catch (error) {
             console.error("Error restoring from GCS:", error);
             // If restoration fails, remove the hash from the map
-            await removeFromFileStoreMap(hash);
+            await removeFromFileStoreMap(hashKey);
             context.res = {
               status: 404,
               body: `Hash ${hash} not found`,
@@ -477,7 +483,7 @@ async function CortexFileHandler(context, req) {
           : false;
         if (!finalPrimaryCheck && !finalGCSCheck) {
           context.log(`Failed to restore file. Removing from map: ${hash}`);
-          await removeFromFileStoreMap(hash);
+          await removeFromFileStoreMap(hashKey);
           context.res = {
             status: 404,
             body: `Hash ${hash} not found`,
@@ -593,7 +599,7 @@ async function CortexFileHandler(context, req) {
         }
 
         //update redis timestamp with current time
-        await setFileStoreMap(hash, hashResult);
+        await setFileStoreMap(hashKey, hashResult);
 
         context.res = {
           status: 200,
@@ -603,7 +609,7 @@ async function CortexFileHandler(context, req) {
       } catch (error) {
         context.log(`Error checking file existence: ${error}`);
         // If there's an error checking file existence, remove the hash from the map
-        await removeFromFileStoreMap(scopedHash);
+        await removeFromFileStoreMap(hashKey);
         context.res = {
           status: 404,
           body: `Hash ${hash} not found`,
@@ -628,7 +634,12 @@ async function CortexFileHandler(context, req) {
     // Container parameter is ignored - always uses default container from env var
     const result = await uploadBlob(context, req, saveToLocal, null, hash);
     if (result?.hash && context?.res?.body) {
-      await setFileStoreMap(result.hash, context.res.body);
+      const hashKey = getScopedHashKey(result.hash, resolvedContextId);
+      // Store contextId alongside the entry for debugging/traceability
+      if (resolvedContextId && typeof context.res.body === "object" && context.res.body) {
+        context.res.body.contextId = resolvedContextId;
+      }
+      await setFileStoreMap(hashKey, context.res.body);
     }
     return;
   }
diff --git a/helper-apps/cortex-file-handler/src/redis.js b/helper-apps/cortex-file-handler/src/redis.js
index 97aa3b92..d605de2a 100644
--- a/helper-apps/cortex-file-handler/src/redis.js
+++ b/helper-apps/cortex-file-handler/src/redis.js
@@ -1,16 +1,44 @@
 import Redis from "ioredis";
+import { getDefaultContainerName } from "./constants.js";
 
 const connectionString = process.env["REDIS_CONNECTION_STRING"];
 
 /**
- * Get hash key for Redis storage
- * No scoping needed - single container only
+ * Get key for Redis storage.
+ *
+ * IMPORTANT:
+ * - We **never** write hash+container scoped keys anymore (legacy only).
+ * - We *do* support (optional) hash+contextId scoping for per-user/per-context storage.
+ * - For reads, we can fall back to legacy hash+container keys if they still exist in Redis.
+ *
+ * Key format:
+ * - No context:        "<hash>"
+ * - With contextId:     "<hash>:ctx:<contextId>"
+ *
  * @param {string} hash - The file hash
- * @returns {string} The hash key (just the hash itself)
+ * @param {string|null} contextId - Optional context id
+ * @returns {string} The redis key for this hash/context
  */
-export const getScopedHashKey = (hash) => {
-  // No scoping - just return the hash directly
-  return hash;
+export const getScopedHashKey = (hash, contextId = null) => {
+  if (!hash) return hash;
+  if (!contextId) return hash;
+  return `${hash}:ctx:${contextId}`;
+};
+
+const tryParseCtxKey = (key) => {
+  if (!key || typeof key !== "string") return null;
+  const marker = ":ctx:";
+  const idx = key.indexOf(marker);
+  if (idx === -1) return null;
+  const hash = key.slice(0, idx);
+  const contextId = key.slice(idx + marker.length);
+  if (!hash || !contextId) return null;
+  return { hash, contextId };
+};
+
+const legacyContainerKey = (hash, containerName) => {
+  if (!hash || !containerName) return null;
+  return `${hash}:${containerName}`;
 };
 
 // Create a mock client for test environment when Redis is not configured
@@ -220,23 +248,31 @@ const setFileStoreMap = async (key, value) => {
 const getFileStoreMap = async (key, skipLazyCleanup = false) => {
   try {
     let value = await client.hget("FileStoreMap", key);
-    
-    // Backwards compatibility: if not found and key is for default container, try legacy key
-    if (!value && key && key.includes(':')) {
-      const [hash, containerName] = key.split(':', 2);
-      const defaultContainerName = getDefaultContainerName();
-      
-      // If this is the default container, try the legacy key (hash without container)
-      if (containerName === defaultContainerName) {
-        console.log(`Key ${key} not found, trying legacy key ${hash} for backwards compatibility`);
-        value = await client.hget("FileStoreMap", hash);
-        
-        // If found with legacy key, migrate it to the new scoped key
-        if (value) {
-          console.log(`Found value with legacy key ${hash}, migrating to new key ${key}`);
-          await client.hset("FileStoreMap", key, value);
-          // Optionally remove the old key after migration
-          // await client.hdel("FileStoreMap", hash);
+
+    // Backwards compatibility for unscoped keys only:
+    // If unscoped hash doesn't exist, fall back to legacy hash+container key (if still present).
+    // SECURITY: Context-scoped keys (hash:ctx:contextId) NEVER fall back - they must match exactly.
+    if (!value && key) {
+      const ctx = tryParseCtxKey(key);
+      const baseHash = ctx?.hash || key;
+
+      // Only allow fallback for unscoped keys (not context-scoped)
+      // Context-scoped keys are security-isolated and must match exactly
+      if (!ctx && baseHash && !String(baseHash).includes(":")) {
+        const defaultContainerName = getDefaultContainerName();
+        const legacyKey = legacyContainerKey(baseHash, defaultContainerName);
+        if (legacyKey) {
+          value = await client.hget("FileStoreMap", legacyKey);
+          if (value) {
+            console.log(
+              `Found legacy container-scoped key ${legacyKey} for hash ${baseHash}; migrating to unscoped key`,
+            );
+            // Migrate to unscoped key (we do NOT write legacy container-scoped keys)
+            await client.hset("FileStoreMap", baseHash, value);
+            // Delete the legacy key after migration
+            await client.hdel("FileStoreMap", legacyKey);
+            console.log(`Deleted legacy key ${legacyKey} after migration`);
+          }
         }
       }
     }
@@ -319,11 +355,29 @@ const removeFromFileStoreMap = async (key) => {
     // hdel returns the number of keys that were removed.
     // If the key does not exist, 0 is returned.
     const result = await client.hdel("FileStoreMap", key);
-    if (result === 0) {
-      console.log(`The key ${key} does not exist`);
-    } else {
+    if (result > 0) {
       console.log(`The key ${key} was removed successfully`);
     }
+
+    // Always try to clean up legacy container-scoped entry as well.
+    // This ensures we don't leave orphaned legacy keys behind.
+    const ctx = tryParseCtxKey(key);
+    const baseHash = ctx?.hash || key;
+    // Only attempt legacy cleanup if baseHash doesn't contain a colon (not already scoped)
+    if (!String(baseHash).includes(":")) {
+      const defaultContainerName = getDefaultContainerName();
+      const legacyKey = legacyContainerKey(baseHash, defaultContainerName);
+      if (legacyKey) {
+        const legacyResult = await client.hdel("FileStoreMap", legacyKey);
+        if (legacyResult > 0) {
+          console.log(`Removed legacy key ${legacyKey} successfully`);
+        }
+      }
+    }
+
+    if (result === 0) {
+      console.log(`The key ${key} does not exist (may have been migrated or already deleted)`);
+    }
   } catch (error) {
     console.error(`Error removing key from FileStoreMap: ${error}`);
   }
diff --git a/helper-apps/cortex-file-handler/tests/containerNameParsing.test.js b/helper-apps/cortex-file-handler/tests/containerNameParsing.test.js
index efd0b0a3..9be7177c 100644
--- a/helper-apps/cortex-file-handler/tests/containerNameParsing.test.js
+++ b/helper-apps/cortex-file-handler/tests/containerNameParsing.test.js
@@ -3,6 +3,7 @@ import {
   AZURE_STORAGE_CONTAINER_NAME, 
   getDefaultContainerName,
 } from "../src/blobHandler.js";
+import { getContainerName } from "../src/constants.js";
 
 // Mock environment variables for testing
 const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
@@ -43,3 +44,77 @@ test("container name should default to cortextempfiles when env var is not set",
   const result = getContainerName();
   t.is(result, "cortextempfiles");
 });
+
+test("getContainerName should handle comma-separated legacy values and use the last one", (t) => {
+  // Set comma-separated value (legacy format)
+  process.env.AZURE_STORAGE_CONTAINER_NAME = "container1,container2,container3";
+  
+  // Capture console.warn to verify warning is logged
+  const warnings = [];
+  const originalWarn = console.warn;
+  console.warn = (message) => {
+    warnings.push(message);
+    originalWarn(message);
+  };
+  
+  try {
+    const result = getContainerName();
+    
+    // Should return the last container name
+    t.is(result, "container3", "Should return the last container from comma-separated list");
+    
+    // Should log a warning
+    t.true(warnings.length > 0, "Should log a warning about comma-separated values");
+    t.true(
+      warnings.some(w => w.includes("AZURE_STORAGE_CONTAINER_NAME contains comma-separated values")),
+      "Warning should mention comma-separated values"
+    );
+  } finally {
+    console.warn = originalWarn;
+  }
+});
+
+test("getContainerName should handle comma-separated values with spaces", (t) => {
+  // Set comma-separated value with spaces
+  process.env.AZURE_STORAGE_CONTAINER_NAME = "container1 , container2 , container3 ";
+  
+  const warnings = [];
+  const originalWarn = console.warn;
+  console.warn = (message) => {
+    warnings.push(message);
+    originalWarn(message);
+  };
+  
+  try {
+    const result = getContainerName();
+    
+    // Should return the last container name (trimmed)
+    t.is(result, "container3", "Should return the last container (trimmed) from comma-separated list");
+  } finally {
+    console.warn = originalWarn;
+  }
+});
+
+test("getContainerName should handle single container name (no comma)", (t) => {
+  // Set single container name
+  process.env.AZURE_STORAGE_CONTAINER_NAME = "my-container";
+  
+  const warnings = [];
+  const originalWarn = console.warn;
+  console.warn = (message) => {
+    warnings.push(message);
+    originalWarn(message);
+  };
+  
+  try {
+    const result = getContainerName();
+    
+    // Should return the container name as-is
+    t.is(result, "my-container", "Should return the container name as-is when no comma");
+    
+    // Should NOT log a warning
+    t.is(warnings.length, 0, "Should NOT log a warning for single container name");
+  } finally {
+    console.warn = originalWarn;
+  }
+});
diff --git a/helper-apps/cortex-file-handler/tests/deleteOperations.test.js b/helper-apps/cortex-file-handler/tests/deleteOperations.test.js
index 16dbe110..b70f67c8 100644
--- a/helper-apps/cortex-file-handler/tests/deleteOperations.test.js
+++ b/helper-apps/cortex-file-handler/tests/deleteOperations.test.js
@@ -534,9 +534,7 @@ test.serial("should handle backwards compatibility key removal correctly", async
 
     // Manually create a legacy unscoped key to test backwards compatibility
     const { setFileStoreMap, getFileStoreMap, getScopedHashKey } = await import("../src/redis.js");
-    const { getDefaultContainerName } = await import("../src/constants.js");
-    const defaultContainer = getDefaultContainerName();
-    const scopedHash = getScopedHashKey(testHash, defaultContainer);
+    const scopedHash = getScopedHashKey(testHash);
     const hashResult = await getFileStoreMap(scopedHash);
     
     if (hashResult) {
diff --git a/helper-apps/cortex-file-handler/tests/redisMigration.test.js b/helper-apps/cortex-file-handler/tests/redisMigration.test.js
new file mode 100644
index 00000000..42fbe417
--- /dev/null
+++ b/helper-apps/cortex-file-handler/tests/redisMigration.test.js
@@ -0,0 +1,519 @@
+import test from "ava";
+import { v4 as uuidv4 } from "uuid";
+
+import {
+  setFileStoreMap,
+  getFileStoreMap,
+  removeFromFileStoreMap,
+  getScopedHashKey,
+  client,
+} from "../src/redis.js";
+import { getDefaultContainerName } from "../src/constants.js";
+
+/**
+ * Tests for Redis key migration logic.
+ * 
+ * Key formats:
+ * - Legacy: `<hash>:<containerName>` (read-only, migrated on access)
+ * - Current: `<hash>` (unscoped) or `<hash>:ctx:<contextId>` (context-scoped)
+ * 
+ * Migration behavior:
+ * - On read: If legacy key found, copy to new key, delete legacy key
+ * - On write: Always write to new format only
+ * - On delete: Clean up both new and legacy keys
+ */
+
+// Helper to create a legacy key directly in Redis (simulating old data)
+async function setLegacyKey(hash, containerName, value) {
+  const legacyKey = `${hash}:${containerName}`;
+  await client.hset("FileStoreMap", legacyKey, JSON.stringify(value));
+  return legacyKey;
+}
+
+// Helper to check if a key exists in Redis
+async function keyExists(key) {
+  const value = await client.hget("FileStoreMap", key);
+  return value !== null;
+}
+
+// Helper to get raw value from Redis (without migration logic)
+async function getRawKey(key) {
+  const value = await client.hget("FileStoreMap", key);
+  return value ? JSON.parse(value) : null;
+}
+
+// Helper to delete a key directly
+async function deleteRawKey(key) {
+  await client.hdel("FileStoreMap", key);
+}
+
+test.beforeEach(() => {
+  // Tests use the mock Redis client automatically (NODE_ENV=test)
+});
+
+// =============================================================================
+// getScopedHashKey tests
+// =============================================================================
+
+test("getScopedHashKey - returns hash when no contextId", (t) => {
+  const hash = "abc123";
+  const result = getScopedHashKey(hash);
+  t.is(result, "abc123");
+});
+
+test("getScopedHashKey - returns hash when contextId is null", (t) => {
+  const hash = "abc123";
+  const result = getScopedHashKey(hash, null);
+  t.is(result, "abc123");
+});
+
+test("getScopedHashKey - returns context-scoped key when contextId provided", (t) => {
+  const hash = "abc123";
+  const contextId = "user-456";
+  const result = getScopedHashKey(hash, contextId);
+  t.is(result, "abc123:ctx:user-456");
+});
+
+test("getScopedHashKey - handles empty hash", (t) => {
+  t.is(getScopedHashKey(""), "");
+  t.is(getScopedHashKey(null), null);
+  t.is(getScopedHashKey(undefined), undefined);
+});
+
+// =============================================================================
+// Legacy key migration on READ
+// =============================================================================
+
+test("getFileStoreMap - migrates legacy container-scoped key to unscoped key", async (t) => {
+  const hash = `test-migrate-${uuidv4()}`;
+  const containerName = getDefaultContainerName();
+  const legacyKey = `${hash}:${containerName}`;
+  const testData = {
+    url: "http://example.com/file.txt",
+    filename: "file.txt",
+    timestamp: new Date().toISOString(),
+  };
+
+  // Set up legacy key directly in Redis
+  await setLegacyKey(hash, containerName, testData);
+
+  // Verify legacy key exists before migration
+  t.true(await keyExists(legacyKey), "Legacy key should exist before read");
+  t.false(await keyExists(hash), "New key should not exist before read");
+
+  // Read using unscoped hash - should trigger migration
+  const result = await getFileStoreMap(hash, true); // skipLazyCleanup=true to avoid storage checks
+
+  // Verify data was returned correctly
+  t.truthy(result, "Should return the migrated data");
+  t.is(result.url, testData.url);
+  t.is(result.filename, testData.filename);
+
+  // Verify migration occurred: new key exists, legacy key deleted
+  t.true(await keyExists(hash), "New unscoped key should exist after migration");
+  t.false(await keyExists(legacyKey), "Legacy key should be deleted after migration");
+
+  // Cleanup
+  await deleteRawKey(hash);
+});
+
+test("getFileStoreMap - does not migrate when unscoped key already exists", async (t) => {
+  const hash = `test-no-migrate-${uuidv4()}`;
+  const containerName = getDefaultContainerName();
+  const legacyKey = `${hash}:${containerName}`;
+
+  const currentData = { url: "http://current.com/file.txt", filename: "current.txt" };
+  const legacyData = { url: "http://legacy.com/file.txt", filename: "legacy.txt" };
+
+  // Set up both keys
+  await client.hset("FileStoreMap", hash, JSON.stringify(currentData));
+  await setLegacyKey(hash, containerName, legacyData);
+
+  // Read using unscoped hash
+  const result = await getFileStoreMap(hash, true);
+
+  // Should return current data, not legacy
+  t.is(result.url, currentData.url, "Should return current data, not legacy");
+
+  // Legacy key should still exist (not touched since current key was found first)
+  t.true(await keyExists(legacyKey), "Legacy key should still exist");
+
+  // Cleanup
+  await deleteRawKey(hash);
+  await deleteRawKey(legacyKey);
+});
+
+test("getFileStoreMap - context-scoped key does NOT fall back to unscoped hash (security)", async (t) => {
+  const hash = `test-ctx-no-fallback-${uuidv4()}`;
+  const contextId = "user-123";
+  const contextKey = `${hash}:ctx:${contextId}`;
+
+  const unscopedData = { url: "http://unscoped.com/file.txt", filename: "unscoped.txt" };
+
+  // Only set unscoped key (no context-scoped key)
+  await client.hset("FileStoreMap", hash, JSON.stringify(unscopedData));
+
+  // Read using context-scoped key - should NOT fall back for security
+  const result = await getFileStoreMap(contextKey, true);
+
+  // Should NOT return unscoped data (security isolation)
+  t.is(result, null, "Should NOT fall back to unscoped data for security");
+  
+  // Unscoped key should still exist
+  t.true(await keyExists(hash), "Unscoped key should still exist");
+
+  // Cleanup
+  await deleteRawKey(hash);
+});
+
+test("getFileStoreMap - context-scoped key does NOT fall back through unscoped to legacy (security)", async (t) => {
+  const hash = `test-ctx-legacy-no-fallback-${uuidv4()}`;
+  const contextId = "user-456";
+  const contextKey = `${hash}:ctx:${contextId}`;
+  const containerName = getDefaultContainerName();
+  const legacyKey = `${hash}:${containerName}`;
+
+  const legacyData = { url: "http://legacy.com/file.txt", filename: "legacy.txt" };
+
+  // Only set legacy key (no context-scoped or unscoped keys)
+  await setLegacyKey(hash, containerName, legacyData);
+
+  // Read using context-scoped key - should NOT fall back for security
+  const result = await getFileStoreMap(contextKey, true);
+
+  // Should NOT return legacy data (security isolation)
+  t.is(result, null, "Should NOT fall back to legacy data for security");
+  
+  // Legacy key should still exist (not migrated)
+  t.true(await keyExists(legacyKey), "Legacy key should still exist");
+  t.false(await keyExists(hash), "Unscoped key should NOT be created");
+
+  // Cleanup
+  await deleteRawKey(legacyKey);
+});
+
+// =============================================================================
+// Write behavior - always uses new format
+// =============================================================================
+
+test("setFileStoreMap - writes to the key provided (unscoped)", async (t) => {
+  const hash = `test-write-unscoped-${uuidv4()}`;
+  const testData = { url: "http://example.com/file.txt", filename: "file.txt" };
+
+  await setFileStoreMap(hash, testData);
+
+  // Verify it was written to the unscoped key
+  const result = await getRawKey(hash);
+  t.truthy(result);
+  t.is(result.url, testData.url);
+  t.truthy(result.timestamp, "Should add timestamp");
+
+  // Cleanup
+  await deleteRawKey(hash);
+});
+
+test("setFileStoreMap - writes to context-scoped key when provided", async (t) => {
+  const hash = `test-write-ctx-${uuidv4()}`;
+  const contextId = "user-789";
+  const contextKey = getScopedHashKey(hash, contextId);
+  const testData = { url: "http://example.com/file.txt", filename: "file.txt" };
+
+  await setFileStoreMap(contextKey, testData);
+
+  // Verify it was written to the context-scoped key
+  const result = await getRawKey(contextKey);
+  t.truthy(result);
+  t.is(result.url, testData.url);
+
+  // Unscoped key should NOT exist
+  t.false(await keyExists(hash), "Unscoped key should not be created");
+
+  // Cleanup
+  await deleteRawKey(contextKey);
+});
+
+// =============================================================================
+// Delete behavior - cleans up both new and legacy keys
+// =============================================================================
+
+test("removeFromFileStoreMap - deletes unscoped key and legacy key", async (t) => {
+  const hash = `test-delete-both-${uuidv4()}`;
+  const containerName = getDefaultContainerName();
+  const legacyKey = `${hash}:${containerName}`;
+
+  const testData = { url: "http://example.com/file.txt", filename: "file.txt" };
+
+  // Set up both keys
+  await client.hset("FileStoreMap", hash, JSON.stringify(testData));
+  await setLegacyKey(hash, containerName, testData);
+
+  // Verify both exist
+  t.true(await keyExists(hash));
+  t.true(await keyExists(legacyKey));
+
+  // Delete using unscoped hash
+  await removeFromFileStoreMap(hash);
+
+  // Both should be gone
+  t.false(await keyExists(hash), "Unscoped key should be deleted");
+  t.false(await keyExists(legacyKey), "Legacy key should also be deleted");
+});
+
+test("removeFromFileStoreMap - deletes legacy key even when unscoped doesn't exist", async (t) => {
+  const hash = `test-delete-legacy-only-${uuidv4()}`;
+  const containerName = getDefaultContainerName();
+  const legacyKey = `${hash}:${containerName}`;
+
+  const testData = { url: "http://example.com/file.txt", filename: "file.txt" };
+
+  // Only set legacy key
+  await setLegacyKey(hash, containerName, testData);
+
+  // Verify only legacy exists
+  t.false(await keyExists(hash));
+  t.true(await keyExists(legacyKey));
+
+  // Delete using unscoped hash
+  await removeFromFileStoreMap(hash);
+
+  // Legacy should be gone
+  t.false(await keyExists(legacyKey), "Legacy key should be deleted");
+});
+
+test("removeFromFileStoreMap - handles context-scoped key deletion", async (t) => {
+  const hash = `test-delete-ctx-${uuidv4()}`;
+  const contextId = "user-delete";
+  const contextKey = `${hash}:ctx:${contextId}`;
+  const containerName = getDefaultContainerName();
+  const legacyKey = `${hash}:${containerName}`;
+
+  const testData = { url: "http://example.com/file.txt", filename: "file.txt" };
+
+  // Set up context-scoped key and legacy key
+  await client.hset("FileStoreMap", contextKey, JSON.stringify(testData));
+  await setLegacyKey(hash, containerName, testData);
+
+  // Delete using context-scoped key
+  await removeFromFileStoreMap(contextKey);
+
+  // Context key should be deleted
+  t.false(await keyExists(contextKey), "Context-scoped key should be deleted");
+  
+  // Legacy key should also be deleted (cleanup based on base hash)
+  t.false(await keyExists(legacyKey), "Legacy key should also be deleted");
+});
+
+// =============================================================================
+// Edge cases
+// =============================================================================
+
+test("getFileStoreMap - returns null when no keys exist", async (t) => {
+  const hash = `test-nonexistent-${uuidv4()}`;
+  const result = await getFileStoreMap(hash, true);
+  t.is(result, null);
+});
+
+test("migration - preserves all original data fields", async (t) => {
+  const hash = `test-preserve-fields-${uuidv4()}`;
+  const containerName = getDefaultContainerName();
+  
+  const originalData = {
+    url: "http://example.com/file.txt",
+    gcs: "gs://bucket/file.txt",
+    filename: "file.txt",
+    hash: hash,
+    timestamp: "2024-01-01T00:00:00.000Z",
+    customField: "custom-value",
+    nested: { key: "value" },
+  };
+
+  // Set up legacy key
+  await setLegacyKey(hash, containerName, originalData);
+
+  // Read to trigger migration
+  const result = await getFileStoreMap(hash, true);
+
+  // Verify all fields are preserved
+  t.is(result.url, originalData.url);
+  t.is(result.gcs, originalData.gcs);
+  t.is(result.filename, originalData.filename);
+  t.is(result.hash, originalData.hash);
+  t.is(result.timestamp, originalData.timestamp);
+  t.is(result.customField, originalData.customField);
+  t.deepEqual(result.nested, originalData.nested);
+
+  // Cleanup
+  await deleteRawKey(hash);
+});
+
+test("migration - does not affect keys with colons in hash", async (t) => {
+  // Keys that already contain colons (like context-scoped keys) should not
+  // trigger legacy migration logic
+  const contextKey = `somehash:ctx:user123`;
+  const testData = { url: "http://example.com/file.txt", filename: "file.txt" };
+
+  await client.hset("FileStoreMap", contextKey, JSON.stringify(testData));
+
+  // Reading should just return the data without trying legacy migration
+  const result = await getFileStoreMap(contextKey, true);
+  t.truthy(result);
+  t.is(result.url, testData.url);
+
+  // Cleanup
+  await deleteRawKey(contextKey);
+});
+
+// =============================================================================
+// Security: Context-scoped isolation
+// =============================================================================
+
+test("getFileStoreMap - context-scoped file cannot be accessed without contextId", async (t) => {
+  const hash = `test-security-${uuidv4()}`;
+  const contextId = "user-secure";
+  const contextKey = `${hash}:ctx:${contextId}`;
+  const testData = {
+    url: "http://example.com/secure-file.txt",
+    filename: "secure-file.txt",
+    timestamp: new Date().toISOString(),
+  };
+
+  // Write file with contextId
+  await setFileStoreMap(contextKey, testData);
+
+  // Verify context-scoped key exists
+  t.true(await keyExists(contextKey), "Context-scoped key should exist");
+
+  // Try to read WITHOUT contextId - should NOT find it
+  const unscopedResult = await getFileStoreMap(hash, true);
+  t.is(unscopedResult, null, "Should NOT be able to read context-scoped file without contextId");
+
+  // Try to read WITH correct contextId - should find it
+  const scopedResult = await getFileStoreMap(contextKey, true);
+  t.truthy(scopedResult, "Should be able to read with correct contextId");
+  t.is(scopedResult.url, testData.url);
+
+  // Cleanup
+  await deleteRawKey(contextKey);
+});
+
+test("getFileStoreMap - context-scoped file cannot be accessed with wrong contextId", async (t) => {
+  const hash = `test-security-wrong-${uuidv4()}`;
+  const correctContextId = "user-correct";
+  const wrongContextId = "user-wrong";
+  const correctKey = `${hash}:ctx:${correctContextId}`;
+  const wrongKey = `${hash}:ctx:${wrongContextId}`;
+  const testData = {
+    url: "http://example.com/secure-file.txt",
+    filename: "secure-file.txt",
+    timestamp: new Date().toISOString(),
+  };
+
+  // Write file with correct contextId
+  await setFileStoreMap(correctKey, testData);
+
+  // Try to read with wrong contextId - should NOT find it
+  const wrongResult = await getFileStoreMap(wrongKey, true);
+  t.is(wrongResult, null, "Should NOT be able to read with wrong contextId");
+
+  // Verify correct contextId still works
+  const correctResult = await getFileStoreMap(correctKey, true);
+  t.truthy(correctResult, "Should still be able to read with correct contextId");
+
+  // Cleanup
+  await deleteRawKey(correctKey);
+});
+
+test("removeFromFileStoreMap - context-scoped file cannot be deleted without contextId", async (t) => {
+  const hash = `test-security-delete-${uuidv4()}`;
+  const contextId = "user-delete-secure";
+  const contextKey = `${hash}:ctx:${contextId}`;
+  const testData = {
+    url: "http://example.com/secure-file.txt",
+    filename: "secure-file.txt",
+    timestamp: new Date().toISOString(),
+  };
+
+  // Write file with contextId
+  await setFileStoreMap(contextKey, testData);
+  t.true(await keyExists(contextKey), "Context-scoped key should exist");
+
+  // Try to delete WITHOUT contextId - should NOT delete context-scoped file
+  await removeFromFileStoreMap(hash);
+  t.true(await keyExists(contextKey), "Context-scoped key should still exist after unscoped delete attempt");
+
+  // Delete WITH correct contextId - should work
+  await removeFromFileStoreMap(contextKey);
+  t.false(await keyExists(contextKey), "Context-scoped key should be deleted with correct contextId");
+});
+
+test("getFileStoreMap - unscoped file can be read without contextId", async (t) => {
+  const hash = `test-unscoped-${uuidv4()}`;
+  const testData = {
+    url: "http://example.com/unscoped-file.txt",
+    filename: "unscoped-file.txt",
+    timestamp: new Date().toISOString(),
+  };
+
+  // Write file without contextId (unscoped)
+  await setFileStoreMap(hash, testData);
+
+  // Should be able to read without contextId
+  const result = await getFileStoreMap(hash, true);
+  t.truthy(result, "Should be able to read unscoped file without contextId");
+  t.is(result.url, testData.url);
+
+  // Cleanup
+  await deleteRawKey(hash);
+});
+
+test("getFileStoreMap - unscoped file can fall back to legacy container-scoped key", async (t) => {
+  const hash = `test-legacy-fallback-${uuidv4()}`;
+  const containerName = getDefaultContainerName();
+  const legacyKey = `${hash}:${containerName}`;
+  const testData = {
+    url: "http://example.com/legacy-file.txt",
+    filename: "legacy-file.txt",
+    timestamp: new Date().toISOString(),
+  };
+
+  // Set up legacy key (no unscoped or context-scoped key exists)
+  await setLegacyKey(hash, containerName, testData);
+
+  // Reading unscoped hash should find and migrate legacy key
+  const result = await getFileStoreMap(hash, true);
+  t.truthy(result, "Should find legacy key when reading unscoped hash");
+  t.is(result.url, testData.url);
+
+  // Legacy key should be migrated (deleted)
+  t.false(await keyExists(legacyKey), "Legacy key should be deleted after migration");
+  t.true(await keyExists(hash), "Unscoped key should exist after migration");
+
+  // Cleanup
+  await deleteRawKey(hash);
+});
+
+test("getFileStoreMap - context-scoped read does NOT fall back to unscoped or legacy", async (t) => {
+  const hash = `test-no-fallback-${uuidv4()}`;
+  const contextId = "user-no-fallback";
+  const contextKey = `${hash}:ctx:${contextId}`;
+  const containerName = getDefaultContainerName();
+  const legacyKey = `${hash}:${containerName}`;
+  const unscopedData = { url: "http://example.com/unscoped.txt", filename: "unscoped.txt" };
+  const legacyData = { url: "http://example.com/legacy.txt", filename: "legacy.txt" };
+
+  // Set up unscoped and legacy keys (but NOT context-scoped)
+  await setFileStoreMap(hash, unscopedData);
+  await setLegacyKey(hash, containerName, legacyData);
+
+  // Try to read with contextId - should NOT find unscoped or legacy
+  const result = await getFileStoreMap(contextKey, true);
+  t.is(result, null, "Context-scoped read should NOT fall back to unscoped or legacy keys");
+
+  // Verify unscoped and legacy keys still exist
+  t.true(await keyExists(hash), "Unscoped key should still exist");
+  t.true(await keyExists(legacyKey), "Legacy key should still exist");
+
+  // Cleanup
+  await deleteRawKey(hash);
+  await deleteRawKey(legacyKey);
+});
diff --git a/helper-apps/cortex-file-handler/tests/setRetention.test.js b/helper-apps/cortex-file-handler/tests/setRetention.test.js
index e0492657..756566c6 100644
--- a/helper-apps/cortex-file-handler/tests/setRetention.test.js
+++ b/helper-apps/cortex-file-handler/tests/setRetention.test.js
@@ -18,7 +18,6 @@ import {
   removeFromFileStoreMap,
   getScopedHashKey
 } from "../src/redis.js";
-import { getDefaultContainerName } from "../src/constants.js";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -136,8 +135,7 @@ test.serial("should set file retention to permanent", async (t) => {
     // Cleanup
     try {
       const { getScopedHashKey } = await import("../src/redis.js");
-      const container = getDefaultContainerName();
-      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+      await removeFromFileStoreMap(getScopedHashKey(testHash));
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -180,8 +178,7 @@ test.serial("should set file retention to temporary", async (t) => {
     // Cleanup
     try {
       const { getScopedHashKey } = await import("../src/redis.js");
-      const container = getDefaultContainerName();
-      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+      await removeFromFileStoreMap(getScopedHashKey(testHash));
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -217,8 +214,7 @@ test.serial("should set retention using request body parameters", async (t) => {
     // Cleanup
     try {
       const { getScopedHashKey } = await import("../src/redis.js");
-      const container = getDefaultContainerName();
-      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+      await removeFromFileStoreMap(getScopedHashKey(testHash));
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -299,8 +295,7 @@ test.serial("should update Redis map with retention information", async (t) => {
     await new Promise((resolve) => setTimeout(resolve, 1000));
 
     // Verify Redis entry exists
-    const container = getDefaultContainerName();
-    const scopedHash = getScopedHashKey(testHash, container);
+    const scopedHash = getScopedHashKey(testHash);
     const oldEntry = await getFileStoreMap(scopedHash);
     t.truthy(oldEntry, "Redis entry should exist before setting retention");
 
@@ -322,8 +317,7 @@ test.serial("should update Redis map with retention information", async (t) => {
     // Cleanup
     try {
       const { getScopedHashKey } = await import("../src/redis.js");
-      const container = getDefaultContainerName();
-      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+      await removeFromFileStoreMap(getScopedHashKey(testHash));
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -372,8 +366,7 @@ test.serial("should preserve file metadata after setting retention", async (t) =
     // Cleanup
     try {
       const { getScopedHashKey } = await import("../src/redis.js");
-      const container = getDefaultContainerName();
-      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+      await removeFromFileStoreMap(getScopedHashKey(testHash));
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -424,8 +417,7 @@ test.serial("should support operation=setRetention query parameter", async (t) =
     // Cleanup
     try {
       const { getScopedHashKey } = await import("../src/redis.js");
-      const container = getDefaultContainerName();
-      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+      await removeFromFileStoreMap(getScopedHashKey(testHash));
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -485,8 +477,7 @@ test.serial("should preserve GCS URL when setting retention", async (t) => {
     // Cleanup
     try {
       const { getScopedHashKey } = await import("../src/redis.js");
-      const container = getDefaultContainerName();
-      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+      await removeFromFileStoreMap(getScopedHashKey(testHash));
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -524,8 +515,7 @@ test.serial("should always include shortLivedUrl in response", async (t) => {
     // Cleanup
     try {
       const { getScopedHashKey } = await import("../src/redis.js");
-      const container = getDefaultContainerName();
-      await removeFromFileStoreMap(getScopedHashKey(testHash, container));
+      await removeFromFileStoreMap(getScopedHashKey(testHash));
     } catch (e) {
       // Ignore cleanup errors
     }

From 34ba4aa08987634bc69dd9595aa962c571a8c4e5 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Tue, 16 Dec 2025 10:28:31 -0700
Subject: [PATCH 05/27] feat: enhance file handling with context-aware
 functions and improved retention management

- Introduced `fetchFileFromUrl` to streamline file retrieval from URLs, supporting context IDs for scoped file storage.
- Updated `buildFileHandlerUrl` to handle context IDs and improve query parameter management.
- Enhanced existing functions like `getMediaChunks`, `markCompletedForCleanUp`, and `deleteFileByHash` to accept context IDs, ensuring better file management in multi-tenant environments.
- Improved retention management by adding `setRetentionForHash` to allow setting file retention policies.
- Updated various tools and plugins to utilize new context-aware file handling functions, ensuring consistent behavior across the application.
- Enhanced tests to validate new functionality and ensure robust error handling.
---
 lib/fileUtils.js                              | 281 ++++++++++++------
 .../system/entity/tools/sys_tool_editfile.js  |   5 +-
 .../entity/tools/sys_tool_file_collection.js  |  22 +-
 .../system/entity/tools/sys_tool_image.js     |   8 +-
 .../entity/tools/sys_tool_image_gemini.js     |   7 +-
 .../entity/tools/sys_tool_view_image.js       |   2 +-
 .../system/entity/tools/sys_tool_writefile.js |   6 +-
 server/executeWorkspace.js                    |   2 +-
 server/plugins/azureCognitivePlugin.js        |  23 +-
 server/plugins/azureVideoTranslatePlugin.js   |  25 +-
 tests/unit/core/shortLivedUrl.test.js         |   2 +-
 11 files changed, 253 insertions(+), 130 deletions(-)

diff --git a/lib/fileUtils.js b/lib/fileUtils.js
index 88c30897..ca28f3b3 100644
--- a/lib/fileUtils.js
+++ b/lib/fileUtils.js
@@ -84,6 +84,65 @@ async function computeBufferHash(buffer) {
     return xxh64.digest().toString(16);
 }
 
+/**
+ * Fetch/load a file from URL via file handler
+ * Downloads file from URL, processes it, and returns the result
+ * @param {string} fileUrl - URL of file to fetch
+ * @param {string} requestId - Request ID for tracking
+ * @param {string|null} contextId - Optional context ID for scoped file storage
+ * @param {boolean} save - Whether to save the file (default: false)
+ * @returns {Promise<Object>} Response data with file information
+ */
+async function fetchFileFromUrl(fileUrl, requestId, contextId = null, save = false) {
+    const fileHandlerUrl = MEDIA_API_URL;
+    if (!fileHandlerUrl || fileHandlerUrl === 'null') {
+        throw new Error('File handler URL is not configured');
+    }
+
+    const url = buildFileHandlerUrl(fileHandlerUrl, {
+        fetch: fileUrl,
+        requestId,
+        ...(contextId ? { contextId } : {}),
+        ...(save ? { save: true } : {})
+    });
+
+    const response = await axios.get(url, { timeout: 60000 });
+    
+    if (!response.data?.url && !Array.isArray(response.data)) {
+        throw new Error("File handler did not return valid data");
+    }
+
+    return response.data;
+}
+
+/**
+ * Build a file handler URL with query parameters
+ * Handles separator detection (? vs &) and parameter encoding
+ * @param {string} baseUrl - Base file handler URL
+ * @param {Object} params - Query parameters as key-value pairs (null/undefined values are skipped)
+ * @returns {string} Complete URL with query parameters
+ */
+function buildFileHandlerUrl(baseUrl, params = {}) {
+    if (!baseUrl) {
+        throw new Error('baseUrl is required');
+    }
+    
+    const separator = baseUrl.includes('?') ? '&' : '?';
+    const queryParams = [];
+    
+    Object.entries(params).forEach(([key, value]) => {
+        if (value != null && value !== '') {
+            queryParams.push(`${encodeURIComponent(key)}=${encodeURIComponent(value)}`);
+        }
+    });
+    
+    if (queryParams.length === 0) {
+        return baseUrl;
+    }
+    
+    return `${baseUrl}${separator}${queryParams.join('&')}`;
+}
+
 async function deleteTempPath(path) {
     try {
         if (!path) {
@@ -147,11 +206,22 @@ const downloadFile = async (fileUrl) => {
     });
 };
 
-async function getMediaChunks(file, requestId) {
+/**
+ * Get media chunks from file handler (for chunked media files)
+ * @param {string} file - File URL or URI
+ * @param {string} requestId - Request ID for tracking
+ * @param {string|null} contextId - Optional context ID for scoped file storage
+ * @returns {Promise<Array>} Array of chunk URLs
+ */
+async function getMediaChunks(file, requestId, contextId = null) {
     try {
         if (MEDIA_API_URL) {
-            //call helper api and get list of file uris
-            const res = await axios.get(MEDIA_API_URL, { params: { uri: file, requestId } });
+            const url = buildFileHandlerUrl(MEDIA_API_URL, {
+                uri: file,
+                requestId,
+                ...(contextId ? { contextId } : {})
+            });
+            const res = await axios.get(url, { timeout: 30000 });
             return res.data;
         } else {
             logger.info(`No API_URL set, returning file as chunk`);
@@ -163,27 +233,37 @@ async function getMediaChunks(file, requestId) {
     }
 }
 
-async function markCompletedForCleanUp(requestId) {
+/**
+ * Mark a request as completed for cleanup in file handler
+ * @param {string} requestId - Request ID to mark as completed
+ * @param {string|null} contextId - Optional context ID for scoped file storage
+ * @returns {Promise<Object|null>} Response data or null
+ */
+async function markCompletedForCleanUp(requestId, contextId = null) {
     try {
         if (MEDIA_API_URL) {
-            //call helper api to mark processing as completed
-            const res = await axios.delete(MEDIA_API_URL, { params: { requestId } });
+            const url = buildFileHandlerUrl(MEDIA_API_URL, {
+                requestId,
+                ...(contextId ? { contextId } : {})
+            });
+            const res = await axios.delete(url, { timeout: 15000 });
             logger.info(`Marked request ${requestId} as completed: ${JSON.stringify(res.data)}`);
             return res.data;
         }
     } catch (err) {
         logger.error(`Error marking request ${requestId} as completed: ${err}`);
     }
+    return null;
 }
 
 /**
  * Delete a file from cloud storage by hash
  * @param {string} hash - File hash to delete
  * @param {pathwayResolver} pathwayResolver - Optional pathway resolver for logging
- * @param {string} container - Optional container name where the file is stored
+ * @param {string|null} contextId - Optional but strongly recommended context id for scoped hashes
  * @returns {Promise<boolean>} True if file was deleted, false if not found or error
  */
-async function deleteFileByHash(hash, pathwayResolver = null, container = null) {
+async function deleteFileByHash(hash, pathwayResolver = null, contextId = null) {
     if (!hash || typeof hash !== 'string') {
         logger.warn('deleteFileByHash: hash is required and must be a string');
         return false;
@@ -196,13 +276,10 @@ async function deleteFileByHash(hash, pathwayResolver = null, container = null)
     }
 
     try {
-        const separator = fileHandlerUrl.includes('?') ? '&' : '?';
-        let deleteUrl = `${fileHandlerUrl}${separator}hash=${encodeURIComponent(hash)}`;
-        
-        // Add container parameter if provided
-        if (container) {
-            deleteUrl += `&container=${encodeURIComponent(container)}`;
-        }
+        const deleteUrl = buildFileHandlerUrl(fileHandlerUrl, {
+            hash,
+            ...(contextId ? { contextId } : {})
+        });
         
         const response = await axios.delete(deleteUrl, {
             validateStatus: (status) => status >= 200 && status < 500, // Accept 200-499 as valid responses
@@ -606,10 +683,8 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
         throw new Error("contextId and filename are required");
     }
 
-    // Determine container based on permanent flag
-    const containerName = permanent && process.env.CORTEX_MEDIA_PERMANENT_STORE_NAME 
-        ? process.env.CORTEX_MEDIA_PERMANENT_STORE_NAME 
-        : null;
+    // If permanent=true, set retention=permanent to keep file forever
+    const desiredRetention = permanent ? 'permanent' : 'temporary';
 
     // If fileUrl is provided and url is not already a cloud URL, upload the file first
     let finalUrl = url;
@@ -620,11 +695,22 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
         // Upload the file from the URL
         // uploadFileToCloud will download it, compute hash, check if it exists, and upload if needed
         // It uploads the local file stream, not the URL, to avoid triggering remoteFile fetch
-        const uploadResult = await uploadFileToCloud(fileUrl, null, filename, pathwayResolver, containerName);
+        const uploadResult = await uploadFileToCloud(fileUrl, null, filename, pathwayResolver, contextId);
         finalUrl = uploadResult.url;
         finalGcs = uploadResult.gcs;
         finalHash = uploadResult.hash || hash;
     }
+
+    // If the caller asked for permanence/privacy and we have a hash, update retention (best-effort)
+    if (finalHash && desiredRetention === 'permanent') {
+        try {
+            await setRetentionForHash(finalHash, desiredRetention, contextId, pathwayResolver);
+        } catch (e) {
+            const msg = `Failed to set retention=${desiredRetention} for hash ${finalHash}: ${e?.message || String(e)}`;
+            if (pathwayResolver?.logWarning) pathwayResolver.logWarning(msg);
+            else logger.warn(msg);
+        }
+    }
     
     if (!finalUrl) {
         throw new Error("url or fileUrl is required");
@@ -646,7 +732,7 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
         tags: Array.isArray(tags) ? tags : [],
         notes: notes || '',
         hash: finalHash || null,
-        permanent: permanent || false,
+        permanent: permanent,
         addedDate: new Date().toISOString(),
         lastAccessed: new Date().toISOString()
     };
@@ -1122,7 +1208,9 @@ async function generateFileMessageContent(fileParam, contextId, contextKey = nul
     }
 
     // Resolve to short-lived URL if possible
-    const fileWithShortLivedUrl = await ensureShortLivedUrl(foundFile, MEDIA_API_URL);
+    // Note: contextId is not available in this function, so we pass null
+    // This is acceptable as short-lived URLs work without contextId
+    const fileWithShortLivedUrl = await ensureShortLivedUrl(foundFile, MEDIA_API_URL, null);
 
     return {
         type: 'image_url',
@@ -1195,30 +1283,25 @@ function injectFileIntoChatHistory(chatHistory, fileContent) {
  * @param {string} hash - File hash to check
  * @param {string} fileHandlerUrl - File handler service URL
  * @param {pathwayResolver} pathwayResolver - Optional pathway resolver for logging
- * @param {string} container - Optional container name
+ * @param {string|null} contextId - Optional but strongly recommended context id for scoped hashes
  * @param {number} shortLivedMinutes - Optional duration for short-lived URL (default: 5)
- * @returns {Promise<Object|null>} {url, gcs, hash} if file exists, null otherwise
+ * @returns {Promise<Object|null>} {url, gcs, hash, filename} if file exists, null otherwise
  *   url: shortLivedUrl if available (prefers converted), otherwise regular URL
  *   gcs: GCS URL (prefers converted, no short-lived version for GCS)
+ *   filename: Original filename from file handler (if available)
  */
-async function checkHashExists(hash, fileHandlerUrl, pathwayResolver = null, container = null, shortLivedMinutes = 5) {
+async function checkHashExists(hash, fileHandlerUrl, pathwayResolver = null, contextId = null, shortLivedMinutes = 5) {
     if (!hash || !fileHandlerUrl) {
         return null;
     }
     
     try {
-        const separator = fileHandlerUrl.includes('?') ? '&' : '?';
-        let checkHashUrl = `${fileHandlerUrl}${separator}hash=${hash}&checkHash=true`;
-        
-        // Add container parameter if provided
-        if (container) {
-            checkHashUrl += `&container=${encodeURIComponent(container)}`;
-        }
-        
-        // Request short-lived URL
-        if (shortLivedMinutes) {
-            checkHashUrl += `&shortLivedMinutes=${shortLivedMinutes}`;
-        }
+        const checkHashUrl = buildFileHandlerUrl(fileHandlerUrl, {
+            hash,
+            checkHash: true,
+            ...(contextId ? { contextId } : {}),
+            ...(shortLivedMinutes ? { shortLivedMinutes } : {})
+        });
         
         const checkResponse = await axios.get(checkHashUrl, {
             timeout: 10000,
@@ -1237,7 +1320,8 @@ async function checkHashExists(hash, fileHandlerUrl, pathwayResolver = null, con
             return {
                 url: url, // shortLivedUrl if available (prefers converted), otherwise regular URL
                 gcs: gcs, // GCS URL (prefers converted, no short-lived version for GCS)
-                hash: data.hash || hash
+                hash: data.hash || hash,
+                filename: data.filename || null // Include filename from response
             };
         }
         
@@ -1268,21 +1352,23 @@ async function checkHashExists(hash, fileHandlerUrl, pathwayResolver = null, con
  * @param {number} shortLivedMinutes - Optional duration for short-lived URL (default: 5)
  * @returns {Promise<Object>} File object with url set to shortLivedUrl (or original if not available)
  */
-async function ensureShortLivedUrl(fileObject, fileHandlerUrl, shortLivedMinutes = 5) {
+async function ensureShortLivedUrl(fileObject, fileHandlerUrl, contextId = null, shortLivedMinutes = 5) {
     if (!fileObject || !fileObject.hash || !fileHandlerUrl) {
         // No hash or no file handler - return original object
         return fileObject;
     }
     
     try {
-        const resolved = await checkHashExists(fileObject.hash, fileHandlerUrl, null, null, shortLivedMinutes);
+        const resolved = await checkHashExists(fileObject.hash, fileHandlerUrl, null, contextId, shortLivedMinutes);
         if (resolved && resolved.url) {
             // Return file object with url replaced by shortLivedUrl (or fallback to regular url)
             // GCS URL comes from checkHash (no short-lived version for GCS)
+            // Preserve filename from original, but use resolved filename if original doesn't have one
             return {
                 ...fileObject,
                 url: resolved.url, // shortLivedUrl (or fallback)
-                gcs: resolved.gcs || fileObject.gcs || null // GCS from checkHash
+                gcs: resolved.gcs || fileObject.gcs || null, // GCS from checkHash
+                filename: fileObject.filename || resolved.filename || fileObject.filename // Preserve original, fallback to resolved
             };
         }
     } catch (error) {
@@ -1294,6 +1380,49 @@ async function ensureShortLivedUrl(fileObject, fileHandlerUrl, shortLivedMinutes
     return fileObject;
 }
 
+/**
+ * Update a file's retention tag via cortex-file-handler (best-effort helper).
+ * cortex-file-handler defaults uploads to retention=temporary; use this to set permanent retention.
+ *
+ * @param {string} hash
+ * @param {'temporary'|'permanent'} retention
+ * @param {string|null} contextId
+ * @param {pathwayResolver|null} pathwayResolver
+ */
+/**
+ * Set file retention (temporary or permanent) via file handler
+ * @param {string} hash - File hash
+ * @param {'temporary'|'permanent'} retention - Retention value
+ * @param {string|null} contextId - Optional context ID for scoped file storage
+ * @param {pathwayResolver|null} pathwayResolver - Optional pathway resolver for logging
+ * @returns {Promise<Object|null>} Response data or null
+ */
+async function setRetentionForHash(hash, retention, contextId = null, pathwayResolver = null) {
+    if (!hash || !retention) return null;
+    const fileHandlerUrl = MEDIA_API_URL;
+    if (!fileHandlerUrl || fileHandlerUrl === 'null') return null;
+
+    const body = {
+        hash,
+        retention,
+        setRetention: true,
+        ...(contextId ? { contextId } : {})
+    };
+
+    try {
+        const res = await axios.post(fileHandlerUrl, body, { timeout: 15000 });
+        return res?.data || null;
+    } catch (error) {
+        const errorMsg = error?.message || String(error);
+        if (pathwayResolver?.logWarning) {
+            pathwayResolver.logWarning(`Failed to set retention=${retention} for hash ${hash}: ${errorMsg}`);
+        } else {
+            logger.warn(`Failed to set retention=${retention} for hash ${hash}: ${errorMsg}`);
+        }
+        return null;
+    }
+}
+
 /**
  * Generic function to upload a file to cloud storage
  * Handles both URLs (downloads then uploads) and base64 data
@@ -1302,9 +1431,10 @@ async function ensureShortLivedUrl(fileObject, fileHandlerUrl, shortLivedMinutes
  * @param {string} mimeType - MIME type of the file (optional for URLs)
  * @param {string} filename - Optional filename (will be inferred if not provided)
  * @param {pathwayResolver} pathwayResolver - Optional pathway resolver for logging
+ * @param {string} contextId - Optional context ID for scoped file storage
  * @returns {Promise<Object>} {url, gcs, hash}
  */
-async function uploadFileToCloud(fileInput, mimeType = null, filename = null, pathwayResolver = null, containerName = null) {
+async function uploadFileToCloud(fileInput, mimeType = null, filename = null, pathwayResolver = null, contextId = null) {
     let tempFilePath = null;
     let tempDir = null;
     let fileBuffer = null;
@@ -1373,8 +1503,8 @@ async function uploadFileToCloud(fileInput, mimeType = null, filename = null, pa
         if (fileBuffer) {
             fileHash = await computeBufferHash(fileBuffer);
             
-            // Check if file already exists using checkHash (with container if specified)
-            const existingFile = await checkHashExists(fileHash, fileHandlerUrl, pathwayResolver, containerName);
+            // Check if file already exists using checkHash (context-scoped when possible)
+            const existingFile = await checkHashExists(fileHash, fileHandlerUrl, pathwayResolver, contextId);
             if (existingFile) {
                 return existingFile;
             }
@@ -1419,14 +1549,16 @@ async function uploadFileToCloud(fileInput, mimeType = null, filename = null, pa
         if (fileHash) {
             formData.append('hash', fileHash);
         }
-        // Add container if specified
-        if (containerName) {
-            formData.append('container', containerName);
+        // container is no longer supported; include contextId (recommended) for scoped hashes
+        // contextId goes in formData body for POST requests, not in URL
+        if (contextId) {
+            formData.append('contextId', contextId);
         }
         
-        // Append requestId parameter
-        const separator = fileHandlerUrl.includes('?') ? '&' : '?';
-        const uploadUrl = `${fileHandlerUrl}${separator}requestId=${requestId}`;
+        // Build upload URL with requestId (contextId goes in formData body, not URL)
+        const uploadUrl = buildFileHandlerUrl(fileHandlerUrl, {
+            requestId
+        });
         
         // Upload file
         const uploadResponse = await axios.post(uploadUrl, formData, {
@@ -1495,8 +1627,8 @@ async function uploadFileToCloud(fileInput, mimeType = null, filename = null, pa
 
 // Helper function to upload base64 image data to cloud storage
 // Now uses the generic uploadFileToCloud function
-const uploadImageToCloud = async (base64Data, mimeType, pathwayResolver = null) => {
-    return await uploadFileToCloud(base64Data, mimeType, null, pathwayResolver);
+const uploadImageToCloud = async (base64Data, mimeType, pathwayResolver = null, contextId = null) => {
+    return await uploadFileToCloud(base64Data, mimeType, null, pathwayResolver, contextId);
 };
 
 /**
@@ -1505,7 +1637,7 @@ const uploadImageToCloud = async (base64Data, mimeType, pathwayResolver = null)
  * @param {Object} config - Configuration object with file service endpoints
  * @returns {Promise<Array<string>>} Array of stringified file content objects
  */
-async function resolveFileHashesToContent(fileHashes, config) {
+async function resolveFileHashesToContent(fileHashes, config, contextId = null) {
     if (!fileHashes || fileHashes.length === 0) return [];
 
     const fileContentPromises = fileHashes.map(async (hash) => {
@@ -1514,42 +1646,20 @@ async function resolveFileHashesToContent(fileHashes, config) {
             const fileHandlerUrl = config?.get?.('whisperMediaApiUrl');
             
             if (fileHandlerUrl && fileHandlerUrl !== 'null') {
-                // Use shared checkHashExists function - it already returns shortLivedUrl in url field
-                const existingFile = await checkHashExists(hash, fileHandlerUrl);
+                // Use shared checkHashExists function - it returns shortLivedUrl, gcs, hash, and filename
+                // This makes a single API call instead of two
+                const existingFile = await checkHashExists(hash, fileHandlerUrl, null, contextId, 5);
                 if (existingFile) {
-                    // checkHashExists already returns shortLivedUrl (prefers converted) in url field
-                    // and GCS URL (prefers converted) in gcs field
-                    // We need filename from the checkHash response, so make a direct call
-                    try {
-                        const separator = fileHandlerUrl.includes('?') ? '&' : '?';
-                        const checkHashUrl = `${fileHandlerUrl}${separator}hash=${hash}&checkHash=true&shortLivedMinutes=5`;
-                        const response = await axios.get(checkHashUrl, {
-                            timeout: 10000,
-                            validateStatus: (status) => status >= 200 && status < 500
-                        });
-                        
-                        if (response.status === 200 && response.data) {
-                            const data = response.data;
-                            return JSON.stringify({
-                                type: "image_url",
-                                url: data.shortLivedUrl || data.converted?.url || data.url,
-                                image_url: { url: data.shortLivedUrl || data.converted?.url || data.url },
-                                gcs: data.converted?.gcs || data.gcs || null, // GCS from checkHash (no short-lived)
-                                originalFilename: data.filename,
-                                hash: hash
-                            });
-                        }
-                    } catch (error) {
-                        // Fallback to existingFile data if direct call fails
-                    }
-                    
-                    // Fallback: use data from checkHashExists
+                    // checkHashExists already returns:
+                    // - shortLivedUrl (prefers converted) in url field
+                    // - GCS URL (prefers converted) in gcs field
+                    // - filename in filename field
                     return JSON.stringify({
                         type: "image_url",
-                        url: existingFile.url, // Already has shortLivedUrl
+                        url: existingFile.url, // Already has shortLivedUrl (prefers converted)
                         image_url: { url: existingFile.url },
-                        gcs: existingFile.gcs || null, // GCS from checkHash
-                        originalFilename: null,
+                        gcs: existingFile.gcs || null, // GCS URL (prefers converted, no short-lived)
+                        originalFilename: existingFile.filename || null, // Filename from single API call
                         hash: hash
                     });
                 }
@@ -1679,6 +1789,7 @@ export {
     deleteFileByHash,
     downloadFile,
     generateUniqueFilename,
+    fetchFileFromUrl,
     getMediaChunks,
     markCompletedForCleanUp,
     extractFileMetadataFromContent,
diff --git a/pathways/system/entity/tools/sys_tool_editfile.js b/pathways/system/entity/tools/sys_tool_editfile.js
index a787153c..721a4c0a 100644
--- a/pathways/system/entity/tools/sys_tool_editfile.js
+++ b/pathways/system/entity/tools/sys_tool_editfile.js
@@ -307,7 +307,8 @@ export default {
                 fileBuffer,
                 mimeType,
                 filename,
-                resolver
+                resolver,
+                contextId
             );
 
             if (!uploadResult || !uploadResult.url) {
@@ -346,7 +347,7 @@ export default {
                 (async () => {
                     try {
                         logger.info(`Deleting old file version with hash ${oldHashToDelete} (background task)`);
-                        await deleteFileByHash(oldHashToDelete, resolver);
+                        await deleteFileByHash(oldHashToDelete, resolver, contextId);
                     } catch (cleanupError) {
                         logger.warn(`Failed to cleanup old file version (hash: ${oldHashToDelete}): ${cleanupError.message}`);
                     }
diff --git a/pathways/system/entity/tools/sys_tool_file_collection.js b/pathways/system/entity/tools/sys_tool_file_collection.js
index 304af65b..8558be67 100644
--- a/pathways/system/entity/tools/sys_tool_file_collection.js
+++ b/pathways/system/entity/tools/sys_tool_file_collection.js
@@ -48,7 +48,7 @@ export default {
                         },
                         permanent: {
                             type: "boolean",
-                            description: "Optional: If true, the file will be stored indefinitely instead of being subject to the default 30 day storage limit. Default: false"
+                            description: "Optional: If true, the file will be stored indefinitely (retention=permanent). Default: false."
                         },
                         userMessage: {
                             type: "string",
@@ -342,7 +342,7 @@ export default {
                 // Use optimistic locking to remove files from collection FIRST
                 // Capture hashes INSIDE the lock to avoid race conditions with concurrent edits
                 const fileIdsToRemove = new Set(filesToRemove.map(f => f.id));
-                const hashesToDelete = [];
+                    const hashesToDelete = [];
                 const finalCollection = await modifyFileCollectionWithLock(contextId, contextKey, (collection) => {
                     // Capture hashes and container info of files that will be removed (at current lock time)
                     collection.forEach(file => {
@@ -350,7 +350,7 @@ export default {
                             hashesToDelete.push({
                                 hash: file.hash,
                                 filename: file.filename || 'unknown',
-                                permanent: file.permanent || false
+                                permanent: file.permanent ?? false
                             });
                         }
                     });
@@ -362,16 +362,18 @@ export default {
                 // Delete files from cloud storage ASYNC (fire and forget, but log errors)
                 // We do this after updating collection so user gets fast response and files are "gone" from UI immediately
                 // Use hashes captured inside the lock to ensure we delete the correct files
+                // IMPORTANT: Don't delete permanent files from cloud storage - they should persist
                 (async () => {
-                    const { config } = await import('../../../../config.js');
-                    const permanentContainerName = process.env.CORTEX_MEDIA_PERMANENT_STORE_NAME;
-                    
                     for (const fileInfo of hashesToDelete) {
+                        // Skip deletion if file is marked as permanent
+                        if (fileInfo.permanent) {
+                            logger.info(`Skipping cloud deletion for permanent file: ${fileInfo.filename} (hash: ${fileInfo.hash})`);
+                            continue;
+                        }
+                        
                         try {
-                            // Determine container based on permanent flag
-                            const container = fileInfo.permanent && permanentContainerName ? permanentContainerName : null;
-                            logger.info(`Deleting file from cloud storage: ${fileInfo.filename} (hash: ${fileInfo.hash}${container ? `, container: ${container}` : ''})`);
-                            await deleteFileByHash(fileInfo.hash, resolver, container);
+                            logger.info(`Deleting file from cloud storage: ${fileInfo.filename} (hash: ${fileInfo.hash})`);
+                            await deleteFileByHash(fileInfo.hash, resolver, contextId);
                         } catch (error) {
                             logger.warn(`Failed to delete file ${fileInfo.filename} (hash: ${fileInfo.hash}) from cloud storage: ${error?.message || String(error)}`);
                         }
diff --git a/pathways/system/entity/tools/sys_tool_image.js b/pathways/system/entity/tools/sys_tool_image.js
index 50f93c13..0622a915 100644
--- a/pathways/system/entity/tools/sys_tool_image.js
+++ b/pathways/system/entity/tools/sys_tool_image.js
@@ -167,7 +167,8 @@ export default {
                                     imageUrl,
                                     mimeType,
                                     null, // filename will be generated
-                                    pathwayResolver
+                                    pathwayResolver,
+                                    args.contextId
                                 );
                                 
                                 const uploadedUrl = uploadResult.url || uploadResult;
@@ -215,7 +216,10 @@ export default {
                                             isModification 
                                                 ? `Modified image from prompt: ${args.detailedInstructions || 'image modification'}`
                                                 : `Generated image from prompt: ${args.detailedInstructions || 'image generation'}`,
-                                            uploadedHash
+                                            uploadedHash,
+                                            null, // fileUrl - not needed since we already uploaded
+                                            pathwayResolver,
+                                            true // permanent => retention=permanent
                                         );
                                     } catch (collectionError) {
                                         // Log but don't fail - file collection is optional
diff --git a/pathways/system/entity/tools/sys_tool_image_gemini.js b/pathways/system/entity/tools/sys_tool_image_gemini.js
index bcb96113..3068a62a 100644
--- a/pathways/system/entity/tools/sys_tool_image_gemini.js
+++ b/pathways/system/entity/tools/sys_tool_image_gemini.js
@@ -140,7 +140,7 @@ export default {
                         if (artifact.type === 'image' && artifact.data && artifact.mimeType) {
                             try {
                                 // Upload image to cloud storage (returns {url, gcs, hash})
-                                const uploadResult = await uploadImageToCloud(artifact.data, artifact.mimeType, pathwayResolver);
+                                const uploadResult = await uploadImageToCloud(artifact.data, artifact.mimeType, pathwayResolver, args.contextId);
                                 
                                 const imageUrl = uploadResult.url || uploadResult;
                                 const imageGcs = uploadResult.gcs || null;
@@ -188,7 +188,10 @@ export default {
                                             isModification 
                                                 ? `Modified image from prompt: ${args.detailedInstructions || 'image modification'}`
                                                 : `Generated image from prompt: ${args.detailedInstructions || 'image generation'}`,
-                                            imageHash
+                                            imageHash,
+                                            null,
+                                            pathwayResolver,
+                                            true // permanent => retention=permanent
                                         );
                                     } catch (collectionError) {
                                         // Log but don't fail - file collection is optional
diff --git a/pathways/system/entity/tools/sys_tool_view_image.js b/pathways/system/entity/tools/sys_tool_view_image.js
index 5d42925a..a4e60ac9 100644
--- a/pathways/system/entity/tools/sys_tool_view_image.js
+++ b/pathways/system/entity/tools/sys_tool_view_image.js
@@ -70,7 +70,7 @@ export default {
 
                 // Resolve to short-lived URL if possible
                 const fileHandlerUrl = config.get('whisperMediaApiUrl');
-                const fileWithShortLivedUrl = await ensureShortLivedUrl(foundFile, fileHandlerUrl);
+                const fileWithShortLivedUrl = await ensureShortLivedUrl(foundFile, fileHandlerUrl, contextId);
 
                 // Add to imageUrls array
                 imageUrls.push({
diff --git a/pathways/system/entity/tools/sys_tool_writefile.js b/pathways/system/entity/tools/sys_tool_writefile.js
index 53b83e37..d6bb3daa 100644
--- a/pathways/system/entity/tools/sys_tool_writefile.js
+++ b/pathways/system/entity/tools/sys_tool_writefile.js
@@ -138,7 +138,8 @@ export default {
                 fileBuffer,
                 mimeType,
                 filename,
-                resolver
+                resolver,
+                contextId
             );
 
             if (!uploadResult || !uploadResult.url) {
@@ -159,7 +160,8 @@ export default {
                         notes,
                         uploadResult.hash || null,
                         null, // fileUrl - not needed since we already uploaded
-                        resolver
+                        resolver,
+                        true // permanent => retention=permanent
                     );
                 } catch (collectionError) {
                     // Log but don't fail - file collection is optional
diff --git a/server/executeWorkspace.js b/server/executeWorkspace.js
index 0f23bdd3..adcad20e 100644
--- a/server/executeWorkspace.js
+++ b/server/executeWorkspace.js
@@ -25,7 +25,7 @@ const resolveAndAddFileContent = async (pathways, pathwayArgs, requestId, config
         if (pathway.fileHashes && pathway.fileHashes.length > 0) {
             try {
                 const { resolveFileHashesToContent } = await import('../lib/fileUtils.js');
-                const fileContent = await resolveFileHashesToContent(pathway.fileHashes, config);
+                const fileContent = await resolveFileHashesToContent(pathway.fileHashes, config, pathwayArgs?.contextId || null);
                 
                 // Add file content to chatHistory if not already present (only do this once)
                 if (!fileContentAdded) {
diff --git a/server/plugins/azureCognitivePlugin.js b/server/plugins/azureCognitivePlugin.js
index e5ef762f..8975c3fd 100644
--- a/server/plugins/azureCognitivePlugin.js
+++ b/server/plugins/azureCognitivePlugin.js
@@ -5,6 +5,7 @@ import { v4 as uuidv4 } from 'uuid';
 import path from 'path';
 import { config } from '../../config.js';
 import { axios } from '../../lib/requestExecutor.js';
+import { fetchFileFromUrl, markCompletedForCleanUp } from '../../lib/fileUtils.js';
 import logger from '../../lib/logger.js';
 import { getSemanticChunks } from '../chunker.js';
 
@@ -199,16 +200,10 @@ class AzureCognitivePlugin extends ModelPlugin {
     }
 
     async markCompletedForCleanUp(requestId) {
-        try {
-            if (API_URL) {
-                //call helper api to mark processing as completed
-                const res = await axios.delete(API_URL, { params: { requestId } });
-                logger.info(`Marked request ${requestId} as completed: ${res.data}`);
-                return res.data;
-            }
-        } catch (err) {
-            logger.error(`Error marking request ${requestId} as completed: ${err}`);
-        }
+        // Use encapsulated function from fileUtils
+        // Note: savedContextId is available in execute() method via cortexRequest.pathwayResolver
+        // For now, pass null as contextId since this is a cleanup operation
+        return await markCompletedForCleanUp(requestId, null);
     }
 
     // Execute the request to the Azure Cognitive API
@@ -228,8 +223,12 @@ class AzureCognitivePlugin extends ModelPlugin {
             const extension = path.extname(file).toLowerCase();
             if (!DIRECT_FILE_EXTENSIONS.includes(extension)) {
                 try {
-                    const { data }  = await axios.get(API_URL, { params: { uri: file, requestId, save: true } });
-                    url = data[0];
+                    // Use encapsulated file handler function with save=true for conversion
+                    // Use savedContextId as contextId if available
+                    const contextId = savedContextId || requestId;
+                    const data = await fetchFileFromUrl(file, requestId, contextId, true);
+                    // Response is an array for converted files
+                    url = Array.isArray(data) ? data[0] : data.url;
                 } catch (error) {
                     logger.error(`Error converting file ${file} to txt: ${error}`);
                     await this.markCompletedForCleanUp(requestId);
diff --git a/server/plugins/azureVideoTranslatePlugin.js b/server/plugins/azureVideoTranslatePlugin.js
index b0db7273..1a77de4b 100644
--- a/server/plugins/azureVideoTranslatePlugin.js
+++ b/server/plugins/azureVideoTranslatePlugin.js
@@ -2,6 +2,7 @@
 import ModelPlugin from "./modelPlugin.js";
 import logger from "../../lib/logger.js";
 import { publishRequestProgress } from "../../lib/redisSubscription.js";
+import { fetchFileFromUrl } from "../../lib/fileUtils.js";
 import crypto from 'crypto';
 import axios from 'axios';
 import {config} from "../../config.js";
@@ -48,7 +49,7 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
         }
     }
 
-    async uploadToFileHandler(videoUrl) {
+    async uploadToFileHandler(videoUrl, contextId = null) {
         try {
             // Get the file handler URL from config
             const fileHandlerUrl = config.get("whisperMediaApiUrl");
@@ -66,19 +67,17 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
             }, 5000);
 
             try {
-                // Start the fetch request
-                const response = await axios.get(fileHandlerUrl, {
-                    params: {
-                        requestId: this.requestId,
-                        fetch: videoUrl
-                    }
-                });
-
-                if (!response.data?.url) {
+                // Use encapsulated file handler function
+                const response = await fetchFileFromUrl(videoUrl, this.requestId, contextId, false);
+                
+                // Response can be an array (for chunked files) or an object with url
+                const resultUrl = Array.isArray(response) ? response[0] : response.url;
+                
+                if (!resultUrl) {
                     throw new Error("File handler did not return a valid URL");
                 }
 
-                return response.data.url;
+                return resultUrl;
             } finally {
                 // Always clear the heartbeat interval
                 clearInterval(heartbeat);
@@ -308,7 +307,9 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
             // If the video is not from Azure storage, upload it to file handler
             if (!videoInfo.isAzureUrl) {
                 logger.debug('Video is not from Azure storage, uploading to file handler...');
-                videoUrl = await this.uploadToFileHandler(videoUrl);
+                // Use savedContextId as contextId for scoped file storage (fallback to requestId if not available)
+                const contextId = cortexRequest.pathwayResolver?.savedContextId || this.requestId;
+                videoUrl = await this.uploadToFileHandler(videoUrl, contextId);
                 logger.debug(`Video uploaded to file handler: ${videoUrl}`);
             }
 
diff --git a/tests/unit/core/shortLivedUrl.test.js b/tests/unit/core/shortLivedUrl.test.js
index 3a1050e8..9bbe9f7c 100644
--- a/tests/unit/core/shortLivedUrl.test.js
+++ b/tests/unit/core/shortLivedUrl.test.js
@@ -249,7 +249,7 @@ test('ensureShortLivedUrl should respect shortLivedMinutes parameter', async t =
 
     const axiosGetStub = t.context.sandbox.replace(axios, 'get', sinon.stub().resolves(mockResponse));
 
-    await ensureShortLivedUrl(fileObject, fileHandlerUrl, shortLivedMinutes);
+    await ensureShortLivedUrl(fileObject, fileHandlerUrl, null, shortLivedMinutes);
 
     // Verify axios was called with correct shortLivedMinutes
     const callArgs = axiosGetStub.getCall(0).args;

From 97c7e4f877498b892009d5eb1638ff6a5679c4ee Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Tue, 16 Dec 2025 12:45:22 -0700
Subject: [PATCH 06/27] feat: enhance blob upload and retention management with
 contextId support

- Updated `uploadBlob` and `uploadFile` functions to accept and extract `contextId` from form fields, improving context-aware file handling.
- Modified `setRetention` to include `contextId` for scoped file storage, ensuring accurate retention management.
- Implemented logic to default uploads to temporary status, aligning with file collection practices.
- Enhanced Redis cleanup logic to skip permanent files, ensuring they are not removed during age-based cleanup.
- Added tests to validate context-aware retention and cleanup behavior, ensuring robust functionality across scenarios.
---
 .../cortex-file-handler/src/blobHandler.js    |  19 +++
 helper-apps/cortex-file-handler/src/index.js  |  13 +-
 helper-apps/cortex-file-handler/src/redis.js  |   6 +-
 .../src/services/storage/StorageService.js    |  26 ++--
 .../cortex-file-handler/tests/cleanup.test.js |  59 ++++++++-
 .../tests/setRetention.test.js                | 117 ++++++++++++++++--
 6 files changed, 215 insertions(+), 25 deletions(-)

diff --git a/helper-apps/cortex-file-handler/src/blobHandler.js b/helper-apps/cortex-file-handler/src/blobHandler.js
index 78384145..6f62c1bb 100644
--- a/helper-apps/cortex-file-handler/src/blobHandler.js
+++ b/helper-apps/cortex-file-handler/src/blobHandler.js
@@ -273,6 +273,7 @@ function uploadBlob(
               uploadName, // Use the LLM-friendly filename
               resolve,
               hash,
+              fields, // Pass fields for contextId extraction
             );
             resolve(result);
           } catch (error) {
@@ -489,6 +490,15 @@ function uploadBlob(
                 }, {}),
               };
               if (hash) result.hash = hash;
+              
+              // Extract contextId from form fields if present
+              if (fields && fields.contextId) {
+                result.contextId = fields.contextId;
+              }
+              
+              // All uploads default to temporary (permanent: false) to match file collection logic
+              result.permanent = false;
+              
               // Container parameter is ignored - always uses default container from env var
               
               // Ensure shortLivedUrl is always present
@@ -710,6 +720,7 @@ async function uploadFile(
   filename,
   resolve,
   hash = null,
+  fields = null, // Optional fields from form data (for contextId)
 ) {
   try {
     if (!file) {
@@ -819,6 +830,14 @@ async function uploadFile(
       result.hash = hash;
     }
     
+    // Extract contextId from form fields if present (only available for multipart uploads)
+    if (fields && fields.contextId) {
+      result.contextId = fields.contextId;
+    }
+    
+    // All uploads default to temporary (permanent: false) to match file collection logic
+    result.permanent = false;
+    
     // Container parameter is ignored - always uses default container from env var
     
     // Ensure shortLivedUrl is always present
diff --git a/helper-apps/cortex-file-handler/src/index.js b/helper-apps/cortex-file-handler/src/index.js
index d37890fd..4c73e941 100644
--- a/helper-apps/cortex-file-handler/src/index.js
+++ b/helper-apps/cortex-file-handler/src/index.js
@@ -253,7 +253,7 @@ async function CortexFileHandler(context, req) {
     }
 
     try {
-      const result = await storageService.setRetention(fileHash, retention, context);
+      const result = await storageService.setRetention(fileHash, retention, context, resolvedContextId);
       context.res = {
         status: 200,
         body: result,
@@ -310,6 +310,9 @@ async function CortexFileHandler(context, req) {
       // Container parameter is ignored - always uses default container from env var
       const res = await storageService.uploadFile(context, filename, '', null, null);
 
+      // All uploads default to temporary (permanent: false) to match file collection logic
+      res.permanent = false;
+
       //Update Redis (using hash or URL as the key)
       // Container parameter is ignored - always uses default container from env var
       await setFileStoreMap(cacheKey, res);
@@ -634,10 +637,12 @@ async function CortexFileHandler(context, req) {
     // Container parameter is ignored - always uses default container from env var
     const result = await uploadBlob(context, req, saveToLocal, null, hash);
     if (result?.hash && context?.res?.body) {
-      const hashKey = getScopedHashKey(result.hash, resolvedContextId);
+      // Use contextId from result (extracted from form fields) or from resolvedContextId (query/body)
+      const uploadContextId = result.contextId || resolvedContextId;
+      const hashKey = getScopedHashKey(result.hash, uploadContextId);
       // Store contextId alongside the entry for debugging/traceability
-      if (resolvedContextId && typeof context.res.body === "object" && context.res.body) {
-        context.res.body.contextId = resolvedContextId;
+      if (uploadContextId && typeof context.res.body === "object" && context.res.body) {
+        context.res.body.contextId = uploadContextId;
       }
       await setFileStoreMap(hashKey, context.res.body);
     }
diff --git a/helper-apps/cortex-file-handler/src/redis.js b/helper-apps/cortex-file-handler/src/redis.js
index d605de2a..0515ab77 100644
--- a/helper-apps/cortex-file-handler/src/redis.js
+++ b/helper-apps/cortex-file-handler/src/redis.js
@@ -418,8 +418,12 @@ const cleanupRedisFileStoreMapAge = async (
     const maxAgeAgo = new Date(Date.now() - maxAgeDays * 24 * 60 * 60 * 1000);
 
     // Convert to array and sort by timestamp (oldest first)
+    // Skip permanent files - they should never be cleaned up by age
     const entries = Object.entries(map)
-      .filter(([_, value]) => value?.timestamp) // Only entries with timestamps
+      .filter(([_, value]) => {
+        // Only entries with timestamps and not permanent (matches file collection logic)
+        return value?.timestamp && value?.permanent !== true;
+      })
       .sort(([_, a], [__, b]) => {
         const timeA = new Date(a.timestamp).getTime();
         const timeB = new Date(b.timestamp).getTime();
diff --git a/helper-apps/cortex-file-handler/src/services/storage/StorageService.js b/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
index 4a35ac4a..ea4f728a 100644
--- a/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
+++ b/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
@@ -266,9 +266,10 @@ export class StorageService {
    * @param {string} hash - The hash of the file
    * @param {string} retention - The retention value ('temporary' or 'permanent')
    * @param {Object} context - Context object for logging
+   * @param {string|null} contextId - Optional context ID for scoped file storage
    * @returns {Promise<Object>} Object containing updated file info
    */
-  async setRetention(hash, retention, context = {}) {
+  async setRetention(hash, retention, context = {}, contextId = null) {
     await this._initialize();
     
     if (!hash) {
@@ -280,12 +281,12 @@ export class StorageService {
     }
 
     // Get Redis functions
-    const { getFileStoreMap, setFileStoreMap } = await import("../../redis.js");
-    const { getDefaultContainerName } = await import("../../constants.js");
+    const { getFileStoreMap, setFileStoreMap, getScopedHashKey } = await import("../../redis.js");
     
-    // Look up file by hash
-    const container = getDefaultContainerName();
-    const hashResult = await getFileStoreMap(hash);
+    // Look up file by hash using context-scoped key if contextId provided
+    // getFileStoreMap already handles fallback to unscoped keys
+    const scopedKey = getScopedHashKey(hash, contextId);
+    const hashResult = await getFileStoreMap(scopedKey);
     
     if (!hashResult) {
       throw new Error(`File with hash ${hash} not found`);
@@ -298,8 +299,8 @@ export class StorageService {
       throw new Error(`File with hash ${hash} has no valid URL`);
     }
 
-    // Get the Azure provider
-    const provider = await this.factory.getAzureProvider(container);
+    // Always use primary provider - single container only
+    const provider = this.primaryProvider;
     
     // Extract blob name from URL
     const blobName = provider.extractBlobNameFromUrl(hashResult.url);
@@ -344,12 +345,15 @@ export class StorageService {
       }
     }
 
-    // Update Redis with new information (including shortLivedUrl)
+    // Update Redis with new information (including shortLivedUrl and permanent flag)
+    // Use the same scoped key that was used for lookup
+    // Store as permanent boolean to match file collection logic
     const newFileInfo = {
       ...hashResult,
       url: hashResult.url, // URL stays the same - same blob, just different tag
       shortLivedUrl: shortLivedUrl,
       gcs: hashResult.gcs,
+      permanent: retention === 'permanent', // Store as boolean to match file collection logic
       timestamp: new Date().toISOString()
     };
     
@@ -357,8 +361,8 @@ export class StorageService {
       newFileInfo.converted = convertedResult;
     }
 
-    await setFileStoreMap(hash, newFileInfo);
-    context.log?.(`Updated Redis map for hash: ${hash}`);
+    await setFileStoreMap(scopedKey, newFileInfo);
+    context.log?.(`Updated Redis map for hash: ${hash}${contextId ? ` (contextId: ${contextId})` : ""}`);
 
     return {
       hash,
diff --git a/helper-apps/cortex-file-handler/tests/cleanup.test.js b/helper-apps/cortex-file-handler/tests/cleanup.test.js
index 5f3abb90..7d8b8358 100644
--- a/helper-apps/cortex-file-handler/tests/cleanup.test.js
+++ b/helper-apps/cortex-file-handler/tests/cleanup.test.js
@@ -296,11 +296,12 @@ test("age-based cleanup should remove old entries", async (t) => {
     const context = { log: console.log };
     const uploadResult = await uploadBlob(context, null, true, testFile); // Use local storage
 
-    // Store the hash in Redis with an old timestamp
+    // Store the hash in Redis with an old timestamp (temporary file)
     const hash = "test-old-entry";
     const oldEntry = {
       ...uploadResult,
       timestamp: createOldTimestamp(8), // 8 days old
+      permanent: false, // Temporary file should be cleaned up
     };
     console.log(`Storing old entry with timestamp: ${oldEntry.timestamp}`);
     await setFileStoreMap(hash, oldEntry);
@@ -376,6 +377,59 @@ test("age-based cleanup should keep recent entries", async (t) => {
   }
 });
 
+test("age-based cleanup should not remove permanent files", async (t) => {
+  // Create a test file and upload it
+  const testFile = await createTestFile("Test content for permanent file test");
+
+  try {
+    const context = { log: console.log };
+    const uploadResult = await uploadBlob(
+      context,
+      null,
+      shouldUseLocalStorage(),
+      testFile,
+    ); // Use appropriate storage
+
+    // Store the hash in Redis with an old timestamp but permanent=true
+    const hash = "test-permanent-entry";
+    const permanentEntry = {
+      ...uploadResult,
+      timestamp: createOldTimestamp(8), // 8 days old
+      permanent: true, // Mark as permanent
+    };
+    console.log(`Storing permanent entry with timestamp: ${permanentEntry.timestamp}`);
+    await setFileStoreMap(hash, permanentEntry);
+
+    // Verify it exists initially (with skipLazyCleanup to avoid interference)
+    const initialResult = await getFileStoreMap(hash, true);
+    t.truthy(initialResult, "Permanent entry should exist initially");
+    t.is(initialResult.permanent, true, "Entry should be marked as permanent");
+
+    // Run age-based cleanup with 7-day threshold
+    const cleaned = await cleanupRedisFileStoreMapAge(7, 10);
+    console.log(
+      `Age cleanup returned ${cleaned.length} entries:`,
+      cleaned.map((c) => c.hash),
+    );
+
+    // Verify the permanent entry was NOT cleaned up
+    const cleanedHash = cleaned.find(
+      (entry) => entry.hash === "test-permanent-entry",
+    );
+    t.falsy(cleanedHash, "Permanent entry should NOT be in cleaned list");
+
+    // Verify the entry still exists in cache
+    const resultAfterCleanup = await getFileStoreMap(hash, true);
+    t.truthy(resultAfterCleanup, "Permanent entry should still exist in cache");
+    t.is(resultAfterCleanup.permanent, true, "Entry should still be marked as permanent");
+
+    // Clean up
+    await removeFromFileStoreMap("test-permanent-entry");
+  } finally {
+    cleanupTestFile(testFile);
+  }
+});
+
 test("age-based cleanup should respect maxEntriesToCheck limit", async (t) => {
   // Create multiple test files and upload them
   const testFiles = [];
@@ -397,11 +451,12 @@ test("age-based cleanup should respect maxEntriesToCheck limit", async (t) => {
         testFile,
       ); // Use appropriate storage
 
-      // Store with old timestamp
+      // Store with old timestamp (temporary files)
       const hash = `test-old-entry-${i}`;
       const oldEntry = {
         ...uploadResult,
         timestamp: createOldTimestamp(8), // 8 days old
+        permanent: false, // Temporary files should be cleaned up
       };
       oldEntries.push(oldEntry);
       await setFileStoreMap(hash, oldEntry);
diff --git a/helper-apps/cortex-file-handler/tests/setRetention.test.js b/helper-apps/cortex-file-handler/tests/setRetention.test.js
index 756566c6..6736672b 100644
--- a/helper-apps/cortex-file-handler/tests/setRetention.test.js
+++ b/helper-apps/cortex-file-handler/tests/setRetention.test.js
@@ -38,11 +38,12 @@ async function createTestFile(content, extension) {
 }
 
 // Helper function to upload file with hash and container
-async function uploadFile(filePath, hash = null, containerName = null) {
+async function uploadFile(filePath, hash = null, containerName = null, contextId = null) {
   const form = new FormData();
   form.append("file", fs.createReadStream(filePath));
   if (hash) form.append("hash", hash);
   if (containerName) form.append("container", containerName);
+  if (contextId) form.append("contextId", contextId);
 
   return await axios.post(baseUrl, form, {
     headers: form.getHeaders(),
@@ -52,11 +53,14 @@ async function uploadFile(filePath, hash = null, containerName = null) {
 }
 
 // Helper function to check if hash exists
-async function checkHashExists(hash, containerName = null) {
+async function checkHashExists(hash, containerName = null, contextId = null) {
   const params = { hash, checkHash: true };
   if (containerName) {
     params.container = containerName;
   }
+  if (contextId) {
+    params.contextId = contextId;
+  }
   return await axios.get(baseUrl, {
     params,
     validateStatus: (status) => true,
@@ -65,17 +69,20 @@ async function checkHashExists(hash, containerName = null) {
 }
 
 // Helper function to set retention
-async function setRetention(hash, retention, useBody = false) {
+async function setRetention(hash, retention, useBody = false, contextId = null) {
+  const bodyOrParams = { hash, retention, setRetention: true };
+  if (contextId) {
+    bodyOrParams.contextId = contextId;
+  }
+  
   if (useBody) {
-    const body = { hash, retention, setRetention: true };
-    return await axios.post(baseUrl, body, {
+    return await axios.post(baseUrl, bodyOrParams, {
       validateStatus: (status) => true,
       timeout: 30000,
     });
   } else {
-    const params = { hash, retention, setRetention: true };
     return await axios.post(baseUrl, null, {
-      params,
+      params: bodyOrParams,
       validateStatus: (status) => true,
       timeout: 30000,
     });
@@ -298,6 +305,7 @@ test.serial("should update Redis map with retention information", async (t) => {
     const scopedHash = getScopedHashKey(testHash);
     const oldEntry = await getFileStoreMap(scopedHash);
     t.truthy(oldEntry, "Redis entry should exist before setting retention");
+    t.is(oldEntry.permanent, false, "New uploads should have permanent=false by default");
 
     // Set retention
     const retentionResponse = await setRetention(testHash, "permanent");
@@ -311,6 +319,7 @@ test.serial("should update Redis map with retention information", async (t) => {
     t.truthy(newEntry, "Redis entry should still exist after setting retention");
     t.is(newEntry.url, retentionResponse.data.url, "Entry should have correct URL");
     t.truthy(newEntry.shortLivedUrl, "Entry should have shortLivedUrl");
+    t.is(newEntry.permanent, true, "Entry should have permanent=true in Redis (matches file collection logic)");
 
   } finally {
     fs.unlinkSync(filePath);
@@ -521,3 +530,97 @@ test.serial("should always include shortLivedUrl in response", async (t) => {
     }
   }
 });
+
+test.serial("should set retention for context-scoped file", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const testContent = "test content for context-scoped retention";
+  const testHash = `test-retention-context-${uuidv4()}`;
+  const contextId = `test-context-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file with contextId
+    uploadResponse = await uploadFile(filePath, testHash, null, contextId);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+    t.is(uploadResponse.data.contextId, contextId, "Should have contextId in response");
+
+    // Wait for Redis to update
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Set retention with the same contextId
+    const retentionResponse = await setRetention(testHash, "permanent", false, contextId);
+    t.is(retentionResponse.status, 200, "Set retention should succeed");
+    t.is(retentionResponse.data.retention, "permanent", "Should have retention set to permanent");
+    t.truthy(retentionResponse.data.shortLivedUrl, "Should have shortLivedUrl");
+
+    // Verify Redis entry was updated with context-scoped key
+    const { getScopedHashKey } = await import("../src/redis.js");
+    const scopedKey = getScopedHashKey(testHash, contextId);
+    const updatedEntry = await getFileStoreMap(scopedKey);
+    t.truthy(updatedEntry, "Should have updated entry in Redis");
+    t.truthy(updatedEntry.shortLivedUrl, "Should have shortLivedUrl in Redis entry");
+    t.is(updatedEntry.permanent, true, "Entry should have permanent=true in Redis (matches file collection logic)");
+
+    // Wait for operations to complete
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Verify file still exists with contextId
+    const checkResponse = await checkHashExists(testHash, null, contextId);
+    t.is(checkResponse.status, 200, "File should still exist after setting retention");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    // Cleanup
+    try {
+      const { getScopedHashKey } = await import("../src/redis.js");
+      const scopedKey = getScopedHashKey(testHash, contextId);
+      await removeFromFileStoreMap(scopedKey);
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should return 404 when contextId doesn't match for setRetention", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const testContent = "test content for context mismatch";
+  const testHash = `test-retention-mismatch-${uuidv4()}`;
+  const contextId1 = `test-context-1-${uuidv4()}`;
+  const contextId2 = `test-context-2-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file with contextId1
+    uploadResponse = await uploadFile(filePath, testHash, null, contextId1);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+
+    // Wait for Redis to update
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    // Try to set retention with different contextId - should fail
+    const retentionResponse = await setRetention(testHash, "permanent", false, contextId2);
+    t.is(retentionResponse.status, 404, "Should return 404 when contextId doesn't match");
+    t.truthy(retentionResponse.data.includes("not found"), "Error message should indicate file not found");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    // Cleanup
+    try {
+      const { getScopedHashKey } = await import("../src/redis.js");
+      const scopedKey = getScopedHashKey(testHash, contextId1);
+      await removeFromFileStoreMap(scopedKey);
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});

From fd63d914f4e051b070f71addb701d28952a91e7b Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Wed, 17 Dec 2025 22:34:12 -0700
Subject: [PATCH 07/27] feat: enhance file handling and GraphQL integration
 with context-aware features

- Introduced contextId support across various file handling functions, improving scoped file management.
- Updated GraphQL resolvers to differentiate between queries and mutations, enhancing the structure and clarity of API interactions.
- Implemented new pathways for reading and updating file metadata, ensuring backward compatibility while streamlining operations.
- Enhanced file collection management by integrating displayFilename persistence, improving user experience during file retrieval.
- Refactored existing tests to validate new context-aware functionalities and ensure robust error handling across scenarios.
---
 config.js                                     | 133 ++++
 .../cortex-file-handler/src/blobHandler.js    |   7 +-
 helper-apps/cortex-file-handler/src/index.js  |  61 +-
 helper-apps/cortex-file-handler/src/redis.js  | 120 ++-
 .../src/services/storage/StorageService.js    |  25 +-
 .../src/utils/logSecurity.js                  |  20 +
 .../tests/fileUpload.test.js                  | 101 +++
 .../tests/setRetention.test.js                |  41 +-
 lib/fileUtils.js                              | 684 ++++++++++--------
 .../entity/files/sys_read_file_collection.js  |  43 ++
 .../entity/files/sys_update_file_metadata.js  |  72 ++
 .../system/entity/memory/sys_read_memory.js   |  17 +-
 .../system/entity/memory/sys_save_memory.js   |  22 +-
 .../system/entity/tools/sys_tool_editfile.js  |  88 ++-
 .../entity/tools/sys_tool_file_collection.js  | 151 ++--
 .../entity/tools/sys_tool_view_image.js       |   1 -
 .../rest_streaming/sys_claude_37_sonnet.js    |  21 -
 .../rest_streaming/sys_claude_41_opus.js      |  21 -
 .../rest_streaming/sys_claude_4_sonnet.js     |  21 -
 .../sys_google_gemini_25_flash.js             |  25 -
 .../sys_google_gemini_25_pro.js               |  25 -
 pathways/system/rest_streaming/sys_grok_4.js  |  23 -
 .../sys_grok_4_fast_non_reasoning.js          |  23 -
 .../sys_grok_4_fast_reasoning.js              |  23 -
 .../system/rest_streaming/sys_ollama_chat.js  |  21 -
 .../rest_streaming/sys_ollama_completion.js   |  14 -
 .../system/rest_streaming/sys_openai_chat.js  |  22 -
 .../rest_streaming/sys_openai_chat_gpt41.js   |  22 -
 .../sys_openai_chat_gpt41_mini.js             |  21 -
 .../sys_openai_chat_gpt41_nano.js             |  21 -
 .../sys_openai_chat_gpt4_omni.js              |  21 -
 .../sys_openai_chat_gpt4_omni_mini.js         |  21 -
 .../rest_streaming/sys_openai_chat_gpt5.js    |  21 -
 .../sys_openai_chat_gpt5_chat.js              |  21 -
 .../sys_openai_chat_gpt5_mini.js              |  21 -
 .../sys_openai_chat_gpt5_nano.js              |  21 -
 .../rest_streaming/sys_openai_chat_o3.js      |  22 -
 .../rest_streaming/sys_openai_chat_o3_mini.js |  22 -
 .../rest_streaming/sys_openai_completion.js   |   9 -
 server/graphql.js                             |  17 +-
 server/typeDef.js                             |  21 +-
 .../features/tools/fileCollection.test.js     | 184 ++---
 .../features/tools/writefile.test.js          |  59 +-
 tests/unit/core/fileCollection.test.js        |  31 +-
 44 files changed, 1225 insertions(+), 1155 deletions(-)
 create mode 100644 pathways/system/entity/files/sys_read_file_collection.js
 create mode 100644 pathways/system/entity/files/sys_update_file_metadata.js
 delete mode 100644 pathways/system/rest_streaming/sys_claude_37_sonnet.js
 delete mode 100644 pathways/system/rest_streaming/sys_claude_41_opus.js
 delete mode 100644 pathways/system/rest_streaming/sys_claude_4_sonnet.js
 delete mode 100644 pathways/system/rest_streaming/sys_google_gemini_25_flash.js
 delete mode 100644 pathways/system/rest_streaming/sys_google_gemini_25_pro.js
 delete mode 100644 pathways/system/rest_streaming/sys_grok_4.js
 delete mode 100644 pathways/system/rest_streaming/sys_grok_4_fast_non_reasoning.js
 delete mode 100644 pathways/system/rest_streaming/sys_grok_4_fast_reasoning.js
 delete mode 100644 pathways/system/rest_streaming/sys_ollama_chat.js
 delete mode 100644 pathways/system/rest_streaming/sys_ollama_completion.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_chat.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_chat_gpt41.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_chat_gpt41_mini.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_chat_gpt41_nano.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_chat_gpt4_omni.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_chat_gpt4_omni_mini.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_chat_gpt5.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_chat_gpt5_chat.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_chat_gpt5_mini.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_chat_gpt5_nano.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_chat_o3.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_chat_o3_mini.js
 delete mode 100644 pathways/system/rest_streaming/sys_openai_completion.js

diff --git a/config.js b/config.js
index ff8a88a0..24c362a6 100644
--- a/config.js
+++ b/config.js
@@ -9,6 +9,7 @@ import logger from './lib/logger.js';
 import PathwayManager from './lib/pathwayManager.js';
 import { readdir } from 'fs/promises';
 import { entityConstants } from './lib/entityConstants.js';
+import { Prompt } from './server/prompt.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -140,6 +141,7 @@ var config = convict({
         default: {
             "oai-gpturbo": {
                 "type": "OPENAI-CHAT",
+                "emulateOpenAICompletionModel": "*",
                 "url": "https://api.openai.com/v1/chat/completions",
                 "headers": {
                     "Authorization": "Bearer {{OPENAI_API_KEY}}",
@@ -207,6 +209,7 @@ var config = convict({
             },
             "oai-gpt5": {
                 "type": "OPENAI-REASONING-VISION",
+                "emulateOpenAIChatModel": "gpt-5",
                 "url": "https://api.openai.com/v1/chat/completions",
                 "headers": {
                     "Authorization": "Bearer {{OPENAI_API_KEY}}",
@@ -237,6 +240,7 @@ var config = convict({
             },
             "oai-gpt4o": {
                 "type": "OPENAI-VISION",
+                "emulateOpenAIChatModel": "gpt-4o",
                 "url": "https://api.openai.com/v1/chat/completions",
                 "headers": {
                     "Authorization": "Bearer {{OPENAI_API_KEY}}",
@@ -267,6 +271,7 @@ var config = convict({
             },
             "oai-gpt41": {
                 "type": "OPENAI-VISION",
+                "emulateOpenAIChatModel": "gpt-4.1",
                 "url": "https://api.openai.com/v1/chat/completions",
                 "headers": {
                     "Authorization": "Bearer {{OPENAI_API_KEY}}",
@@ -327,6 +332,10 @@ var config = convict({
             },
             "oai-o3-mini": {
                 "type": "OPENAI-REASONING",
+                "emulateOpenAIChatModel": "o3-mini",
+                "restStreaming": {
+                    "enableDuplicateRequests": false
+                },
                 "url": "https://api.openai.com/v1/chat/completions",
                 "headers": {
                     "Authorization": "Bearer {{OPENAI_API_KEY}}",
@@ -464,6 +473,10 @@ var config = convict({
             },
             "ollama-chat": {
                 "type": "OLLAMA-CHAT",
+                "emulateOpenAIChatModel": "ollama-chat",
+                "restStreaming": {
+                    "timeout": 300
+                },
                 "url": "{{ollamaUrl}}/api/chat",
                 "headers": {
                   "Content-Type": "application/json"
@@ -474,6 +487,10 @@ var config = convict({
             },
             "ollama-completion": {
                 "type": "OLLAMA-COMPLETION",
+                "emulateOpenAICompletionModel": "ollama-completion",
+                "restStreaming": {
+                    "timeout": 300
+                },
                 "url": "{{ollamaUrl}}/api/generate",
                 "headers": {
                   "Content-Type": "application/json"
@@ -507,6 +524,7 @@ var config = convict({
             },
             "claude-37-sonnet-vertex": {
                 "type": "CLAUDE-3-VERTEX",
+                "emulateOpenAIChatModel": "claude-3.7-sonnet",
                 "url": "{{claudeVertexUrl}}",
                 "headers": {
                     "Content-Type": "application/json"
@@ -519,6 +537,7 @@ var config = convict({
             },
             "claude-4-sonnet-vertex": {
                 "type": "CLAUDE-4-VERTEX",
+                "emulateOpenAIChatModel": "claude-4-sonnet",
                 "url": "{{claudeVertexUrl}}",
                 "headers": {
                     "Content-Type": "application/json"
@@ -531,6 +550,15 @@ var config = convict({
             },
             "gemini-flash-25-vision": {
                 "type": "GEMINI-1.5-VISION",
+                "emulateOpenAIChatModel": "gemini-flash-25",
+                "restStreaming": {
+                    "geminiSafetySettings": [
+                        {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_ONLY_HIGH"},
+                        {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_ONLY_HIGH"},
+                        {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_ONLY_HIGH"},
+                        {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_ONLY_HIGH"}
+                    ]
+                },
                 "url": "{{geminiFlashUrl}}",
                 "headers": {
                     "Content-Type": "application/json"
@@ -557,6 +585,13 @@ var config = convict({
             },
             "xai-grok-4": {
                 "type": "GROK-VISION",
+                "emulateOpenAIChatModel": "grok-4",
+                "restStreaming": {
+                    "inputParameters": {
+                        "stream": false,
+                        "search_parameters": ""
+                    }
+                },
                 "url": "https://api.x.ai/v1/chat/completions",
                 "headers": {
                     "Authorization": "Bearer {{XAI_API_KEY}}",
@@ -587,6 +622,13 @@ var config = convict({
             },
             "xai-grok-4-fast-reasoning": {
                 "type": "GROK-VISION",
+                "emulateOpenAIChatModel": "grok-4-fast-reasoning",
+                "restStreaming": {
+                    "inputParameters": {
+                        "stream": false,
+                        "search_parameters": ""
+                    }
+                },
                 "url": "https://api.x.ai/v1/chat/completions",
                 "headers": {
                     "Authorization": "Bearer {{XAI_API_KEY}}",
@@ -602,6 +644,13 @@ var config = convict({
             },
             "xai-grok-4-fast-non-reasoning": {
                 "type": "GROK-VISION",
+                "emulateOpenAIChatModel": "grok-4-fast-non-reasoning",
+                "restStreaming": {
+                    "inputParameters": {
+                        "stream": false,
+                        "search_parameters": ""
+                    }
+                },
                 "url": "https://api.x.ai/v1/chat/completions",
                 "headers": {
                     "Authorization": "Bearer {{XAI_API_KEY}}",
@@ -942,6 +991,90 @@ const buildPathways = async (config) => {
         process.exit(1);
     }
 
+    // Generate REST streaming pathways from model configs
+    const generateRestStreamingPathways = (models) => {
+        const restPathways = {};
+        
+        for (const [modelName, modelConfig] of Object.entries(models || {})) {
+            if (!modelConfig) continue;
+            
+            // Check for chat model emulation
+            if (modelConfig.emulateOpenAIChatModel) {
+                const pathwayName = `sys_rest_streaming_${modelName.replace(/-/g, '_')}`;
+                const restConfig = modelConfig.restStreaming || {};
+                
+                // Default input parameters for chat models
+                // Special case: oai-gpt4o (default) uses empty array, others use object array
+                const defaultInputParams = modelName === 'oai-gpt4o' 
+                    ? {
+                        messages: [],
+                        tools: '',
+                        tool_choice: 'auto',
+                        functions: ''
+                    }
+                    : {
+                        messages: [{role: '', content: []}],
+                        tools: '',
+                        tool_choice: 'auto'
+                    };
+                
+                // Merge with any custom input parameters
+                const inputParameters = restConfig.inputParameters 
+                    ? { ...defaultInputParams, ...restConfig.inputParameters }
+                    : defaultInputParams;
+                
+                // Special handling for certain models
+                if (modelName.startsWith('oai-') && !modelName.includes('gpturbo') && modelName !== 'oai-gpt4o') {
+                    inputParameters.functions = '';
+                }
+                
+                restPathways[pathwayName] = {
+                    prompt: [
+                        new Prompt({ messages: ["{{messages}}"] })
+                    ],
+                    inputParameters,
+                    model: modelName,
+                    useInputChunking: false,
+                    emulateOpenAIChatModel: modelConfig.emulateOpenAIChatModel,
+                    ...(restConfig.geminiSafetySettings && { geminiSafetySettings: restConfig.geminiSafetySettings }),
+                    ...(restConfig.enableDuplicateRequests !== undefined && { enableDuplicateRequests: restConfig.enableDuplicateRequests }),
+                    ...(restConfig.timeout && { timeout: restConfig.timeout })
+                };
+            }
+            
+            // Check for completion model emulation
+            if (modelConfig.emulateOpenAICompletionModel) {
+                const pathwayName = `sys_rest_streaming_${modelName.replace(/-/g, '_')}_completion`;
+                const restConfig = modelConfig.restStreaming || {};
+                
+                restPathways[pathwayName] = {
+                    prompt: `{{text}}`,
+                    inputParameters: restConfig.inputParameters || {
+                        text: '',
+                        ...(modelName.includes('ollama') && { ollamaModel: '' })
+                    },
+                    model: modelName,
+                    useInputChunking: false,
+                    emulateOpenAICompletionModel: modelConfig.emulateOpenAICompletionModel,
+                    ...(restConfig.timeout && { timeout: restConfig.timeout })
+                };
+            }
+        }
+        
+        return restPathways;
+    };
+    
+    // Generate REST streaming pathways from models
+    const models = config.get('models');
+    const generatedRestPathways = models ? generateRestStreamingPathways(models) : {};
+    
+    if (Object.keys(generatedRestPathways).length > 0) {
+        logger.info(`Generated ${Object.keys(generatedRestPathways).length} REST streaming pathways from model configs`);
+    }
+    
+    // Merge generated pathways into loaded pathways (they can be overridden by file-based pathways)
+    Object.assign(loadedPathways, generatedRestPathways);
+    
     // This is where we integrate pathway overrides from the config
     // file. This can run into a partial definition issue if the
     // config file contains pathways that no longer exist.
diff --git a/helper-apps/cortex-file-handler/src/blobHandler.js b/helper-apps/cortex-file-handler/src/blobHandler.js
index 6f62c1bb..6be2dc36 100644
--- a/helper-apps/cortex-file-handler/src/blobHandler.js
+++ b/helper-apps/cortex-file-handler/src/blobHandler.js
@@ -338,8 +338,8 @@ function uploadBlob(
             }
 
             // Prepare for streaming to cloud destinations
-            const filename = info.filename;
-            const fileExtension = path.extname(filename);
+            const displayFilename = info.filename; // Preserve original filename for metadata
+            const fileExtension = path.extname(displayFilename);
             const shortId = generateShortId();
             const uploadName = `${shortId}${fileExtension}`;
             // Extract content-type from busboy info (preserves charset if provided)
@@ -479,6 +479,7 @@ function uploadBlob(
               const result = {
                 message: `File '${uploadName}' uploaded successfully.`,
                 filename: uploadName,
+                displayFilename: displayFilename, // Store original filename in metadata
                 ...results.reduce((acc, item) => {
                   if (item.type === "primary") {
                     acc.url = item.result.url || item.result;
@@ -813,6 +814,8 @@ async function uploadFile(
     // Wait for original uploads to complete
     context.log("Waiting for all storage uploads to complete...");
     const results = await Promise.all(storagePromises);
+    // Note: filename parameter here is the uploadName (generated short ID), not the original
+    // For local file path uploads, we don't have the original filename, so originalFilename will be undefined
     const result = {
       message: `File '${uploadName}' ${saveToLocal ? "saved to folder" : "uploaded"} successfully.`,
       filename: uploadName,
diff --git a/helper-apps/cortex-file-handler/src/index.js b/helper-apps/cortex-file-handler/src/index.js
index 4c73e941..7b5cb4a1 100644
--- a/helper-apps/cortex-file-handler/src/index.js
+++ b/helper-apps/cortex-file-handler/src/index.js
@@ -10,7 +10,6 @@ import { ensureEncoded, ensureFileExtension, urlExists } from "./helper.js";
 import {
   cleanupRedisFileStoreMap,
   getFileStoreMap,
-  getScopedHashKey,
   publishRequestProgress,
   removeFromFileStoreMap,
   setFileStoreMap,
@@ -20,6 +19,7 @@ import { FileConversionService } from "./services/FileConversionService.js";
 import { StorageService } from "./services/storage/StorageService.js";
 import { uploadBlob } from "./blobHandler.js";
 import { generateShortId } from "./utils/filenameUtils.js";
+import { redactContextId } from "./utils/logSecurity.js";
 
 // Hybrid cleanup approach:
 // 1. Lazy cleanup: Check file existence when cache entries are accessed (in getFileStoreMap)
@@ -118,7 +118,7 @@ async function CortexFileHandler(context, req) {
                 : "upload";
 
   context.log(
-    `Processing ${req.method} request - ${requestId ? `requestId: ${requestId}, ` : ""}${uri ? `uri: ${uri}, ` : ""}${hash ? `hash: ${hash}, ` : ""}${resolvedContextId ? `contextId: ${resolvedContextId}, ` : ""}operation: ${operation}`,
+    `Processing ${req.method} request - ${requestId ? `requestId: ${requestId}, ` : ""}${uri ? `uri: ${uri}, ` : ""}${hash ? `hash: ${hash}, ` : ""}${resolvedContextId ? `contextId: ${redactContextId(resolvedContextId)}, ` : ""}operation: ${operation}`,
   );
 
   // Trigger lightweight age-based cleanup (runs every 100 requests)
@@ -176,7 +176,7 @@ async function CortexFileHandler(context, req) {
     if (deleteHash && !deleteRequestId) {
       try {
         // Container parameter is ignored - always uses default container from env var
-        const deleted = await storageService.deleteFileByHash(deleteHash);
+        const deleted = await storageService.deleteFileByHash(deleteHash, resolvedContextId);
         context.res = {
           status: 200,
           body: { 
@@ -205,11 +205,10 @@ async function CortexFileHandler(context, req) {
 
     // First, get the hash from the map if it exists
     if (deleteHash) {
-      const deleteKey = getScopedHashKey(deleteHash, resolvedContextId);
-      const hashResult = await getFileStoreMap(deleteKey);
+      const hashResult = await getFileStoreMap(deleteHash, false, resolvedContextId);
       if (hashResult) {
-        context.log(`Found hash in map for deletion: ${deleteHash}${resolvedContextId ? ` (contextId: ${resolvedContextId})` : ""}`);
-        await removeFromFileStoreMap(deleteKey);
+        context.log(`Found hash in map for deletion: ${deleteHash}${resolvedContextId ? ` (contextId: ${redactContextId(resolvedContextId)})` : ""}`);
+        await removeFromFileStoreMap(deleteHash, resolvedContextId);
       }
     }
 
@@ -284,15 +283,21 @@ async function CortexFileHandler(context, req) {
       }
 
       // Check if file already exists (using hash or URL as the key)
-      const cacheKey = hash ? getScopedHashKey(hash, resolvedContextId) : remoteUrl;
-      const exists = await getFileStoreMap(cacheKey);
+      // For hash lookups, use raw hash; for URL lookups, use URL as key (unscoped only)
+      const exists = hash 
+        ? await getFileStoreMap(hash, false, resolvedContextId)
+        : await getFileStoreMap(remoteUrl, false, null); // URL lookups are unscoped
       if (exists) {
         context.res = {
           status: 200,
           body: exists,
         };
         //update redis timestamp with current time
-        await setFileStoreMap(cacheKey, exists);
+        if (hash) {
+          await setFileStoreMap(hash, exists, resolvedContextId);
+        } else {
+          await setFileStoreMap(remoteUrl, exists, null); // URL lookups are unscoped
+        }
         return;
       }
 
@@ -315,7 +320,11 @@ async function CortexFileHandler(context, req) {
 
       //Update Redis (using hash or URL as the key)
       // Container parameter is ignored - always uses default container from env var
-      await setFileStoreMap(cacheKey, res);
+      if (hash) {
+        await setFileStoreMap(hash, res, resolvedContextId);
+      } else {
+        await setFileStoreMap(remoteUrl, res, null); // URL lookups are unscoped
+      }
 
       // Return the file URL
       context.res = {
@@ -343,10 +352,9 @@ async function CortexFileHandler(context, req) {
 
   if (hash && clearHash) {
     try {
-      const hashKey = getScopedHashKey(hash, resolvedContextId);
-      const hashValue = await getFileStoreMap(hashKey);
+      const hashValue = await getFileStoreMap(hash, false, resolvedContextId);
       if (hashValue) {
-        await removeFromFileStoreMap(hashKey);
+        await removeFromFileStoreMap(hash, resolvedContextId);
         context.res = {
           status: 200,
           body: `Hash ${hash} removed`,
@@ -368,11 +376,10 @@ async function CortexFileHandler(context, req) {
   }
 
   if (hash && checkHash) {
-    const hashKey = getScopedHashKey(hash, resolvedContextId);
-    let hashResult = await getFileStoreMap(hashKey, true); // Skip lazy cleanup to handle it ourselves
+    let hashResult = await getFileStoreMap(hash, true, resolvedContextId); // Skip lazy cleanup to handle it ourselves
 
     if (hashResult) {
-      context.log(`File exists in map: ${hash}${resolvedContextId ? ` (contextId: ${resolvedContextId})` : ""}`);
+      context.log(`File exists in map: ${hash}${resolvedContextId ? ` (contextId: ${redactContextId(resolvedContextId)})` : ""}`);
 
       // Log the URL retrieved from Redis before checking existence
       context.log(`Checking existence of URL from Redis: ${hashResult?.url}`);
@@ -391,7 +398,7 @@ async function CortexFileHandler(context, req) {
           context.log(
             `File not found in any storage. Removing from map: ${hash}`,
           );
-          await removeFromFileStoreMap(hashKey);
+          await removeFromFileStoreMap(hash, resolvedContextId);
           context.res = {
             status: 404,
             body: `Hash ${hash} not found in storage`,
@@ -410,7 +417,7 @@ async function CortexFileHandler(context, req) {
           } catch (error) {
             context.log(`Error restoring to GCS: ${error}`);
             // If restoration fails, remove the hash from the map
-            await removeFromFileStoreMap(hashKey);
+            await removeFromFileStoreMap(hash, resolvedContextId);
             context.res = {
               status: 404,
               body: `Hash ${hash} not found`,
@@ -468,7 +475,7 @@ async function CortexFileHandler(context, req) {
           } catch (error) {
             console.error("Error restoring from GCS:", error);
             // If restoration fails, remove the hash from the map
-            await removeFromFileStoreMap(hashKey);
+            await removeFromFileStoreMap(hash, resolvedContextId);
             context.res = {
               status: 404,
               body: `Hash ${hash} not found`,
@@ -486,7 +493,7 @@ async function CortexFileHandler(context, req) {
           : false;
         if (!finalPrimaryCheck && !finalGCSCheck) {
           context.log(`Failed to restore file. Removing from map: ${hash}`);
-          await removeFromFileStoreMap(hashKey);
+          await removeFromFileStoreMap(hash, resolvedContextId);
           context.res = {
             status: 404,
             body: `Hash ${hash} not found`,
@@ -503,6 +510,11 @@ async function CortexFileHandler(context, req) {
           hash: hashResult.hash,
           timestamp: new Date().toISOString(),
         };
+        
+        // Include displayFilename if it exists in Redis record
+        if (hashResult.displayFilename) {
+          response.displayFilename = hashResult.displayFilename;
+        }
 
         // Ensure converted version exists and is synced across storage providers
         try {
@@ -602,7 +614,7 @@ async function CortexFileHandler(context, req) {
         }
 
         //update redis timestamp with current time
-        await setFileStoreMap(hashKey, hashResult);
+        await setFileStoreMap(hash, hashResult, resolvedContextId);
 
         context.res = {
           status: 200,
@@ -612,7 +624,7 @@ async function CortexFileHandler(context, req) {
       } catch (error) {
         context.log(`Error checking file existence: ${error}`);
         // If there's an error checking file existence, remove the hash from the map
-        await removeFromFileStoreMap(hashKey);
+        await removeFromFileStoreMap(hash, resolvedContextId);
         context.res = {
           status: 404,
           body: `Hash ${hash} not found`,
@@ -639,12 +651,11 @@ async function CortexFileHandler(context, req) {
     if (result?.hash && context?.res?.body) {
       // Use contextId from result (extracted from form fields) or from resolvedContextId (query/body)
       const uploadContextId = result.contextId || resolvedContextId;
-      const hashKey = getScopedHashKey(result.hash, uploadContextId);
       // Store contextId alongside the entry for debugging/traceability
       if (uploadContextId && typeof context.res.body === "object" && context.res.body) {
         context.res.body.contextId = uploadContextId;
       }
-      await setFileStoreMap(hashKey, context.res.body);
+      await setFileStoreMap(result.hash, context.res.body, uploadContextId);
     }
     return;
   }
diff --git a/helper-apps/cortex-file-handler/src/redis.js b/helper-apps/cortex-file-handler/src/redis.js
index 0515ab77..94347790 100644
--- a/helper-apps/cortex-file-handler/src/redis.js
+++ b/helper-apps/cortex-file-handler/src/redis.js
@@ -233,32 +233,95 @@ const getAllFileStoreMap = async () => {
 };
 
 // Function to set key value in "FileStoreMap" hash map
-const setFileStoreMap = async (key, value) => {
+// If contextId is provided, writes to context-scoped map: FileStoreMap:ctx:<contextId>
+// Otherwise writes to unscoped map: FileStoreMap
+// Key is always the raw hash (no scoping in the key itself)
+const setFileStoreMap = async (hash, value, contextId = null) => {
   try {
+    if (!hash) {
+      console.error("setFileStoreMap: hash is required");
+      return;
+    }
+    
+    // Create a copy of value to avoid mutating the original
+    const valueToStore = { ...value };
+    
+    // Remove 'message' field - it's only for the upload response, not for persistence
+    delete valueToStore.message;
+    
     // Only set timestamp if one doesn't already exist
-    if (!value.timestamp) {
-      value.timestamp = new Date().toISOString();
+    if (!valueToStore.timestamp) {
+      valueToStore.timestamp = new Date().toISOString();
+    }
+    
+    // Determine which map to write to
+    if (contextId) {
+      // Write to context-scoped map with raw hash as key
+      const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+      await client.hset(contextMapKey, hash, JSON.stringify(valueToStore));
+    } else {
+      // Write to unscoped map (backward compatibility)
+      await client.hset("FileStoreMap", hash, JSON.stringify(valueToStore));
     }
-    await client.hset("FileStoreMap", key, JSON.stringify(value));
   } catch (error) {
     console.error(`Error setting key in FileStoreMap: ${error}`);
   }
 };
 
-const getFileStoreMap = async (key, skipLazyCleanup = false) => {
+// Function to get all files for a context from context-scoped hash map
+const getAllFilesForContext = async (contextId) => {
   try {
-    let value = await client.hget("FileStoreMap", key);
+    if (!contextId) {
+      return {};
+    }
+    const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+    const allKeyValuePairs = await client.hgetall(contextMapKey);
+    // Parse each JSON value in the returned object
+    for (const key in allKeyValuePairs) {
+      try {
+        allKeyValuePairs[key] = JSON.parse(allKeyValuePairs[key]);
+      } catch (error) {
+        console.error(`Error parsing JSON for key ${key}: ${error}`);
+        // keep original value if parsing failed
+      }
+    }
+    return allKeyValuePairs;
+  } catch (error) {
+    // Redact contextId in error logs for security
+    const { redactContextId } = await import("./utils/logSecurity.js");
+    const redactedContextId = redactContextId(contextId);
+    console.error(`Error getting all files for context ${redactedContextId}: ${error}`);
+    return {};
+  }
+};
 
+const getFileStoreMap = async (hash, skipLazyCleanup = false, contextId = null) => {
+  try {
+    if (!hash) {
+      return null;
+    }
+    
+    // Try context-scoped map first if contextId is provided
+    let value = null;
+    if (contextId) {
+      const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+      value = await client.hget(contextMapKey, hash);
+    }
+    
+    // Fall back to unscoped map if not found
+    if (!value) {
+      value = await client.hget("FileStoreMap", hash);
+    }
+    
     // Backwards compatibility for unscoped keys only:
     // If unscoped hash doesn't exist, fall back to legacy hash+container key (if still present).
-    // SECURITY: Context-scoped keys (hash:ctx:contextId) NEVER fall back - they must match exactly.
-    if (!value && key) {
-      const ctx = tryParseCtxKey(key);
-      const baseHash = ctx?.hash || key;
+    // SECURITY: Context-scoped lookups NEVER fall back - they must match exactly.
+    if (!value && hash && !contextId) {
+      const baseHash = hash;
 
       // Only allow fallback for unscoped keys (not context-scoped)
       // Context-scoped keys are security-isolated and must match exactly
-      if (!ctx && baseHash && !String(baseHash).includes(":")) {
+      if (baseHash && !String(baseHash).includes(":")) {
         const defaultContainerName = getDefaultContainerName();
         const legacyKey = legacyContainerKey(baseHash, defaultContainerName);
         if (legacyKey) {
@@ -323,10 +386,14 @@ const getFileStoreMap = async (key, skipLazyCleanup = false) => {
             }
 
             // Remove stale entry if both primary and backup are missing
+            // Need to extract contextId from the key if it was scoped
             if (shouldRemove) {
-              await removeFromFileStoreMap(key);
+              // For lazy cleanup, we don't have contextId, so try unscoped first
+              // If the key was scoped, we'd need contextId, but lazy cleanup doesn't have it
+              // So we'll just try to remove from unscoped map
+              await removeFromFileStoreMap(hash, null);
               console.log(
-                `Lazy cleanup: Removed stale cache entry for key ${key}`,
+                `Lazy cleanup: Removed stale cache entry for hash ${hash}`,
               );
               return null; // Return null since file no longer exists
             }
@@ -350,19 +417,31 @@ const getFileStoreMap = async (key, skipLazyCleanup = false) => {
 };
 
 // Function to remove key from "FileStoreMap" hash map
-const removeFromFileStoreMap = async (key) => {
+// If contextId is provided, removes from context-scoped map
+// Otherwise removes from unscoped map
+// Hash is always the raw hash (no scoping in the key itself)
+const removeFromFileStoreMap = async (hash, contextId = null) => {
   try {
-    // hdel returns the number of keys that were removed.
-    // If the key does not exist, 0 is returned.
-    const result = await client.hdel("FileStoreMap", key);
+    if (!hash) {
+      return;
+    }
+    
+    let result = 0;
+    if (contextId) {
+      // Remove from context-scoped map
+      const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+      result = await client.hdel(contextMapKey, hash);
+    } else {
+      // Remove from unscoped map
+      result = await client.hdel("FileStoreMap", hash);
+    }
     if (result > 0) {
-      console.log(`The key ${key} was removed successfully`);
+      console.log(`The hash ${hash} was removed successfully`);
     }
 
     // Always try to clean up legacy container-scoped entry as well.
     // This ensures we don't leave orphaned legacy keys behind.
-    const ctx = tryParseCtxKey(key);
-    const baseHash = ctx?.hash || key;
+    const baseHash = hash;
     // Only attempt legacy cleanup if baseHash doesn't contain a colon (not already scoped)
     if (!String(baseHash).includes(":")) {
       const defaultContainerName = getDefaultContainerName();
@@ -491,6 +570,7 @@ export {
   setFileStoreMap,
   getFileStoreMap,
   removeFromFileStoreMap,
+  getAllFilesForContext,
   cleanupRedisFileStoreMap,
   cleanupRedisFileStoreMapAge,
   acquireLock,
diff --git a/helper-apps/cortex-file-handler/src/services/storage/StorageService.js b/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
index ea4f728a..3c038703 100644
--- a/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
+++ b/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
@@ -177,9 +177,10 @@ export class StorageService {
   /**
    * Delete a single file by its hash from both primary and backup storage
    * @param {string} hash - The hash of the file to delete
+   * @param {string|null} contextId - Optional context ID for context-scoped files
    * @returns {Promise<Object>} Object containing deletion results and file info
    */
-  async deleteFileByHash(hash) {
+  async deleteFileByHash(hash, contextId = null) {
     await this._initialize();
     
     if (!hash) {
@@ -190,11 +191,11 @@ export class StorageService {
 
     // Get and remove file information from Redis map
     const { getFileStoreMap, removeFromFileStoreMap } = await import("../../redis.js");
-    const hashResult = await getFileStoreMap(hash);
+    const hashResult = await getFileStoreMap(hash, false, contextId);
     
     if (hashResult) {
       // Remove from Redis
-      await removeFromFileStoreMap(hash);
+      await removeFromFileStoreMap(hash, contextId);
     }
     
     if (!hashResult) {
@@ -256,7 +257,8 @@ export class StorageService {
     return {
       hash,
       deleted: results,
-      filename: hashResult.filename
+      filename: hashResult.filename,
+      ...(hashResult.displayFilename && { displayFilename: hashResult.displayFilename })
     };
   }
 
@@ -281,12 +283,10 @@ export class StorageService {
     }
 
     // Get Redis functions
-    const { getFileStoreMap, setFileStoreMap, getScopedHashKey } = await import("../../redis.js");
+    const { getFileStoreMap, setFileStoreMap } = await import("../../redis.js");
     
-    // Look up file by hash using context-scoped key if contextId provided
-    // getFileStoreMap already handles fallback to unscoped keys
-    const scopedKey = getScopedHashKey(hash, contextId);
-    const hashResult = await getFileStoreMap(scopedKey);
+    // Look up file by hash - getFileStoreMap handles context-scoped maps automatically
+    const hashResult = await getFileStoreMap(hash, false, contextId);
     
     if (!hashResult) {
       throw new Error(`File with hash ${hash} not found`);
@@ -346,7 +346,6 @@ export class StorageService {
     }
 
     // Update Redis with new information (including shortLivedUrl and permanent flag)
-    // Use the same scoped key that was used for lookup
     // Store as permanent boolean to match file collection logic
     const newFileInfo = {
       ...hashResult,
@@ -361,12 +360,14 @@ export class StorageService {
       newFileInfo.converted = convertedResult;
     }
 
-    await setFileStoreMap(scopedKey, newFileInfo);
-    context.log?.(`Updated Redis map for hash: ${hash}${contextId ? ` (contextId: ${contextId})` : ""}`);
+    await setFileStoreMap(hash, newFileInfo, contextId);
+    const { redactContextId } = await import("../../utils/logSecurity.js");
+    context.log?.(`Updated Redis map for hash: ${hash}${contextId ? ` (contextId: ${redactContextId(contextId)})` : ""}`);
 
     return {
       hash,
       filename: hashResult.filename,
+      ...(hashResult.displayFilename && { displayFilename: hashResult.displayFilename }),
       retention: retention,
       url: hashResult.url,
       shortLivedUrl: shortLivedUrl,
diff --git a/helper-apps/cortex-file-handler/src/utils/logSecurity.js b/helper-apps/cortex-file-handler/src/utils/logSecurity.js
index cd384759..ce4e84b9 100644
--- a/helper-apps/cortex-file-handler/src/utils/logSecurity.js
+++ b/helper-apps/cortex-file-handler/src/utils/logSecurity.js
@@ -2,6 +2,26 @@
  * Security utilities for logging - redacts sensitive information from logs
  */
 
+/**
+ * Redacts contextId for security in logs - shows first 4 and last 4 characters
+ * @param {string|null|undefined} contextId - The contextId to redact
+ * @returns {string} - Redacted contextId (e.g., "abcd...xyz1") or empty string if null/undefined
+ */
+export function redactContextId(contextId) {
+  if (!contextId || typeof contextId !== 'string') return '';
+  
+  // If contextId is 8 characters or less, just show first 2 and last 2
+  if (contextId.length <= 8) {
+    if (contextId.length <= 4) {
+      return '****'; // Too short to show anything meaningful
+    }
+    return `${contextId.substring(0, 2)}...${contextId.substring(contextId.length - 2)}`;
+  }
+  
+  // Show first 4 and last 4 characters for longer IDs
+  return `${contextId.substring(0, 4)}...${contextId.substring(contextId.length - 4)}`;
+}
+
 /**
  * Redacts SAS tokens from Azure Blob Storage URLs for security in logs
  * @param {string} url - The URL that may contain a SAS token
diff --git a/helper-apps/cortex-file-handler/tests/fileUpload.test.js b/helper-apps/cortex-file-handler/tests/fileUpload.test.js
index 40bba0eb..2fc959af 100644
--- a/helper-apps/cortex-file-handler/tests/fileUpload.test.js
+++ b/helper-apps/cortex-file-handler/tests/fileUpload.test.js
@@ -627,3 +627,104 @@ The emdash should be preserved correctly when the file is downloaded.`;
     }
   }
 });
+
+// DisplayFilename persistence and retrieval tests
+test.serial("should persist and return displayFilename in all responses", async (t) => {
+  const originalFilename = "my-original-file-name-with-special-chars-123.txt";
+  const fileContent = "test content for displayFilename";
+  const hash = "test-displayfilename-" + uuidv4();
+  
+  // Create a temporary file
+  const filePath = await createTestFile(fileContent, "txt");
+  let uploadResponse;
+  let checkHashResponse;
+  let deleteResponse;
+  
+  try {
+    // Upload file with original filename specified
+    const form = new FormData();
+    form.append("file", fs.createReadStream(filePath), originalFilename);
+    form.append("hash", hash);
+    
+    uploadResponse = await axios.post(baseUrl, form, {
+      headers: {
+        ...form.getHeaders(),
+        "Content-Type": "multipart/form-data",
+      },
+      validateStatus: (status) => true,
+      timeout: 30000,
+    });
+    
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+    t.truthy(uploadResponse.data.filename, "Response should contain filename");
+    t.is(
+      uploadResponse.data.displayFilename,
+      originalFilename,
+      "Upload response should contain displayFilename matching original filename"
+    );
+    
+    // Wait for Redis operations to complete
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    
+    // Check hash - should return displayFilename
+    checkHashResponse = await axios.get(baseUrl, {
+      params: {
+        hash,
+        checkHash: true,
+      },
+      validateStatus: (status) => true,
+    });
+    
+    t.is(checkHashResponse.status, 200, "Hash check should succeed");
+    t.is(
+      checkHashResponse.data.displayFilename,
+      originalFilename,
+      "checkHash response should contain displayFilename matching original filename"
+    );
+    t.is(
+      checkHashResponse.data.filename,
+      uploadResponse.data.filename,
+      "checkHash response should contain same filename as upload"
+    );
+    
+    // Test setRetention - should return displayFilename
+    const retentionResponse = await axios.get(baseUrl, {
+      params: {
+        hash,
+        setRetention: true,
+        retention: "permanent",
+      },
+      validateStatus: (status) => true,
+    });
+    
+    t.is(retentionResponse.status, 200, "setRetention should succeed");
+    t.is(
+      retentionResponse.data.displayFilename,
+      originalFilename,
+      "setRetention response should contain displayFilename"
+    );
+    
+    // Test delete - should return displayFilename
+    deleteResponse = await axios.delete(baseUrl, {
+      params: {
+        hash,
+      },
+      validateStatus: (status) => true,
+    });
+    
+    t.is(deleteResponse.status, 200, "Delete should succeed");
+    t.is(
+      deleteResponse.data.deleted.filename,
+      uploadResponse.data.filename,
+      "Delete response should contain filename"
+    );
+    t.is(
+      deleteResponse.data.deleted.displayFilename,
+      originalFilename,
+      "Delete response should contain displayFilename"
+    );
+  } finally {
+    fs.unlinkSync(filePath);
+    // Cleanup is handled by delete operation above
+  }
+});
diff --git a/helper-apps/cortex-file-handler/tests/setRetention.test.js b/helper-apps/cortex-file-handler/tests/setRetention.test.js
index 6736672b..2f43215f 100644
--- a/helper-apps/cortex-file-handler/tests/setRetention.test.js
+++ b/helper-apps/cortex-file-handler/tests/setRetention.test.js
@@ -141,8 +141,7 @@ test.serial("should set file retention to permanent", async (t) => {
     fs.unlinkSync(filePath);
     // Cleanup
     try {
-      const { getScopedHashKey } = await import("../src/redis.js");
-      await removeFromFileStoreMap(getScopedHashKey(testHash));
+      await removeFromFileStoreMap(testHash);
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -184,8 +183,7 @@ test.serial("should set file retention to temporary", async (t) => {
     fs.unlinkSync(filePath);
     // Cleanup
     try {
-      const { getScopedHashKey } = await import("../src/redis.js");
-      await removeFromFileStoreMap(getScopedHashKey(testHash));
+      await removeFromFileStoreMap(testHash);
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -220,8 +218,7 @@ test.serial("should set retention using request body parameters", async (t) => {
     fs.unlinkSync(filePath);
     // Cleanup
     try {
-      const { getScopedHashKey } = await import("../src/redis.js");
-      await removeFromFileStoreMap(getScopedHashKey(testHash));
+      await removeFromFileStoreMap(testHash);
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -302,8 +299,7 @@ test.serial("should update Redis map with retention information", async (t) => {
     await new Promise((resolve) => setTimeout(resolve, 1000));
 
     // Verify Redis entry exists
-    const scopedHash = getScopedHashKey(testHash);
-    const oldEntry = await getFileStoreMap(scopedHash);
+    const oldEntry = await getFileStoreMap(testHash);
     t.truthy(oldEntry, "Redis entry should exist before setting retention");
     t.is(oldEntry.permanent, false, "New uploads should have permanent=false by default");
 
@@ -315,7 +311,7 @@ test.serial("should update Redis map with retention information", async (t) => {
     await new Promise((resolve) => setTimeout(resolve, 1000));
 
     // Verify Redis entry is updated
-    const newEntry = await getFileStoreMap(scopedHash);
+    const newEntry = await getFileStoreMap(testHash);
     t.truthy(newEntry, "Redis entry should still exist after setting retention");
     t.is(newEntry.url, retentionResponse.data.url, "Entry should have correct URL");
     t.truthy(newEntry.shortLivedUrl, "Entry should have shortLivedUrl");
@@ -325,8 +321,7 @@ test.serial("should update Redis map with retention information", async (t) => {
     fs.unlinkSync(filePath);
     // Cleanup
     try {
-      const { getScopedHashKey } = await import("../src/redis.js");
-      await removeFromFileStoreMap(getScopedHashKey(testHash));
+      await removeFromFileStoreMap(testHash);
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -374,8 +369,7 @@ test.serial("should preserve file metadata after setting retention", async (t) =
     fs.unlinkSync(filePath);
     // Cleanup
     try {
-      const { getScopedHashKey } = await import("../src/redis.js");
-      await removeFromFileStoreMap(getScopedHashKey(testHash));
+      await removeFromFileStoreMap(testHash);
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -425,8 +419,7 @@ test.serial("should support operation=setRetention query parameter", async (t) =
     fs.unlinkSync(filePath);
     // Cleanup
     try {
-      const { getScopedHashKey } = await import("../src/redis.js");
-      await removeFromFileStoreMap(getScopedHashKey(testHash));
+      await removeFromFileStoreMap(testHash);
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -485,8 +478,7 @@ test.serial("should preserve GCS URL when setting retention", async (t) => {
     fs.unlinkSync(filePath);
     // Cleanup
     try {
-      const { getScopedHashKey } = await import("../src/redis.js");
-      await removeFromFileStoreMap(getScopedHashKey(testHash));
+      await removeFromFileStoreMap(testHash);
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -523,8 +515,7 @@ test.serial("should always include shortLivedUrl in response", async (t) => {
     fs.unlinkSync(filePath);
     // Cleanup
     try {
-      const { getScopedHashKey } = await import("../src/redis.js");
-      await removeFromFileStoreMap(getScopedHashKey(testHash));
+      await removeFromFileStoreMap(testHash);
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -559,9 +550,7 @@ test.serial("should set retention for context-scoped file", async (t) => {
     t.truthy(retentionResponse.data.shortLivedUrl, "Should have shortLivedUrl");
 
     // Verify Redis entry was updated with context-scoped key
-    const { getScopedHashKey } = await import("../src/redis.js");
-    const scopedKey = getScopedHashKey(testHash, contextId);
-    const updatedEntry = await getFileStoreMap(scopedKey);
+    const updatedEntry = await getFileStoreMap(testHash, false, contextId);
     t.truthy(updatedEntry, "Should have updated entry in Redis");
     t.truthy(updatedEntry.shortLivedUrl, "Should have shortLivedUrl in Redis entry");
     t.is(updatedEntry.permanent, true, "Entry should have permanent=true in Redis (matches file collection logic)");
@@ -577,9 +566,7 @@ test.serial("should set retention for context-scoped file", async (t) => {
     fs.unlinkSync(filePath);
     // Cleanup
     try {
-      const { getScopedHashKey } = await import("../src/redis.js");
-      const scopedKey = getScopedHashKey(testHash, contextId);
-      await removeFromFileStoreMap(scopedKey);
+      await removeFromFileStoreMap(testHash, contextId);
     } catch (e) {
       // Ignore cleanup errors
     }
@@ -616,9 +603,7 @@ test.serial("should return 404 when contextId doesn't match for setRetention", a
     fs.unlinkSync(filePath);
     // Cleanup
     try {
-      const { getScopedHashKey } = await import("../src/redis.js");
-      const scopedKey = getScopedHashKey(testHash, contextId1);
-      await removeFromFileStoreMap(scopedKey);
+      await removeFromFileStoreMap(testHash, contextId1);
     } catch (e) {
       // Ignore cleanup errors
     }
diff --git a/lib/fileUtils.js b/lib/fileUtils.js
index ca28f3b3..6a13c6f3 100644
--- a/lib/fileUtils.js
+++ b/lib/fileUtils.js
@@ -312,6 +312,8 @@ async function deleteFileByHash(hash, pathwayResolver = null, contextId = null)
 
 // Helper function to extract file metadata from a content object
 // Returns normalized format with url and gcs (for file collection storage)
+// Note: displayFilename is not extracted from messages - it's set by CFH on upload,
+// or by sys_update_file_metadata.js, or by file collection tools
 function extractFileMetadataFromContent(contentObj) {
     const files = [];
     
@@ -319,7 +321,6 @@ function extractFileMetadataFromContent(contentObj) {
         files.push({
             url: contentObj.image_url.url,
             gcs: contentObj.gcs || null,
-            filename: contentObj.originalFilename || contentObj.name || contentObj.filename || null,
             hash: contentObj.hash || null,
             type: 'image_url'
         });
@@ -327,7 +328,6 @@ function extractFileMetadataFromContent(contentObj) {
         files.push({
             url: contentObj.url,
             gcs: contentObj.gcs || null,
-            filename: contentObj.originalFilename || contentObj.name || contentObj.filename || null,
             hash: contentObj.hash || null,
             type: 'file'
         });
@@ -336,7 +336,6 @@ function extractFileMetadataFromContent(contentObj) {
         files.push({
             url: contentObj.url,
             gcs: contentObj.gcs || null,
-            filename: contentObj.originalFilename || contentObj.name || contentObj.filename || null,
             hash: contentObj.hash || null,
             type: contentObj.type || 'file'
         });
@@ -349,23 +348,50 @@ function extractFileMetadataFromContent(contentObj) {
 const fileCollectionCache = new Map();
 const CACHE_TTL = 5000; // 5 seconds
 
-/**
- * Generate a unique version string with timestamp + random component
- * This handles clock skew by ensuring uniqueness even when timestamps collide
- * @returns {string} Version string in format: "timestamp-random"
- */
-function generateVersion() {
-    const timestamp = Date.now();
-    const random = Math.random().toString(36).substring(2, 10);
-    return `${timestamp}-${random}`;
+// Singleton Redis client for file collection operations
+let redisClientSingleton = null;
+
+// Helper to get Redis client for direct hash map access
+async function getRedisClient() {
+    if (redisClientSingleton) {
+        return redisClientSingleton;
+    }
+    
+    try {
+        const { config } = await import('../config.js');
+        const connectionString = config.get('storageConnectionString');
+        if (!connectionString) {
+            return null;
+        }
+        
+        // Import Redis and create client
+        const Redis = (await import('ioredis')).default;
+        redisClientSingleton = new Redis(connectionString, {
+            maxRetriesPerRequest: null,
+            enableReadyCheck: true,
+            lazyConnect: false,
+            connectTimeout: 10000,
+        });
+        
+        // Handle errors
+        redisClientSingleton.on('error', async (error) => {
+            const logger = (await import('./logger.js')).default;
+            logger.error(`Redis client error in fileUtils: ${error}`);
+        });
+        
+        return redisClientSingleton;
+    } catch (e) {
+        return null;
+    }
 }
 
+
 /**
  * Get cache key for file collection
  */
 function getCollectionCacheKey(contextId, contextKey) {
-    // Use memoryFiles section key for cache
-    return `${contextId}-memoryFiles-${contextKey || 'default'}`;
+    // Cache key for file collection (legacy format maintained for cache compatibility)
+    return `${contextId}-fileCollection-${contextKey || 'default'}`;
 }
 
 /**
@@ -417,15 +443,14 @@ function extractFilesFromChatHistory(chatHistory) {
 
 /**
  * Load file collection from memory system or cache
- * Returns both the collection data and version for optimistic locking
  * @param {string} contextId - Context ID for the file collection
  * @param {string} contextKey - Optional context key for encryption
  * @param {boolean} useCache - Whether to check cache first (default: true)
- * @returns {Promise<{files: Array, version: string}>} File collection with version
+ * @returns {Promise<Array>} File collection array
  */
-async function loadFileCollectionWithVersion(contextId, contextKey = null, useCache = true) {
+async function loadFileCollection(contextId, contextKey = null, useCache = true) {
     if (!contextId) {
-        return { files: [], version: generateVersion() };
+        return [];
     }
 
     const cacheKey = getCollectionCacheKey(contextId, contextKey);
@@ -434,234 +459,232 @@ async function loadFileCollectionWithVersion(contextId, contextKey = null, useCa
     if (useCache && fileCollectionCache.has(cacheKey)) {
         const cached = fileCollectionCache.get(cacheKey);
         if (Date.now() - cached.timestamp < CACHE_TTL) {
-            return { files: cached.collection, version: cached.version || generateVersion() };
+            return cached.collection;
         }
     }
 
-    // Load from memory system
-    const { callPathway } = await import('./pathwayTools.js');
+    // Load from context-scoped Redis hash map (FileStoreMap:ctx:<contextId>)
     let files = [];
-    let version = generateVersion();
     
     try {
-        const memoryContent = await callPathway('sys_read_memory', { 
-            contextId, 
-            section: 'memoryFiles',
-            contextKey 
-        });
-        if (memoryContent) {
-            const parsed = JSON.parse(memoryContent);
+        const redisClient = await getRedisClient();
+        
+        if (redisClient) {
+            const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+            const allFiles = await redisClient.hgetall(contextMapKey);
             
-            // Handle new format: { version, files }
-            if (parsed && typeof parsed === 'object' && !Array.isArray(parsed) && parsed.files) {
-                files = Array.isArray(parsed.files) ? parsed.files : [];
-                version = parsed.version || generateVersion();
-            }
-            // Handle old format: just an array (backward compatibility)
-            else if (Array.isArray(parsed)) {
-                files = parsed;
-                version = generateVersion(); // Assign new version for migration
-            }
-            // Invalid format
-            else {
-                files = [];
-                version = generateVersion();
-            }
+            // Convert hash map entries to file collection array
+            // Each entry is {hash: fileData}
+            files = Object.entries(allFiles).map(([hash, fileDataStr]) => {
+                try {
+                    const fileData = JSON.parse(fileDataStr);
+                    // Extract file collection metadata (tags, notes, etc.) if present
+                    // Otherwise create minimal entry from CFH data
+                    // Use displayFilename (user-friendly name) instead of filename (CFH-managed)
+                    // Fallback to filename if displayFilename is not set (for files uploaded before displayFilename was added)
+                    return {
+                        id: fileData.id || `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
+                        url: fileData.url,
+                        gcs: fileData.gcs || null,
+                        displayFilename: fileData.displayFilename || fileData.filename || null,
+                        mimeType: fileData.mimeType || null,
+                        tags: fileData.tags || [],
+                        notes: fileData.notes || '',
+                        hash: hash,
+                        permanent: fileData.permanent || false,
+                        addedDate: fileData.addedDate || fileData.timestamp || new Date().toISOString(),
+                        lastAccessed: fileData.lastAccessed || fileData.timestamp || new Date().toISOString()
+                    };
+                } catch (e) {
+                    // Skip invalid entries
+                    return null;
+                }
+            }).filter(Boolean);
+            
+            // Sort by lastAccessed (most recent first)
+            files.sort((a, b) => {
+                const aDate = new Date(a.lastAccessed || a.addedDate || 0);
+                const bDate = new Date(b.lastAccessed || b.addedDate || 0);
+                return bDate - aDate;
+            });
         }
     } catch (e) {
-        // Collection doesn't exist yet, start with empty array
+        // Collection doesn't exist yet or error reading, start with empty array
         files = [];
-        version = generateVersion();
     }
 
     // Update cache
     fileCollectionCache.set(cacheKey, {
         collection: files,
-        version: version,
         timestamp: Date.now()
     });
 
-    return { files, version };
+    return files;
 }
 
 /**
- * Load file collection from memory system or cache
- * @param {string} contextId - Context ID for the file collection
- * @param {string} contextKey - Optional context key for encryption
- * @param {boolean} useCache - Whether to check cache first (default: true)
- * @returns {Promise<Array>} File collection array
+ * Update file metadata in Redis hash map (direct atomic operation)
+ * @param {string} contextId - Context ID
+ * @param {string} hash - File hash
+ * @param {Object} metadata - Metadata to update (displayFilename, id, tags, notes, mimeType, addedDate, lastAccessed, permanent)
+ * Note: Does NOT update CFH core fields (url, gcs, hash, filename) - those are managed by CFH
+ * @returns {Promise<boolean>} True if successful
  */
-async function loadFileCollection(contextId, contextKey = null, useCache = true) {
-    const { files } = await loadFileCollectionWithVersion(contextId, contextKey, useCache);
-    return files;
+async function updateFileMetadata(contextId, hash, metadata) {
+    if (!contextId || !hash) {
+        return false;
+    }
+    
+    try {
+        const redisClient = await getRedisClient();
+        if (!redisClient) {
+            return false;
+        }
+        
+        const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+        // Get existing file data from CFH (if any)
+        const existingDataStr = await redisClient.hget(contextMapKey, hash);
+        let existingData = {};
+        if (existingDataStr) {
+            try {
+                existingData = JSON.parse(existingDataStr);
+            } catch (e) {
+                // Invalid data, start fresh
+                existingData = {};
+            }
+        }
+        
+        // Merge CFH data with Cortex metadata
+        // Only update Cortex-managed fields, preserve CFH fields (url, gcs, hash, filename)
+        const fileData = {
+            ...existingData, // Preserve all CFH data (url, gcs, hash, filename, etc.)
+            // Update only Cortex-managed metadata fields
+            ...(metadata.displayFilename !== undefined && { displayFilename: metadata.displayFilename }),
+            ...(metadata.id !== undefined && { id: metadata.id }),
+            ...(metadata.tags !== undefined && { tags: metadata.tags }),
+            ...(metadata.notes !== undefined && { notes: metadata.notes }),
+            ...(metadata.mimeType !== undefined && { mimeType: metadata.mimeType }),
+            ...(metadata.addedDate !== undefined && { addedDate: metadata.addedDate }),
+            ...(metadata.lastAccessed !== undefined && { lastAccessed: metadata.lastAccessed }),
+            ...(metadata.permanent !== undefined && { permanent: metadata.permanent })
+        };
+        
+        // Write back to hash map (atomic operation)
+        await redisClient.hset(contextMapKey, hash, JSON.stringify(fileData));
+        
+        // Invalidate cache
+        const cacheKey = getCollectionCacheKey(contextId, null);
+        fileCollectionCache.delete(cacheKey);
+        
+        return true;
+    } catch (e) {
+        const logger = (await import('./logger.js')).default;
+        logger.warn(`Failed to update file metadata: ${e.message}`);
+        return false;
+    }
 }
 
 /**
- * Save file collection to memory system with version checking
+ * Save file collection to memory system
+ * Only updates files that have changed (optimized)
  * @param {string} contextId - Context ID for the file collection
- * @param {string} contextKey - Optional context key for encryption
+ * @param {string} contextKey - Optional context key for encryption (unused with hash maps)
  * @param {Array} collection - File collection array
- * @param {string} expectedVersion - Expected version for optimistic locking (if provided)
- * @returns {Promise<boolean>} True if save succeeded, false if version mismatch
+ * @returns {Promise<boolean>} True if successful
  */
-async function saveFileCollectionWithVersion(contextId, contextKey, collection, expectedVersion = null) {
+async function saveFileCollection(contextId, contextKey, collection) {
     const cacheKey = getCollectionCacheKey(contextId, contextKey);
-    const newVersion = generateVersion();
 
     try {
-        const { callPathway } = await import('./pathwayTools.js');
+        const redisClient = await getRedisClient();
+        if (!redisClient) {
+            return false;
+        }
         
-        // If expectedVersion is provided, verify it matches RIGHT before saving
-        // This minimizes the race condition window
-        if (expectedVersion !== null) {
-            // Read directly from memory (bypass cache) to get the absolute latest version
-            const memoryContent = await callPathway('sys_read_memory', { 
-                contextId, 
-                section: 'memoryFiles',
-                contextKey 
-            });
+        const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+        
+        // Get current state to detect changes
+        const currentFiles = await redisClient.hgetall(contextMapKey);
+        
+        // Update only files that changed or are new
+        for (const file of collection) {
+            // Generate hash from URL if not present (for files added without hash)
+            let fileHash = file.hash;
+            if (!fileHash && file.url) {
+                fileHash = await computeBufferHash(Buffer.from(file.url));
+            }
+            if (!fileHash) continue;
             
-            let currentVersion = null;
-            let collectionExists = false;
-            let isOldFormat = false;
+            const currentDataStr = currentFiles[fileHash];
+            let needsUpdate = true;
             
-            if (memoryContent && memoryContent.trim() !== '' && memoryContent.trim() !== '[]') {
-                collectionExists = true;
+            // Check if file actually changed
+            if (currentDataStr) {
                 try {
-                    const parsed = JSON.parse(memoryContent);
-                    // Handle new format: { version, files }
-                    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed) && parsed.version) {
-                        currentVersion = parsed.version;
-                    }
-                    // Handle old format: just an array (no version yet)
-                    else if (Array.isArray(parsed)) {
-                        // Old format - we'll allow migration if the content matches
-                        isOldFormat = true;
-                        currentVersion = null;
+                    const currentData = JSON.parse(currentDataStr);
+                    // Compare metadata fields (ignore CFH fields like url, gcs, timestamp)
+                    if (currentData.id === file.id &&
+                        JSON.stringify(currentData.tags || []) === JSON.stringify(file.tags || []) &&
+                        currentData.notes === (file.notes || '') &&
+                        currentData.mimeType === (file.mimeType || null) &&
+                        currentData.permanent === (file.permanent || false)) {
+                        needsUpdate = false;
                     }
                 } catch (e) {
-                    // Invalid format - treat as version mismatch
-                    currentVersion = null;
+                    // Invalid data, needs update
                 }
             }
             
-            // If collection doesn't exist yet (empty memoryContent or just "[]"), allow the save
-            // since there's nothing to conflict with. The version check is only needed
-            // when there's an existing collection that might have been modified.
-            // Also allow save if we're migrating from old format (isOldFormat) - the migration
-            // will happen on the next load, so we allow this save to proceed.
-            if (collectionExists && !isOldFormat && currentVersion !== expectedVersion) {
-                // Version mismatch - return false to trigger retry
-                return false;
+            if (needsUpdate) {
+                // Get existing CFH data
+                let existingData = {};
+                if (currentDataStr) {
+                    try {
+                        existingData = JSON.parse(currentDataStr);
+                    } catch (e) {
+                        existingData = {};
+                    }
+                }
+                
+                // Merge CFH data with Cortex metadata
+                // Preserve all CFH fields (url, gcs, filename, displayFilename, etc.)
+                const fileData = {
+                    ...existingData, // Preserve all CFH data first
+                    id: file.id,
+                    url: file.url || existingData.url, // Preserve URL (CFH-managed)
+                    gcs: file.gcs || existingData.gcs || null, // Preserve GCS (CFH-managed)
+                    // Preserve CFH's filename (CFH-managed), only update displayFilename (Cortex-managed)
+                    displayFilename: file.displayFilename !== undefined ? file.displayFilename : (existingData.displayFilename || null),
+                    tags: file.tags || [],
+                    notes: file.notes || '',
+                    mimeType: file.mimeType || existingData.mimeType || null,
+                    addedDate: file.addedDate || existingData.timestamp || new Date().toISOString(),
+                    lastAccessed: file.lastAccessed || new Date().toISOString(),
+                    permanent: file.permanent !== undefined ? file.permanent : (existingData.permanent || false)
+                };
+                
+                // Write back to hash map (atomic operation)
+                await redisClient.hset(contextMapKey, fileHash, JSON.stringify(fileData));
             }
         }
         
-        // Save with version (minimal window between check and save)
-        const collectionData = {
-            version: newVersion,
-            files: collection
-        };
-        
-        await callPathway('sys_save_memory', { 
-            contextId, 
-            section: 'memoryFiles',
-            aiMemory: JSON.stringify(collectionData),
-            contextKey 
-        });
+        // Note: We don't remove files from hash map when removed from collection
+        // CFH manages file lifecycle, and files might still exist in storage
 
         // Update cache
         fileCollectionCache.set(cacheKey, {
             collection,
-            version: newVersion,
             timestamp: Date.now()
         });
         
         return true;
     } catch (e) {
-        // Log but don't fail - collection update is best effort
         const logger = (await import('./logger.js')).default;
         logger.warn(`Failed to save file collection: ${e.message}`);
         return false;
     }
 }
 
-/**
- * Save file collection to memory system
- * @param {string} contextId - Context ID for the file collection
- * @param {string} contextKey - Optional context key for encryption
- * @param {Array} collection - File collection array
- */
-async function saveFileCollection(contextId, contextKey, collection) {
-    await saveFileCollectionWithVersion(contextId, contextKey, collection, null);
-}
-
-/**
- * Modify file collection with optimistic locking and automatic retries
- * This is the main function that all modify operations should use to ensure concurrency safety
- * @param {string} contextId - Context ID for the file collection
- * @param {string} contextKey - Optional context key for encryption
- * @param {Function} modifierCallback - Callback function that modifies the collection array
- *   The callback receives (collection) and should return the modified collection array
- * @param {number} maxRetries - Maximum number of retry attempts (default: 5)
- * @returns {Promise<Array>} The final modified collection array
- */
-async function modifyFileCollectionWithLock(contextId, contextKey, modifierCallback, maxRetries = 5) {
-    if (!contextId) {
-        throw new Error("contextId is required");
-    }
-    
-    if (typeof modifierCallback !== 'function') {
-        throw new Error("modifierCallback must be a function");
-    }
-    
-    let lastError = null;
-    
-    for (let attempt = 0; attempt < maxRetries; attempt++) {
-        try {
-            // Load collection with version (skip cache to get latest version)
-            const { files, version } = await loadFileCollectionWithVersion(contextId, contextKey, false);
-            
-            // Create a copy to avoid mutating the original
-            const collectionCopy = [...files];
-            
-            // Execute the modifier callback
-            const modifiedCollection = await modifierCallback(collectionCopy);
-            
-            // Validate that callback returned an array
-            if (!Array.isArray(modifiedCollection)) {
-                throw new Error("modifierCallback must return an array");
-            }
-            
-            // Try to save with version check
-            const saved = await saveFileCollectionWithVersion(contextId, contextKey, modifiedCollection, version);
-            
-            if (saved) {
-                // Success! Return the modified collection
-                return modifiedCollection;
-            }
-            
-            // Version mismatch - will retry on next iteration
-            // Add a small delay to reduce contention (exponential backoff)
-            if (attempt < maxRetries - 1) {
-                const delay = Math.min(10 * Math.pow(2, attempt), 100); // Max 100ms
-                await new Promise(resolve => setTimeout(resolve, delay));
-            }
-        } catch (error) {
-            lastError = error;
-            // For non-version-mismatch errors, we might want to retry or fail immediately
-            // For now, we'll retry a few times then throw
-            if (attempt === maxRetries - 1) {
-                throw error;
-            }
-            // Small delay before retry
-            const delay = Math.min(10 * Math.pow(2, attempt), 100);
-            await new Promise(resolve => setTimeout(resolve, delay));
-        }
-    }
-    
-    // If we get here, all retries failed due to version mismatches
-    throw new Error(`Failed to modify file collection after ${maxRetries} attempts due to concurrent modifications`);
-}
 
 /**
  * Add a file to the file collection
@@ -722,26 +745,73 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
     // Ensure filename has correct extension based on MIME type
     const correctedFilename = ensureFilenameExtension(filename, mimeType);
 
+    // If no hash, generate one from URL for storage key (needed for Redis hash map)
+    const storageHash = finalHash || (finalUrl ? await computeBufferHash(Buffer.from(finalUrl)) : null);
+    
     // Create file entry (before locking to avoid recreating on retry)
     const fileEntry = {
         id: `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
         url: finalUrl,
         gcs: finalGcs || null,
-        filename: correctedFilename,
+        displayFilename: correctedFilename, // Store user-provided filename as displayFilename (filename is managed by CFH)
         mimeType: mimeType,
         tags: Array.isArray(tags) ? tags : [],
         notes: notes || '',
-        hash: finalHash || null,
+        hash: storageHash, // Use storageHash (actual hash or generated from URL)
         permanent: permanent,
         addedDate: new Date().toISOString(),
         lastAccessed: new Date().toISOString()
     };
 
-    // Use optimistic locking to add file to collection
-    await modifyFileCollectionWithLock(contextId, contextKey, (collection) => {
-        collection.push(fileEntry);
-        return collection;
-    });
+    // Write file metadata directly to Redis hash map (atomic operation)
+    // No need for optimistic locking - Redis HSET is atomic per key
+    // If a file with the same hash already exists, update it (same content, possibly different metadata)
+    if (storageHash) {
+        try {
+            const redisClient = await getRedisClient();
+            if (redisClient) {
+                const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+                // Get existing file data from CFH (if any)
+                const existingDataStr = await redisClient.hget(contextMapKey, storageHash);
+                let existingData = {};
+                
+                if (existingDataStr) {
+                    try {
+                        existingData = JSON.parse(existingDataStr);
+                    } catch (e) {
+                        // Invalid data, start fresh
+                        existingData = {};
+                    }
+                }
+                
+                // Merge CFH data with Cortex metadata
+                // If file already exists with same hash, update metadata but keep the existing entry
+                const fileData = {
+                    ...existingData, // Preserve CFH data (url, gcs, filename, etc.)
+                    // Update Cortex metadata (use new ID if this is a new entry, otherwise keep existing)
+                    id: existingData.id || fileEntry.id,
+                    url: finalUrl || existingData.url, // Use new URL if provided, otherwise keep existing
+                    gcs: finalGcs || existingData.gcs || null, // Use new GCS if provided, otherwise keep existing
+                    // Preserve CFH's filename (managed by CFH), store user-provided filename as displayFilename
+                    displayFilename: correctedFilename, // Store user-provided filename as displayFilename
+                    tags: fileEntry.tags.length > 0 ? fileEntry.tags : (existingData.tags || []), // Merge tags if new ones provided
+                    notes: fileEntry.notes || existingData.notes || '', // Keep existing notes if new ones empty
+                    mimeType: fileEntry.mimeType || existingData.mimeType || null,
+                    addedDate: existingData.addedDate || fileEntry.addedDate, // Keep earliest addedDate
+                    lastAccessed: new Date().toISOString(), // Always update lastAccessed
+                    permanent: fileEntry.permanent !== undefined ? fileEntry.permanent : (existingData.permanent || false),
+                    hash: storageHash // Store the hash used as key (actual hash or generated from URL)
+                };
+                
+                // Write back to hash map (atomic operation) - same hash key, just update metadata
+                await redisClient.hset(contextMapKey, storageHash, JSON.stringify(fileData));
+            }
+        } catch (e) {
+            // Log but don't fail - metadata update is best effort
+            const logger = (await import('./logger.js')).default;
+            logger.warn(`Failed to update file metadata in Redis: ${e.message}`);
+        }
+    }
 
     return fileEntry;
 }
@@ -872,96 +942,121 @@ async function syncFilesToCollection(chatHistory, contextId, contextKey = null)
         return await loadFileCollection(contextId, contextKey, true);
     }
 
-    // Use optimistic locking to sync files
-    const collection = await modifyFileCollectionWithLock(contextId, contextKey, (collection) => {
-        // Create a map of existing files by URL and hash for fast lookup
-        const existingFilesMap = new Map();
-        collection.forEach(file => {
-            if (file.url) {
-                existingFilesMap.set(file.url, file);
-            }
-            if (file.gcs) {
-                existingFilesMap.set(file.gcs, file);
-            }
-            if (file.hash) {
-                existingFilesMap.set(`hash:${file.hash}`, file);
+    // Sync files - check individually and update only what's needed (atomic operations)
+    try {
+        const redisClient = await getRedisClient();
+        if (!redisClient) {
+            // No Redis, return existing collection
+            return await loadFileCollection(contextId, contextKey, true);
+        }
+        
+        const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+        const existingFiles = await redisClient.hgetall(contextMapKey);
+        const existingByUrl = new Map();
+        const existingByGcs = new Map();
+        const existingByHash = new Map();
+        
+        // Build lookup maps from existing files
+        for (const [hash, dataStr] of Object.entries(existingFiles)) {
+            try {
+                const data = JSON.parse(dataStr);
+                if (data.url) existingByUrl.set(data.url, hash);
+                if (data.gcs) existingByGcs.set(data.gcs, hash);
+                if (hash) existingByHash.set(hash, hash);
+            } catch (e) {
+                // Skip invalid entries
             }
-        });
-
-        // Add new files that aren't already in the collection
+        }
+        
+        // Add/update files individually (atomic operations)
         for (const file of extractedFiles) {
-            // Check if file already exists by URL or hash
-            const existsByUrl = file.url && existingFilesMap.has(file.url);
-            const existsByGcs = file.gcs && existingFilesMap.has(file.gcs);
-            const existsByHash = file.hash && existingFilesMap.has(`hash:${file.hash}`);
+            const existsByUrl = file.url && existingByUrl.has(file.url);
+            const existsByGcs = file.gcs && existingByGcs.has(file.gcs);
+            const existsByHash = file.hash && existingByHash.has(file.hash);
             
-            if (!existsByUrl && !existsByGcs && !existsByHash) {
-                // New file - add to collection
-                // Determine MIME type from URL (preferring converted URL if available)
-                const mimeType = determineMimeTypeFromUrl(file.url, file.gcs, file.filename);
+            if (!existsByUrl && !existsByGcs && !existsByHash && file.hash) {
+                // File not found in context-scoped map - check if CFH has it (context-scoped or unscoped)
+                // This handles the case where file was uploaded but not yet in this context's collection
+                const existingDataStr = await redisClient.hget(contextMapKey, file.hash);
+                let existingData = null;
                 
-                // Ensure filename has correct extension based on MIME type
-                const correctedFilename = ensureFilenameExtension(file.filename, mimeType);
-                
-                const fileEntry = {
-                    id: `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
-                    url: file.url,
-                    gcs: file.gcs || null,
-                    filename: correctedFilename,
-                    mimeType: mimeType,
-                    hash: file.hash || null,
-                    type: file.type || 'file',
-                    addedDate: new Date().toISOString(),
-                    lastAccessed: new Date().toISOString()
-                };
-                
-                collection.push(fileEntry);
-                existingFilesMap.set(file.url, fileEntry);
-                if (file.gcs) {
-                    existingFilesMap.set(file.gcs, fileEntry);
+                if (existingDataStr) {
+                    try {
+                        existingData = JSON.parse(existingDataStr);
+                    } catch (e) {
+                        // Invalid data, treat as new
+                    }
                 }
-                if (file.hash) {
-                    existingFilesMap.set(`hash:${file.hash}`, fileEntry);
+                
+                // Also check unscoped map (CFH might have written it there)
+                if (!existingData) {
+                    const unscopedDataStr = await redisClient.hget("FileStoreMap", file.hash);
+                    if (unscopedDataStr) {
+                        try {
+                            existingData = JSON.parse(unscopedDataStr);
+                        } catch (e) {
+                            // Invalid data, treat as new
+                        }
+                    }
                 }
-            } else {
-                // File exists - update lastAccessed and merge URLs if needed
-                const existingFile = existsByUrl ? existingFilesMap.get(file.url) :
-                                    existsByGcs ? existingFilesMap.get(file.gcs) :
-                                    existingFilesMap.get(`hash:${file.hash}`);
                 
-                if (existingFile) {
-                    existingFile.lastAccessed = new Date().toISOString();
+                if (existingData) {
+                    // CFH already has this file - merge CFH data with Cortex metadata
+                    // Only set Cortex-managed fields (tags, notes, id, dates), preserve all CFH data
+                    // Ensure mimeType is set (CFH doesn't store it, so we need to determine it)
+                    const mimeType = existingData.mimeType || determineMimeTypeFromUrl(existingData.url, existingData.gcs, existingData.displayFilename);
                     
-                    // Merge or update URLs
-                    if (file.url && file.url !== existingFile.url) {
-                        existingFile.url = file.url;
-                    }
-                    if (file.gcs && file.gcs !== existingFile.gcs) {
-                        existingFile.gcs = file.gcs;
-                    }
+                    const fileData = {
+                        ...existingData, // Preserve all CFH data (url, gcs, filename, displayFilename, permanent, etc.)
+                        mimeType: mimeType, // Ensure mimeType is set
+                        id: existingData.id || `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
+                        tags: existingData.tags || [],
+                        notes: existingData.notes || '',
+                        addedDate: existingData.addedDate || existingData.timestamp || new Date().toISOString(),
+                        lastAccessed: new Date().toISOString()
+                    };
                     
-                    // If MIME type is missing, determine it from current URLs
-                    if (!existingFile.mimeType) {
-                        existingFile.mimeType = determineMimeTypeFromUrl(existingFile.url, existingFile.gcs, existingFile.filename);
-                    }
+                    await redisClient.hset(contextMapKey, file.hash, JSON.stringify(fileData));
+                } else {
+                    // File doesn't exist in CFH - create minimal entry (file referenced in chat but not uploaded)
+                    const mimeType = determineMimeTypeFromUrl(file.url, file.gcs, null);
                     
-                    // Update filename if provided, or ensure existing filename has correct extension
-                    const filenameToUse = file.filename || existingFile.filename;
-                    if (filenameToUse && existingFile.mimeType) {
-                        existingFile.filename = ensureFilenameExtension(filenameToUse, existingFile.mimeType);
-                    }
+                    const fileData = {
+                        url: file.url,
+                        gcs: file.gcs || null,
+                        mimeType: mimeType,
+                        id: `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
+                        tags: [],
+                        notes: '',
+                        hash: file.hash,
+                        permanent: false,
+                        addedDate: new Date().toISOString(),
+                        lastAccessed: new Date().toISOString()
+                    };
                     
-                    if (file.hash && !existingFile.hash) {
-                        existingFile.hash = file.hash;
-                    }
+                    await redisClient.hset(contextMapKey, file.hash, JSON.stringify(fileData));
                 }
+            } else if (file.hash) {
+                // File exists - update lastAccessed directly
+                await updateFileMetadata(contextId, file.hash, {
+                    lastAccessed: new Date().toISOString()
+                });
             }
         }
+        
+        // Invalidate cache
+        const cacheKey = getCollectionCacheKey(contextId, contextKey);
+        fileCollectionCache.delete(cacheKey);
+    } catch (e) {
+        // Fallback: log error and return existing collection
+        const logger = (await import('./logger.js')).default;
+        logger.warn(`Failed to sync files individually: ${e.message}`);
+        // Return existing collection on error
+        return await loadFileCollection(contextId, contextKey, true);
+    }
 
-        return collection;
-    });
-
-    return collection;
+    // Return updated collection
+    return await loadFileCollection(contextId, contextKey, false);
 }
 
 /**
@@ -1002,10 +1097,11 @@ function formatFilesForTemplate(collection) {
     const totalFiles = collection.length;
     const hasMore = totalFiles > 10;
 
-    // Format as compact one line per file: hash | filename | url | date | tags
+    // Format as compact one line per file: hash | displayFilename | url | date | tags
     const fileList = recentFiles.map((file) => {
         const hash = file.hash || '';
-        const filename = file.filename || 'Unnamed file';
+        // Fallback to filename if displayFilename is not set (for files uploaded before displayFilename was added)
+        const displayFilename = file.displayFilename || file.filename || 'Unnamed file';
         const url = file.url || '';
         const dateAdded = file.addedDate 
             ? new Date(file.addedDate).toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' })
@@ -1013,7 +1109,7 @@ function formatFilesForTemplate(collection) {
         const tags = Array.isArray(file.tags) && file.tags.length > 0 
             ? file.tags.join(',') 
             : '';
-        return `${hash} | ${filename} | ${url} | ${dateAdded}${tags ? ' | ' + tags : ''}`;
+        return `${hash} | ${displayFilename} | ${url} | ${dateAdded}${tags ? ' | ' + tags : ''}`;
     }).join('\n');
 
     let result = fileList;
@@ -1048,7 +1144,7 @@ async function getAvailableFiles(chatHistory, contextId, contextKey = null) {
 
 /**
  * Find a file in the collection by ID, URL, hash, or filename
- * First tries exact matches, then falls back to simple "contains" matches on filename, URL, and GCS
+ * First tries exact matches, then falls back to simple "contains" matches on displayFilename, filename, URL, and GCS
  * @param {string} fileParam - File ID, URL (Azure or GCS), hash, or filename
  * @param {Array} collection - File collection array
  * @returns {Object|null} File entry from collection, or null if not found
@@ -1086,7 +1182,16 @@ function findFileInCollection(fileParam, collection) {
             return file;
         }
         
-        // Check by exact filename (case-insensitive, using basename)
+        // Check by exact displayFilename (case-insensitive, using basename)
+        if (file.displayFilename) {
+            const normalizedDisplayFilename = file.displayFilename.toLowerCase();
+            const fileDisplayFilename = path.basename(normalizedDisplayFilename);
+            if (fileDisplayFilename === paramFilename) {
+                return file;
+            }
+        }
+        
+        // Also check by CFH-managed filename (case-insensitive, using basename)
         if (file.filename) {
             const normalizedFilename = file.filename.toLowerCase();
             const fileFilename = path.basename(normalizedFilename);
@@ -1096,11 +1201,19 @@ function findFileInCollection(fileParam, collection) {
         }
     }
 
-    // If no exact match, try simple "contains" matches on filename, url, and gcs
+    // If no exact match, try simple "contains" matches on displayFilename, filename, url, and gcs
     // Only match if parameter is at least 4 characters to avoid false matches
     if (normalizedParam.length >= 4) {
         for (const file of collection) {
-            // Check if filename contains the parameter
+            // Check if displayFilename contains the parameter
+            if (file.displayFilename) {
+                const normalizedDisplayFilename = file.displayFilename.toLowerCase();
+                if (normalizedDisplayFilename.includes(normalizedParam)) {
+                    return file;
+                }
+            }
+            
+            // Check if CFH-managed filename contains the parameter
             if (file.filename) {
                 const normalizedFilename = file.filename.toLowerCase();
                 if (normalizedFilename.includes(normalizedParam)) {
@@ -1216,7 +1329,6 @@ async function generateFileMessageContent(fileParam, contextId, contextKey = nul
         type: 'image_url',
         url: fileWithShortLivedUrl.url,
         gcs: fileWithShortLivedUrl.gcs || null,
-        originalFilename: fileWithShortLivedUrl.filename || null,
         hash: fileWithShortLivedUrl.hash || null
     };
 
@@ -1541,8 +1653,13 @@ async function uploadFileToCloud(fileInput, mimeType = null, filename = null, pa
         
         const requestId = uuidv4();
         const formData = new FormData();
+        
+        // Use the original filename if provided, otherwise fall back to temp file basename
+        // This preserves the friendly filename from the user's message
+        const uploadFilename = filename || path.basename(tempFilePath);
+        
         formData.append('file', fs.createReadStream(tempFilePath), {
-            filename: path.basename(tempFilePath),
+            filename: uploadFilename,
             contentType: mimeType || 'application/octet-stream'
         });
         // Add hash for deduplication if we computed it
@@ -1654,12 +1771,11 @@ async function resolveFileHashesToContent(fileHashes, config, contextId = null)
                     // - shortLivedUrl (prefers converted) in url field
                     // - GCS URL (prefers converted) in gcs field
                     // - filename in filename field
-                    return JSON.stringify({
-                        type: "image_url",
+                            return JSON.stringify({
+                                type: "image_url",
                         url: existingFile.url, // Already has shortLivedUrl (prefers converted)
                         image_url: { url: existingFile.url },
                         gcs: existingFile.gcs || null, // GCS URL (prefers converted, no short-lived)
-                        originalFilename: existingFile.filename || null, // Filename from single API call
                         hash: hash
                     });
                 }
@@ -1805,7 +1921,9 @@ export {
     addFileToCollection,
     loadFileCollection,
     saveFileCollection,
-    modifyFileCollectionWithLock,
+    updateFileMetadata,
+    getCollectionCacheKey,
+    getRedisClient,
     checkHashExists,
     ensureShortLivedUrl,
     uploadFileToCloud,
diff --git a/pathways/system/entity/files/sys_read_file_collection.js b/pathways/system/entity/files/sys_read_file_collection.js
new file mode 100644
index 00000000..b99e61e8
--- /dev/null
+++ b/pathways/system/entity/files/sys_read_file_collection.js
@@ -0,0 +1,43 @@
+// sys_read_file_collection.js
+// GraphQL pathway for reading file collections (replaces sys_read_memory with section: "memoryFiles")
+// Returns file collection as JSON array string for backward compatibility with Labeeb
+
+import { loadFileCollection } from '../../../../lib/fileUtils.js';
+
+export default {
+    inputParameters: {
+        contextId: ``,
+        contextKey: ``,
+        useCache: true
+    },
+    // No format field - returns String directly (like sys_read_memory)
+    model: 'oai-gpt4o',
+
+    resolver: async (_parent, args, _contextValue, _info) => {
+        const { contextId, contextKey = null, useCache = true } = args;
+        
+        // Validate that contextId is provided
+        if (!contextId) {
+            return JSON.stringify({ error: 'Context error' }, null, 2);
+        }
+        
+        try {
+            // Load file collection from Redis hash maps
+            const collection = await loadFileCollection(contextId, contextKey, useCache);
+            
+            // Return as JSON array string for backward compatibility with Labeeb
+            // Labeeb expects either: [] or { version: "...", files: [...] }
+            // Since we removed versioning, we just return the array directly
+            // Ensure we always return a valid JSON array (empty if no files)
+            const result = Array.isArray(collection) ? collection : [];
+            return JSON.stringify(result);
+        } catch (e) {
+            // Log error for debugging
+            const logger = (await import('../../../../lib/logger.js')).default;
+            logger.warn(`Error loading file collection for contextId ${contextId}: ${e.message}`);
+            // Return empty array on error for backward compatibility
+            return "[]";
+        }
+    }
+}
+
diff --git a/pathways/system/entity/files/sys_update_file_metadata.js b/pathways/system/entity/files/sys_update_file_metadata.js
new file mode 100644
index 00000000..27caa099
--- /dev/null
+++ b/pathways/system/entity/files/sys_update_file_metadata.js
@@ -0,0 +1,72 @@
+// sys_update_file_metadata.js
+// GraphQL pathway for updating file metadata (replaces sys_save_memory for renames and metadata updates)
+// Only updates Cortex-managed fields (displayFilename, tags, notes, etc.), not CFH fields (url, gcs, hash, filename)
+
+import { updateFileMetadata } from '../../../../lib/fileUtils.js';
+
+export default {
+    inputParameters: {
+        contextId: ``,
+        hash: ``,
+        displayFilename: { type: 'string' }, // Optional - no default
+        tags: { type: 'array', items: { type: 'string' } }, // Optional - no default
+        notes: { type: 'string' }, // Optional - no default
+        mimeType: { type: 'string' }, // Optional - no default
+        permanent: { type: 'boolean' } // Optional - no default
+    },
+    model: 'oai-gpt4o',
+    isMutation: true, // Declaratively mark this as a Mutation
+
+    resolver: async (_parent, args, _contextValue, _info) => {
+        const { contextId, hash, displayFilename, tags, notes, mimeType, permanent } = args;
+        
+        // Validate required parameters
+        if (!contextId || !hash) {
+            return JSON.stringify({ 
+                success: false, 
+                error: 'contextId and hash are required' 
+            });
+        }
+        
+        try {
+            // Build metadata object with only provided fields
+            const metadata = {};
+            if (displayFilename !== undefined && displayFilename !== null) {
+                metadata.displayFilename = displayFilename;
+            }
+            if (tags !== undefined && tags !== null) {
+                metadata.tags = Array.isArray(tags) ? tags : [];
+            }
+            if (notes !== undefined && notes !== null) {
+                metadata.notes = notes;
+            }
+            if (mimeType !== undefined && mimeType !== null) {
+                metadata.mimeType = mimeType;
+            }
+            if (permanent !== undefined && permanent !== null) {
+                metadata.permanent = Boolean(permanent);
+            }
+            
+            // Update metadata (only Cortex-managed fields)
+            const success = await updateFileMetadata(contextId, hash, metadata);
+            
+            if (success) {
+                return JSON.stringify({ 
+                    success: true,
+                    message: 'File metadata updated successfully'
+                });
+            } else {
+                return JSON.stringify({ 
+                    success: false, 
+                    error: 'Failed to update file metadata' 
+                });
+            }
+        } catch (e) {
+            return JSON.stringify({ 
+                success: false, 
+                error: e.message || 'Unknown error occurred' 
+            });
+        }
+    }
+}
+
diff --git a/pathways/system/entity/memory/sys_read_memory.js b/pathways/system/entity/memory/sys_read_memory.js
index 0300410d..e23c979f 100644
--- a/pathways/system/entity/memory/sys_read_memory.js
+++ b/pathways/system/entity/memory/sys_read_memory.js
@@ -99,31 +99,18 @@ export default {
             return savedContext.memoryContext || "";
         }
 
-        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser', 'memoryContext', 'memoryVersion', 'memoryFiles'];
-        // memoryFiles can only be accessed explicitly, not as part of memoryAll
+        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser', 'memoryContext', 'memoryVersion'];
         const allSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser', 'memoryContext', 'memoryVersion'];
 
         if (section !== 'memoryAll') {
             if (validSections.includes(section)) {
                 const content = (getvWithDoubleDecryption && (await getvWithDoubleDecryption(`${contextId}-${section}`, contextKey))) || "";
-                // memoryFiles is JSON, skip processing but ensure it's a string
-                if (section === 'memoryFiles') {
-                    if (!content) {
-                        return "[]";
-                    }
-                    // If content is already an object (from getvWithDoubleDecryption parsing), stringify it
-                    if (typeof content === 'object') {
-                        return JSON.stringify(content);
-                    }
-                    // Otherwise it's already a string, return as-is
-                    return content;
-                }
                 return processMemoryContent(content, options);
             }
             return "";
         }
 
-        // otherwise, read all sections (excluding memoryFiles) and return them as a JSON object
+        // otherwise, read all sections and return them as a JSON object
         const memoryContents = {};
         for (const section of allSections) {
             if (section === 'memoryContext') continue;
diff --git a/pathways/system/entity/memory/sys_save_memory.js b/pathways/system/entity/memory/sys_save_memory.js
index 611b22ce..00fbad1e 100644
--- a/pathways/system/entity/memory/sys_save_memory.js
+++ b/pathways/system/entity/memory/sys_save_memory.js
@@ -9,6 +9,7 @@ export default {
         contextKey: ``
     },
     model: 'oai-gpt4o',
+    isMutation: true, // Declaratively mark this as a Mutation
     resolver: async (_parent, args, _contextValue, _info) => {
         const { contextId, aiMemory, section = 'memoryAll', contextKey } = args;
 
@@ -29,29 +30,18 @@ export default {
             return aiMemory;
         }
 
-        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser', 'memoryVersion', 'memoryFiles'];
-        // memoryFiles can only be accessed explicitly, not as part of memoryAll
+        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser', 'memoryVersion'];
         const allSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser', 'memoryVersion'];
 
         // Handle single section save
         if (section !== 'memoryAll') {
             if (validSections.includes(section)) {
-                // memoryFiles should be JSON array, validate if provided
-                if (section === 'memoryFiles' && aiMemory && aiMemory.trim() !== '') {
-                    try {
-                        // Validate it's valid JSON (but keep as string for storage)
-                        JSON.parse(aiMemory);
-                    } catch (e) {
-                        // If not valid JSON, return error
-                        return JSON.stringify({ error: 'memoryFiles must be a valid JSON array' });
-                    }
-                }
                 await setvWithDoubleEncryption(`${contextId}-${section}`, aiMemory, contextKey);
             }
             return aiMemory;
         }
 
-        // if the aiMemory is an empty string, set all sections (excluding memoryFiles) to empty strings
+        // if the aiMemory is an empty string, set all sections to empty strings
         if (aiMemory.trim() === "") {
             for (const section of allSections) {
                 await setvWithDoubleEncryption(`${contextId}-${section}`, "", contextKey);
@@ -59,7 +49,7 @@ export default {
             return "";
         }
         
-        // Handle multi-section save (excluding memoryFiles)
+        // Handle multi-section save
         try {
             const memoryObject = JSON.parse(aiMemory);
             for (const section of allSections) {
@@ -67,10 +57,6 @@ export default {
                     await setvWithDoubleEncryption(`${contextId}-${section}`, memoryObject[section], contextKey);
                 }
             }
-            // Explicitly ignore memoryFiles if present in the object
-            if ('memoryFiles' in memoryObject) {
-                // Silently ignore - memoryFiles can only be saved explicitly
-            }
         } catch {
             for (const section of allSections) {
                 await setvWithDoubleEncryption(`${contextId}-${section}`, "", contextKey);
diff --git a/pathways/system/entity/tools/sys_tool_editfile.js b/pathways/system/entity/tools/sys_tool_editfile.js
index 721a4c0a..bb59129f 100644
--- a/pathways/system/entity/tools/sys_tool_editfile.js
+++ b/pathways/system/entity/tools/sys_tool_editfile.js
@@ -2,7 +2,7 @@
 // Entity tool that modifies existing files by replacing line ranges or exact string matches
 import logger from '../../../../lib/logger.js';
 import { axios } from '../../../../lib/requestExecutor.js';
-import { uploadFileToCloud, findFileInCollection, loadFileCollection, saveFileCollection, getMimeTypeFromFilename, resolveFileParameter, deleteFileByHash, modifyFileCollectionWithLock, isTextMimeType } from '../../../../lib/fileUtils.js';
+import { uploadFileToCloud, findFileInCollection, loadFileCollection, saveFileCollection, getMimeTypeFromFilename, resolveFileParameter, deleteFileByHash, isTextMimeType, updateFileMetadata } from '../../../../lib/fileUtils.js';
 
 export default {
     prompt: [],
@@ -315,29 +315,70 @@ export default {
                 throw new Error('Failed to upload modified file to cloud storage');
             }
 
-            // Update the file collection entry with new URL and hash using optimistic locking
-            // Capture the old hash INSIDE the lock to avoid race conditions with concurrent edits
-            let oldHashToDelete = null;
-            const updatedCollection = await modifyFileCollectionWithLock(contextId, contextKey, (collection) => {
-                const fileToUpdate = collection.find(f => f.id === fileIdToUpdate);
-                if (!fileToUpdate) {
-                    throw new Error(`File with ID "${fileIdToUpdate}" not found in collection during update`);
-                }
-                
-                // Capture the old hash BEFORE updating (this is the current hash at lock time)
-                oldHashToDelete = fileToUpdate.hash || null;
-                
-                fileToUpdate.url = uploadResult.url;
-                if (uploadResult.gcs) {
-                    fileToUpdate.gcs = uploadResult.gcs;
-                }
-                if (uploadResult.hash) {
-                    fileToUpdate.hash = uploadResult.hash;
+            // Update the file collection entry directly (atomic operation)
+            // First find the file to get its current hash
+            const currentCollection = await loadFileCollection(contextId, contextKey, false);
+            const fileToUpdate = currentCollection.find(f => f.id === fileIdToUpdate);
+            if (!fileToUpdate) {
+                throw new Error(`File with ID "${fileIdToUpdate}" not found in collection`);
+            }
+            
+            const oldHashToDelete = fileToUpdate.hash || null;
+            
+            // Write new entry with CFH data (url, gcs, hash) + Cortex metadata
+            // If hash changed, this creates a new entry; if same hash, it updates the existing one
+            if (uploadResult.hash) {
+                const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+                const redisClient = await getRedisClient();
+                if (redisClient) {
+                    const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+                    
+                    // Get existing CFH data for the new hash (if any)
+                    const existingDataStr = await redisClient.hget(contextMapKey, uploadResult.hash);
+                    let existingData = {};
+                    if (existingDataStr) {
+                        try {
+                            existingData = JSON.parse(existingDataStr);
+                        } catch (e) {
+                            existingData = {};
+                        }
+                    }
+                    
+                    // Merge CFH data (url, gcs, hash) with Cortex metadata
+                    const fileData = {
+                        ...existingData, // Preserve any existing CFH data
+                        // CFH-managed fields (from upload result)
+                        url: uploadResult.url,
+                        gcs: uploadResult.gcs || null,
+                        hash: uploadResult.hash,
+                        filename: uploadResult.filename || filename, // Use CFH filename if available, otherwise preserve
+                        // Cortex-managed metadata
+                        id: fileToUpdate.id, // Keep same ID
+                        tags: fileToUpdate.tags || [],
+                        notes: fileToUpdate.notes || '',
+                        mimeType: fileToUpdate.mimeType || mimeType || null,
+                        addedDate: fileToUpdate.addedDate, // Keep original added date
+                        lastAccessed: new Date().toISOString(),
+                        permanent: fileToUpdate.permanent || false
+                    };
+                    
+                    // Write new entry (atomic operation)
+                    await redisClient.hset(contextMapKey, uploadResult.hash, JSON.stringify(fileData));
+                    
+                    // If hash changed, remove old entry
+                    if (oldHashToDelete && oldHashToDelete !== uploadResult.hash) {
+                        await redisClient.hdel(contextMapKey, oldHashToDelete);
+                    }
+                    
+                    // Cache will expire naturally (5 second TTL) or can be invalidated by reloading collection
                 }
-                fileToUpdate.lastAccessed = new Date().toISOString();
-                
-                return collection;
-            });
+            } else if (fileToUpdate.hash) {
+                // Same hash, just update Cortex metadata (filename, lastAccessed)
+                await updateFileMetadata(contextId, fileToUpdate.hash, {
+                    filename: filename,
+                    lastAccessed: new Date().toISOString()
+                });
+            }
 
             // Now it is safe to delete the old file version (after lock succeeds)
             // This ensures we're deleting the correct hash even if concurrent edits occurred
@@ -357,6 +398,7 @@ export default {
             }
             
             // Get the updated file info for the result
+            const updatedCollection = await loadFileCollection(contextId, contextKey, false);
             const updatedFile = updatedCollection.find(f => f.id === fileIdToUpdate);
 
             // Build result message
diff --git a/pathways/system/entity/tools/sys_tool_file_collection.js b/pathways/system/entity/tools/sys_tool_file_collection.js
index 8558be67..52403160 100644
--- a/pathways/system/entity/tools/sys_tool_file_collection.js
+++ b/pathways/system/entity/tools/sys_tool_file_collection.js
@@ -1,8 +1,8 @@
 // sys_tool_file_collection.js
 // Tool pathway that manages user file collections (add, search, list files)
-// Uses memory system endpoints (memoryFiles section) for storage
+// Uses Redis hash maps (FileStoreMap:ctx:<contextId>) for storage
 import logger from '../../../../lib/logger.js';
-import { addFileToCollection, loadFileCollection, saveFileCollection, findFileInCollection, deleteFileByHash, modifyFileCollectionWithLock } from '../../../../lib/fileUtils.js';
+import { addFileToCollection, loadFileCollection, saveFileCollection, findFileInCollection, deleteFileByHash, updateFileMetadata } from '../../../../lib/fileUtils.js';
 
 export default {
     prompt: [],
@@ -206,51 +206,42 @@ export default {
                 const safeFilterTags = Array.isArray(filterTags) ? filterTags : [];
                 const queryLower = query.toLowerCase();
                 
-                // Use optimistic locking to update lastAccessed
-                await modifyFileCollectionWithLock(contextId, contextKey, (collection) => {
-                    // Find matching files and update their lastAccessed
-                    const fileIds = new Set();
-                    collection.forEach(file => {
-                        // Skip files without filename
-                        if (!file.filename) return;
-                        
-                        // Search in filename, tags, and notes
-                        const filenameMatch = file.filename.toLowerCase().includes(queryLower);
-                        const notesMatch = file.notes && file.notes.toLowerCase().includes(queryLower);
-                        const tagMatch = Array.isArray(file.tags) && file.tags.some(tag => tag.toLowerCase().includes(queryLower));
-                        
-                        const matchesQuery = filenameMatch || notesMatch || tagMatch;
-                        
-                        // Filter by tags if provided
-                        const matchesTags = safeFilterTags.length === 0 || 
-                            (Array.isArray(file.tags) && safeFilterTags.every(filterTag => 
-                                file.tags.some(tag => tag.toLowerCase() === filterTag.toLowerCase())
-                            ));
-                        
-                        if (matchesQuery && matchesTags) {
-                            fileIds.add(file.id);
-                        }
-                    });
+                // Update lastAccessed for matching files directly (atomic operations)
+                const allFiles = await loadFileCollection(contextId, contextKey, false);
+                const now = new Date().toISOString();
+                
+                // Find matching files and update lastAccessed directly
+                for (const file of allFiles) {
+                    if (!file.hash) continue;
                     
-                    // Update lastAccessed for found files
-                    collection.forEach(file => {
-                        if (fileIds.has(file.id)) {
-                            file.lastAccessed = new Date().toISOString();
-                        }
-                    });
+                    // Fallback to filename if displayFilename is not set (for files uploaded before displayFilename was added)
+                    const displayFilename = file.displayFilename || file.filename || '';
+                    const filenameMatch = displayFilename.toLowerCase().includes(queryLower);
+                    const notesMatch = file.notes && file.notes.toLowerCase().includes(queryLower);
+                    const tagMatch = Array.isArray(file.tags) && file.tags.some(tag => tag.toLowerCase().includes(queryLower));
+                    const matchesQuery = filenameMatch || notesMatch || tagMatch;
                     
-                    return collection;
-                });
+                    const matchesTags = safeFilterTags.length === 0 || 
+                        (Array.isArray(file.tags) && safeFilterTags.every(filterTag => 
+                            file.tags.some(tag => tag.toLowerCase() === filterTag.toLowerCase())
+                        ));
+                    
+                    if (matchesQuery && matchesTags) {
+                        // Update lastAccessed directly (atomic operation)
+                        await updateFileMetadata(contextId, file.hash, {
+                            lastAccessed: now
+                        });
+                    }
+                }
                 
                 // Reload collection to get results (after update)
-                const collection = await loadFileCollection(contextId, contextKey, false);
+                const updatedFiles = await loadFileCollection(contextId, contextKey, false);
                 
                 // Filter and sort results (for display only, not modifying)
-                let results = collection.filter(file => {
-                    // Skip files without filename
-                    if (!file.filename) return false;
+                let results = updatedFiles.filter(file => {
+                    const displayFilename = file.displayFilename || '';
                     
-                    const filenameMatch = file.filename.toLowerCase().includes(queryLower);
+                    const filenameMatch = displayFilename.toLowerCase().includes(queryLower);
                     const notesMatch = file.notes && file.notes.toLowerCase().includes(queryLower);
                     const tagMatch = Array.isArray(file.tags) && file.tags.some(tag => tag.toLowerCase().includes(queryLower));
                     
@@ -264,10 +255,13 @@ export default {
                     return matchesQuery && matchesTags;
                 });
 
-                // Sort by relevance (filename matches first, then by date)
+                // Sort by relevance (displayFilename matches first, then by date)
                 results.sort((a, b) => {
-                    const aFilenameMatch = a.filename && a.filename.toLowerCase().includes(queryLower);
-                    const bFilenameMatch = b.filename && b.filename.toLowerCase().includes(queryLower);
+                    // Fallback to filename if displayFilename is not set
+                    const aDisplayFilename = a.displayFilename || a.filename || '';
+                    const bDisplayFilename = b.displayFilename || b.filename || '';
+                    const aFilenameMatch = aDisplayFilename.toLowerCase().includes(queryLower);
+                    const bFilenameMatch = bDisplayFilename.toLowerCase().includes(queryLower);
                     if (aFilenameMatch && !bFilenameMatch) return -1;
                     if (!aFilenameMatch && bFilenameMatch) return 1;
                     return new Date(b.addedDate) - new Date(a.addedDate);
@@ -282,7 +276,7 @@ export default {
                     count: results.length,
                     files: results.map(f => ({
                         id: f.id,
-                        filename: f.filename,
+                        displayFilename: f.displayFilename || f.filename || null,
                         url: f.url,
                         gcs: f.gcs || null,
                         tags: f.tags,
@@ -326,7 +320,7 @@ export default {
                         if (!filesToRemove.some(f => f.id === foundFile.id)) {
                             filesToRemove.push({
                                 id: foundFile.id,
-                                filename: foundFile.filename,
+                                displayFilename: foundFile.displayFilename || foundFile.filename || null,
                                 hash: foundFile.hash || null
                             });
                         }
@@ -339,25 +333,37 @@ export default {
                     throw new Error(`No files found matching: ${notFoundFiles.join(', ')}`);
                 }
 
-                // Use optimistic locking to remove files from collection FIRST
-                // Capture hashes INSIDE the lock to avoid race conditions with concurrent edits
+                // Remove files directly from hash map (atomic operations)
+                // Load collection to get hashes, then delete entries directly
+                const allFiles = await loadFileCollection(contextId, contextKey, false);
                 const fileIdsToRemove = new Set(filesToRemove.map(f => f.id));
-                    const hashesToDelete = [];
-                const finalCollection = await modifyFileCollectionWithLock(contextId, contextKey, (collection) => {
-                    // Capture hashes and container info of files that will be removed (at current lock time)
-                    collection.forEach(file => {
-                        if (fileIdsToRemove.has(file.id) && file.hash) {
-                            hashesToDelete.push({
-                                hash: file.hash,
-                                filename: file.filename || 'unknown',
-                                permanent: file.permanent ?? false
-                            });
-                        }
-                    });
-                    
-                    // Remove files by ID
-                    return collection.filter(file => !fileIdsToRemove.has(file.id));
+                const hashesToDelete = [];
+                
+                // Collect hashes to delete (hash is always present - either actual hash or generated from URL)
+                allFiles.forEach(file => {
+                    if (fileIdsToRemove.has(file.id) && file.hash) {
+                        hashesToDelete.push({
+                            hash: file.hash,
+                            displayFilename: file.displayFilename || file.filename || 'unknown',
+                            permanent: file.permanent ?? false
+                        });
+                    }
                 });
+                
+                // Delete entries directly from hash map (atomic operations)
+                const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+                const redisClient = await getRedisClient();
+                if (redisClient) {
+                    const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+                    for (const fileInfo of hashesToDelete) {
+                        await redisClient.hdel(contextMapKey, fileInfo.hash);
+                    }
+                    
+                    // Invalidate cache
+                    const { getCollectionCacheKey } = await import('../../../../lib/fileUtils.js');
+                    const cacheKey = getCollectionCacheKey(contextId, contextKey);
+                    // Cache is in fileUtils, we'll let it expire naturally
+                }
 
                 // Delete files from cloud storage ASYNC (fire and forget, but log errors)
                 // We do this after updating collection so user gets fast response and files are "gone" from UI immediately
@@ -367,15 +373,15 @@ export default {
                     for (const fileInfo of hashesToDelete) {
                         // Skip deletion if file is marked as permanent
                         if (fileInfo.permanent) {
-                            logger.info(`Skipping cloud deletion for permanent file: ${fileInfo.filename} (hash: ${fileInfo.hash})`);
+                            logger.info(`Skipping cloud deletion for permanent file: ${fileInfo.displayFilename} (hash: ${fileInfo.hash})`);
                             continue;
                         }
                         
                         try {
-                            logger.info(`Deleting file from cloud storage: ${fileInfo.filename} (hash: ${fileInfo.hash})`);
+                            logger.info(`Deleting file from cloud storage: ${fileInfo.displayFilename} (hash: ${fileInfo.hash})`);
                             await deleteFileByHash(fileInfo.hash, resolver, contextId);
                         } catch (error) {
-                            logger.warn(`Failed to delete file ${fileInfo.filename} (hash: ${fileInfo.hash}) from cloud storage: ${error?.message || String(error)}`);
+                            logger.warn(`Failed to delete file ${fileInfo.displayFilename} (hash: ${fileInfo.hash}) from cloud storage: ${error?.message || String(error)}`);
                         }
                     }
                 })().catch(err => logger.error(`Async cloud deletion error: ${err}`));
@@ -383,6 +389,10 @@ export default {
                 removedCount = filesToRemove.length;
                 removedFiles = filesToRemove;
 
+                // Get remaining files count after deletion
+                const remainingCollection = await loadFileCollection(contextId, contextKey, false);
+                const remainingCount = remainingCollection.length;
+
                 // Build result message
                 let message = `${removedCount} file(s) removed from collection`;
                 
@@ -396,7 +406,7 @@ export default {
                 return JSON.stringify({
                     success: true,
                     removedCount: removedCount,
-                    remainingFiles: finalCollection.length,
+                    remainingFiles: remainingCount,
                     message: message,
                     removedFiles: removedFiles,
                     notFoundFiles: notFoundFiles.length > 0 ? notFoundFiles : undefined
@@ -422,7 +432,12 @@ export default {
                 if (sortBy === 'date') {
                     results.sort((a, b) => new Date(b.addedDate) - new Date(a.addedDate));
                 } else if (sortBy === 'filename') {
-                    results.sort((a, b) => a.filename.localeCompare(b.filename));
+                    results.sort((a, b) => {
+                        // Fallback to filename if displayFilename is not set
+                        const aDisplayFilename = a.displayFilename || a.filename || '';
+                        const bDisplayFilename = b.displayFilename || b.filename || '';
+                        return aDisplayFilename.localeCompare(bDisplayFilename);
+                    });
                 }
 
                 // Limit results
@@ -435,7 +450,7 @@ export default {
                     totalFiles: collection.length,
                     files: results.map(f => ({
                         id: f.id,
-                        filename: f.filename,
+                        displayFilename: f.displayFilename || f.filename || null,
                         url: f.url,
                         gcs: f.gcs || null,
                         tags: f.tags,
diff --git a/pathways/system/entity/tools/sys_tool_view_image.js b/pathways/system/entity/tools/sys_tool_view_image.js
index a4e60ac9..69b29b5c 100644
--- a/pathways/system/entity/tools/sys_tool_view_image.js
+++ b/pathways/system/entity/tools/sys_tool_view_image.js
@@ -78,7 +78,6 @@ export default {
                     url: fileWithShortLivedUrl.url,
                     gcs: fileWithShortLivedUrl.gcs,
                     image_url: { url: fileWithShortLivedUrl.url },
-                    originalFilename: fileWithShortLivedUrl.filename,
                     hash: fileWithShortLivedUrl.hash
                 });
 
diff --git a/pathways/system/rest_streaming/sys_claude_37_sonnet.js b/pathways/system/rest_streaming/sys_claude_37_sonnet.js
deleted file mode 100644
index 7d444c01..00000000
--- a/pathways/system/rest_streaming/sys_claude_37_sonnet.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// sys_claude_37_sonnet.js
-// override handler for claude-37-sonnet
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'claude-37-sonnet-vertex',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'claude-3.7-sonnet',
-} 
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_claude_41_opus.js b/pathways/system/rest_streaming/sys_claude_41_opus.js
deleted file mode 100644
index a45d9fa9..00000000
--- a/pathways/system/rest_streaming/sys_claude_41_opus.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// sys_claude_41_opus.js
-// override handler for claude-41-opus
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'claude-41-opus-vertex',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'claude-4.1-opus',
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_claude_4_sonnet.js b/pathways/system/rest_streaming/sys_claude_4_sonnet.js
deleted file mode 100644
index aa3c33ec..00000000
--- a/pathways/system/rest_streaming/sys_claude_4_sonnet.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// sys_claude_4_sonnet.js
-// override handler for claude-4-sonnet
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'claude-4-sonnet-vertex',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'claude-4-sonnet',
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_google_gemini_25_flash.js b/pathways/system/rest_streaming/sys_google_gemini_25_flash.js
deleted file mode 100644
index a1ba622d..00000000
--- a/pathways/system/rest_streaming/sys_google_gemini_25_flash.js
+++ /dev/null
@@ -1,25 +0,0 @@
-// sys_google_gemini_25_flash.js
-// override handler for gemini-flash-25-vision
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'gemini-flash-25-vision',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gemini-flash-25',
-    geminiSafetySettings: [{category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH'},
-                            {category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_ONLY_HIGH'},
-                            {category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_ONLY_HIGH'},
-                            {category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_ONLY_HIGH'}],
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_google_gemini_25_pro.js b/pathways/system/rest_streaming/sys_google_gemini_25_pro.js
deleted file mode 100644
index 49ea5307..00000000
--- a/pathways/system/rest_streaming/sys_google_gemini_25_pro.js
+++ /dev/null
@@ -1,25 +0,0 @@
-// sys_google_gemini_25_pro.js
-// override handler for gemini-pro-25-vision
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'gemini-pro-25-vision',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gemini-pro-25',
-    geminiSafetySettings: [{category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH'},
-                            {category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_ONLY_HIGH'},
-                            {category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_ONLY_HIGH'},
-                            {category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_ONLY_HIGH'}],
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_grok_4.js b/pathways/system/rest_streaming/sys_grok_4.js
deleted file mode 100644
index 6e783308..00000000
--- a/pathways/system/rest_streaming/sys_grok_4.js
+++ /dev/null
@@ -1,23 +0,0 @@
-// sys_grok_4.js
-// override handler for grok-4
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        stream: false,
-        search_parameters: '',
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'xai-grok-4',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'grok-4'
-} 
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_grok_4_fast_non_reasoning.js b/pathways/system/rest_streaming/sys_grok_4_fast_non_reasoning.js
deleted file mode 100644
index 77cc6a5f..00000000
--- a/pathways/system/rest_streaming/sys_grok_4_fast_non_reasoning.js
+++ /dev/null
@@ -1,23 +0,0 @@
-// sys_grok_4_fast_non_reasoning.js
-// override handler for grok-4-fast-non-reasoning
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        stream: false,
-        search_parameters: '',
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'xai-grok-4-fast-non-reasoning',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'grok-4-fast-non-reasoning'
-}
diff --git a/pathways/system/rest_streaming/sys_grok_4_fast_reasoning.js b/pathways/system/rest_streaming/sys_grok_4_fast_reasoning.js
deleted file mode 100644
index b2cfad26..00000000
--- a/pathways/system/rest_streaming/sys_grok_4_fast_reasoning.js
+++ /dev/null
@@ -1,23 +0,0 @@
-// sys_grok_4_fast_reasoning.js
-// override handler for grok-4-fast-reasoning
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        stream: false,
-        search_parameters: '',
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'xai-grok-4-fast-reasoning',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'grok-4-fast-reasoning'
-}
diff --git a/pathways/system/rest_streaming/sys_ollama_chat.js b/pathways/system/rest_streaming/sys_ollama_chat.js
deleted file mode 100644
index a35b9eb9..00000000
--- a/pathways/system/rest_streaming/sys_ollama_chat.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// sys_ollama_chat.js
-// override handler for ollama chat model
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{ role: '', content: '' }],
-        ollamaModel: '',
-    },
-    model: 'ollama-chat',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'ollama-chat',
-    timeout: 300,
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_ollama_completion.js b/pathways/system/rest_streaming/sys_ollama_completion.js
deleted file mode 100644
index ae316fe0..00000000
--- a/pathways/system/rest_streaming/sys_ollama_completion.js
+++ /dev/null
@@ -1,14 +0,0 @@
-// sys_ollama_completion.js
-// default handler for ollama completion endpoints when REST endpoints are enabled
-
-export default {
-    prompt: `{{text}}`,
-    inputParameters: {
-        text: '',
-        ollamaModel: '',
-    },
-    model: 'ollama-completion',
-    useInputChunking: false,
-    emulateOpenAICompletionModel: 'ollama-completion',
-    timeout: 300,
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_chat.js b/pathways/system/rest_streaming/sys_openai_chat.js
deleted file mode 100644
index 79141de9..00000000
--- a/pathways/system/rest_streaming/sys_openai_chat.js
+++ /dev/null
@@ -1,22 +0,0 @@
-// sys_openai_chat.js
-// override handler for gpt-3.5-turbo
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [],
-        tools: '',
-        tool_choice: 'auto',
-        functions: '',
-    },
-    model: 'oai-gpt4o',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gpt-4o',
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_chat_gpt41.js b/pathways/system/rest_streaming/sys_openai_chat_gpt41.js
deleted file mode 100644
index 59c1c565..00000000
--- a/pathways/system/rest_streaming/sys_openai_chat_gpt41.js
+++ /dev/null
@@ -1,22 +0,0 @@
-// sys_openai_chat_gpt41.js
-// override handler for gpt-41
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        functions: '',
-        tool_choice: 'auto',
-    },
-    model: 'oai-gpt41',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gpt-4.1',
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_chat_gpt41_mini.js b/pathways/system/rest_streaming/sys_openai_chat_gpt41_mini.js
deleted file mode 100644
index 7f3872d8..00000000
--- a/pathways/system/rest_streaming/sys_openai_chat_gpt41_mini.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// sys_openai_chat_gpt41_mini.js
-// override handler for gpt-41-mini
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'oai-gpt41-mini',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gpt-4.1-mini',
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_chat_gpt41_nano.js b/pathways/system/rest_streaming/sys_openai_chat_gpt41_nano.js
deleted file mode 100644
index 1227b74d..00000000
--- a/pathways/system/rest_streaming/sys_openai_chat_gpt41_nano.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// sys_openai_chat_gpt41_nano.js
-// override handler for gpt-41-nano
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'oai-gpt41-nano',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gpt-4.1-nano',
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_chat_gpt4_omni.js b/pathways/system/rest_streaming/sys_openai_chat_gpt4_omni.js
deleted file mode 100644
index 6fb0555b..00000000
--- a/pathways/system/rest_streaming/sys_openai_chat_gpt4_omni.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// sys_openai_chat_gpt4_omni.js
-// override handler for gpt-4-omni
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'oai-gpt4o',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gpt-4o',
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_chat_gpt4_omni_mini.js b/pathways/system/rest_streaming/sys_openai_chat_gpt4_omni_mini.js
deleted file mode 100644
index b7c8d7f1..00000000
--- a/pathways/system/rest_streaming/sys_openai_chat_gpt4_omni_mini.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// sys_openai_chat_gpt4_omni_mini.js
-// override handler for gpt-4-omni-mini
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'oai-gpt4o-mini',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gpt-4o-mini',
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_chat_gpt5.js b/pathways/system/rest_streaming/sys_openai_chat_gpt5.js
deleted file mode 100644
index f8e1c058..00000000
--- a/pathways/system/rest_streaming/sys_openai_chat_gpt5.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// sys_openai_chat_gpt5.js
-// override handler for gpt-5
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'oai-gpt5',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gpt-5',
-} 
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_chat_gpt5_chat.js b/pathways/system/rest_streaming/sys_openai_chat_gpt5_chat.js
deleted file mode 100644
index 41e91127..00000000
--- a/pathways/system/rest_streaming/sys_openai_chat_gpt5_chat.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// sys_openai_chat_gpt5_chat.js
-// override handler for gpt-5-chat
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'oai-gpt5-chat',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gpt-5-chat',
-} 
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_chat_gpt5_mini.js b/pathways/system/rest_streaming/sys_openai_chat_gpt5_mini.js
deleted file mode 100644
index 48ec4568..00000000
--- a/pathways/system/rest_streaming/sys_openai_chat_gpt5_mini.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// sys_openai_chat_gpt5_mini.js
-// override handler for gpt-5-mini
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'oai-gpt5-mini',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gpt-5-mini',
-} 
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_chat_gpt5_nano.js b/pathways/system/rest_streaming/sys_openai_chat_gpt5_nano.js
deleted file mode 100644
index d9e628c8..00000000
--- a/pathways/system/rest_streaming/sys_openai_chat_gpt5_nano.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// sys_openai_chat_gpt5_nano.js
-// override handler for gpt-5-nano
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'oai-gpt5-nano',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gpt-5-nano',
-} 
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_chat_o3.js b/pathways/system/rest_streaming/sys_openai_chat_o3.js
deleted file mode 100644
index 5481691b..00000000
--- a/pathways/system/rest_streaming/sys_openai_chat_o3.js
+++ /dev/null
@@ -1,22 +0,0 @@
-// sys_openai_chat_o3.js
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        functions: '',
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'oai-o3',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'o3',
-    enableDuplicateRequests: false,
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_chat_o3_mini.js b/pathways/system/rest_streaming/sys_openai_chat_o3_mini.js
deleted file mode 100644
index fd68850d..00000000
--- a/pathways/system/rest_streaming/sys_openai_chat_o3_mini.js
+++ /dev/null
@@ -1,22 +0,0 @@
-// sys_openai_chat_o3_mini.js
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [{role: '', content: []}],
-        functions: '',
-        tools: '',
-        tool_choice: 'auto',
-    },
-    model: 'oai-o3-mini',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'o3-mini',
-    enableDuplicateRequests: false,
-}
\ No newline at end of file
diff --git a/pathways/system/rest_streaming/sys_openai_completion.js b/pathways/system/rest_streaming/sys_openai_completion.js
deleted file mode 100644
index 4f4d0ba5..00000000
--- a/pathways/system/rest_streaming/sys_openai_completion.js
+++ /dev/null
@@ -1,9 +0,0 @@
-// sys_openai_completion.js
-// default handler for openAI completion endpoints when REST endpoints are enabled
-
-export default {
-    prompt: `{{text}}`,
-    model: 'oai-gpturbo',
-    useInputChunking: false,
-    emulateOpenAICompletionModel: '*',
-}
\ No newline at end of file
diff --git a/server/graphql.js b/server/graphql.js
index e5b81e3d..e1fc7f2f 100644
--- a/server/graphql.js
+++ b/server/graphql.js
@@ -109,14 +109,24 @@ const getTypedefs = (pathways, pathwayManager) => {
 
 // Resolvers for GraphQL
 const getResolvers = (config, pathways, pathwayManager) => {
-    const resolverFunctions = {};
+    const queryResolvers = {};
+    const mutationResolvers = {};
+    
     for (const [name, pathway] of Object.entries(pathways)) {
         if (pathway.disabled) continue;
-        resolverFunctions[name] = (parent, args, contextValue, info) => {
+        
+        const resolver = (parent, args, contextValue, info) => {
             // add shared state to contextValue
             contextValue.pathway = pathway;
             contextValue.config = config;
             return pathway.rootResolver(parent, args, contextValue, info);
+        };
+        
+        // Check if pathway is a mutation using the isMutation property
+        if (pathway.isMutation) {
+            mutationResolvers[name] = resolver;
+        } else {
+            queryResolvers[name] = resolver;
         }
     }
 
@@ -124,12 +134,13 @@ const getResolvers = (config, pathways, pathwayManager) => {
 
     const resolvers = {
         Query: {
-            ...resolverFunctions,
+            ...queryResolvers,
             executeWorkspace: (parent, args, contextValue, info) => 
                 executeWorkspaceResolver(parent, args, contextValue, info, config, pathwayManager)
         },
         Mutation: {
             'cancelRequest': cancelRequestResolver,
+            ...mutationResolvers,
             ...pathwayManagerResolvers.Mutation
         },
         Subscription: subscriptions,
diff --git a/server/typeDef.js b/server/typeDef.js
index 13bc823d..eabb9dc1 100644
--- a/server/typeDef.js
+++ b/server/typeDef.js
@@ -140,8 +140,8 @@ const getPathwayTypeDef = (name, returnType) => {
   }`
 };
 
-const getPathwayTypeDefAndExtendQuery = (pathway) => {
-  const { name, objName, defaultInputParameters, inputParameters, format } = pathway;
+const buildPathwayTypeDef = (pathway) => {
+  const { name, objName, defaultInputParameters, inputParameters, format, isMutation = false } = pathway;
 
   const fields = format ? format.match(/\b(\w+)\b/g) : null;
   const fieldsStr = !fields ? `` : fields.map((f) => `${f}: String`).join('\n    ');
@@ -160,7 +160,19 @@ const getPathwayTypeDefAndExtendQuery = (pathway) => {
 
   const paramsStr = Object.entries(params)
     .map(([key, value]) => {
+      // Handle undefined values - these become optional parameters without defaults
+      if (value === undefined) {
+        // For undefined, we can't infer type, so default to String
+        // Pathways should use JSON Schema objects for better type inference
+        return `${key}: String`;
+      }
+      
       const { type, defaultValue } = getGraphQlType(value);
+      // For mutations, never include defaults - make all optional parameters truly optional
+      // For queries, only omit defaults if defaultValue is undefined (from JSON Schema without defaults)
+      if (isMutation || defaultValue === undefined) {
+        return `${key}: ${type}`;
+      }
       return `${key}: ${type} = ${defaultValue}`;
     })
     .join('\n');
@@ -172,7 +184,8 @@ const getPathwayTypeDefAndExtendQuery = (pathway) => {
     };
   });
 
-  const gqlDefinition = `${type}\n\n${responseType}\n\nextend type Query {${name}${paramsStr ? `(${paramsStr})` : ''}: ${objName}}`;
+  const extendType = isMutation ? 'Mutation' : 'Query';
+  const gqlDefinition = `${type}\n\n${responseType}\n\nextend type ${extendType} {${name}${paramsStr ? `(${paramsStr})` : ''}: ${objName}}`;
 
   return {
     gqlDefinition,
@@ -181,7 +194,7 @@ const getPathwayTypeDefAndExtendQuery = (pathway) => {
 };
 
 const typeDef = (pathway) => {
-  return getPathwayTypeDefAndExtendQuery(pathway);
+  return buildPathwayTypeDef(pathway);
 };
 
 const userPathwayInputParameters = `text: String, promptNames: [String]`;
diff --git a/tests/integration/features/tools/fileCollection.test.js b/tests/integration/features/tools/fileCollection.test.js
index a4827ace..67b2c573 100644
--- a/tests/integration/features/tools/fileCollection.test.js
+++ b/tests/integration/features/tools/fileCollection.test.js
@@ -4,7 +4,7 @@
 import test from 'ava';
 import serverFactory from '../../../../index.js';
 import { callPathway } from '../../../../lib/pathwayTools.js';
-import { generateFileMessageContent, resolveFileParameter } from '../../../../lib/fileUtils.js';
+import { generateFileMessageContent, resolveFileParameter, loadFileCollection } from '../../../../lib/fileUtils.js';
 
 let testServer;
 
@@ -28,27 +28,15 @@ const createTestContext = () => {
     return contextId;
 };
 
-// Helper to extract files array from stored format (handles both old array format and new {version, files} format)
-const extractFilesFromStored = (stored) => {
-    if (!stored) return [];
-    const parsed = typeof stored === 'string' ? JSON.parse(stored) : stored;
-    // Handle new format: { version, files }
-    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed) && parsed.files) {
-        return Array.isArray(parsed.files) ? parsed.files : [];
-    }
-    // Handle old format: just an array
-    if (Array.isArray(parsed)) {
-        return parsed;
-    }
-    return [];
-};
-
 // Helper to clean up test data
 const cleanup = async (contextId, contextKey = null) => {
     try {
-        const { keyValueStorageClient } = await import('../../../../lib/keyValueStorageClient.js');
-        // Delete the key entirely instead of setting to empty array
-        await keyValueStorageClient.delete(`${contextId}-memoryFiles`);
+        const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+        const redisClient = await getRedisClient();
+        if (redisClient) {
+            const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+            await redisClient.del(contextMapKey);
+        }
     } catch (e) {
         // Ignore cleanup errors
     }
@@ -73,14 +61,10 @@ test('File collection: Add file to collection', async t => {
         t.truthy(parsed.fileId);
         t.true(parsed.message.includes('test.jpg'));
         
-        // Verify it was saved
-        const saved = await callPathway('sys_read_memory', {
-            contextId,
-            section: 'memoryFiles'
-        });
-        const collection = extractFilesFromStored(saved);
+        // Verify it was saved to Redis hash map
+        const collection = await loadFileCollection(contextId, null, false);
         t.is(collection.length, 1);
-        t.is(collection[0].filename, 'test.jpg');
+        t.is(collection[0].displayFilename, 'test.jpg');
         t.is(collection[0].url, 'https://example.com/test.jpg');
         t.is(collection[0].gcs, 'gs://bucket/test.jpg');
         t.deepEqual(collection[0].tags, ['photo', 'test']);
@@ -121,8 +105,8 @@ test('File collection: List files', async t => {
         t.is(parsed.count, 2);
         t.is(parsed.totalFiles, 2);
         t.is(parsed.files.length, 2);
-        t.true(parsed.files.some(f => f.filename === 'file1.jpg'));
-        t.true(parsed.files.some(f => f.filename === 'file2.pdf'));
+        t.true(parsed.files.some(f => f.displayFilename === 'file1.jpg'));
+        t.true(parsed.files.some(f => f.displayFilename === 'file2.pdf'));
     } finally {
         await cleanup(contextId);
     }
@@ -161,7 +145,7 @@ test('File collection: Search files', async t => {
         const parsed1 = JSON.parse(result1);
         t.is(parsed1.success, true);
         t.is(parsed1.count, 1);
-        t.is(parsed1.files[0].filename, 'report.pdf');
+        t.is(parsed1.files[0].displayFilename, 'report.pdf');
         
         // Search by tag
         const result2 = await callPathway('sys_tool_file_collection', {
@@ -173,7 +157,7 @@ test('File collection: Search files', async t => {
         const parsed2 = JSON.parse(result2);
         t.is(parsed2.success, true);
         t.is(parsed2.count, 1);
-        t.is(parsed2.files[0].filename, 'image.jpg');
+        t.is(parsed2.files[0].displayFilename, 'image.jpg');
         
         // Search by notes
         const result3 = await callPathway('sys_tool_file_collection', {
@@ -185,7 +169,7 @@ test('File collection: Search files', async t => {
         const parsed3 = JSON.parse(result3);
         t.is(parsed3.success, true);
         t.is(parsed3.count, 1);
-        t.is(parsed3.files[0].filename, 'image.jpg');
+        t.is(parsed3.files[0].displayFilename, 'image.jpg');
     } finally {
         await cleanup(contextId);
     }
@@ -223,7 +207,7 @@ test('File collection: Remove single file', async t => {
         t.is(parsed.removedCount, 1);
         t.is(parsed.remainingFiles, 1);
         t.is(parsed.removedFiles.length, 1);
-        t.is(parsed.removedFiles[0].filename, 'file1.jpg');
+        t.is(parsed.removedFiles[0].displayFilename, 'file1.jpg');
         t.true(parsed.message.includes('Cloud storage cleanup started in background'));
         
         // Verify it was removed
@@ -349,8 +333,8 @@ test('File collection: List with filters and sorting', async t => {
         });
         
         const parsed1 = JSON.parse(result1);
-        t.is(parsed1.files[0].filename, 'a_file.jpg');
-        t.is(parsed1.files[1].filename, 'z_file.pdf');
+        t.is(parsed1.files[0].displayFilename, 'a_file.jpg');
+        t.is(parsed1.files[1].displayFilename, 'z_file.pdf');
         
         // List filtered by tag
         const result2 = await callPathway('sys_tool_file_collection', {
@@ -361,26 +345,23 @@ test('File collection: List with filters and sorting', async t => {
         
         const parsed2 = JSON.parse(result2);
         t.is(parsed2.count, 1);
-        t.is(parsed2.files[0].filename, 'a_file.jpg');
+        t.is(parsed2.files[0].displayFilename, 'a_file.jpg');
     } finally {
         await cleanup(contextId);
     }
 });
 
-test('Memory system: memoryFiles excluded from memoryAll', async t => {
+test('Memory system: file collections excluded from memoryAll', async t => {
     const contextId = createTestContext();
     
     try {
-        // Save a file collection
-        await callPathway('sys_save_memory', {
-            contextId,
-            section: 'memoryFiles',
-            aiMemory: JSON.stringify([{
-                id: 'test-1',
-                url: 'https://example.com/test.jpg',
-                filename: 'test.jpg'
-            }])
-        });
+        // Save a file collection directly to Redis
+        const { saveFileCollection } = await import('../../../../lib/fileUtils.js');
+        await saveFileCollection(contextId, null, [{
+            id: 'test-1',
+            url: 'https://example.com/test.jpg',
+            displayFilename: 'test.jpg'
+        }]);
         
         // Save other memory
         await callPathway('sys_save_memory', {
@@ -389,7 +370,7 @@ test('Memory system: memoryFiles excluded from memoryAll', async t => {
             aiMemory: 'Test memory content'
         });
         
-        // Read all memory - should not include memoryFiles
+        // Read all memory - should not include file collections
         const allMemory = await callPathway('sys_read_memory', {
             contextId,
             section: 'memoryAll'
@@ -399,34 +380,26 @@ test('Memory system: memoryFiles excluded from memoryAll', async t => {
         t.truthy(parsed.memorySelf);
         t.falsy(parsed.memoryFiles);
         
-        // But should be accessible explicitly
-        const files = await callPathway('sys_read_memory', {
-            contextId,
-            section: 'memoryFiles'
-        });
-        
-        const filesParsed = JSON.parse(files);
-        t.is(filesParsed.length, 1);
-        t.is(filesParsed[0].filename, 'test.jpg');
+        // But should be accessible via loadFileCollection
+        const files = await loadFileCollection(contextId, null, false);
+        t.is(files.length, 1);
+        t.is(files[0].displayFilename, 'test.jpg');
     } finally {
         await cleanup(contextId);
     }
 });
 
-test('Memory system: memoryFiles not cleared by memoryAll clear', async t => {
+test('Memory system: file collections not cleared by memoryAll clear', async t => {
     const contextId = createTestContext();
     
     try {
-        // Save file collection
-        await callPathway('sys_save_memory', {
-            contextId,
-            section: 'memoryFiles',
-            aiMemory: JSON.stringify([{
-                id: 'test-1',
-                url: 'https://example.com/test.jpg',
-                filename: 'test.jpg'
-            }])
-        });
+        // Save file collection directly to Redis
+        const { saveFileCollection } = await import('../../../../lib/fileUtils.js');
+        await saveFileCollection(contextId, null, [{
+            id: 'test-1',
+            url: 'https://example.com/test.jpg',
+            displayFilename: 'test.jpg'
+        }]);
         
         // Clear all memory
         await callPathway('sys_save_memory', {
@@ -435,36 +408,28 @@ test('Memory system: memoryFiles not cleared by memoryAll clear', async t => {
             aiMemory: ''
         });
         
-        // Verify files are still there
-        const files = await callPathway('sys_read_memory', {
-            contextId,
-            section: 'memoryFiles'
-        });
-        
-        const filesParsed = JSON.parse(files);
-        t.is(filesParsed.length, 1);
-        t.is(filesParsed[0].filename, 'test.jpg');
+        // Verify files are still there (file collections are separate from memory system)
+        const files = await loadFileCollection(contextId, null, false);
+        t.is(files.length, 1);
+        t.is(files[0].displayFilename, 'test.jpg');
     } finally {
         await cleanup(contextId);
     }
 });
 
-test('Memory system: memoryFiles ignored in memoryAll save', async t => {
+test('Memory system: file collections ignored in memoryAll save', async t => {
     const contextId = createTestContext();
     
     try {
-        // Save file collection first
-        await callPathway('sys_save_memory', {
-            contextId,
-            section: 'memoryFiles',
-            aiMemory: JSON.stringify([{
-                id: 'original',
-                cloudUrl: 'https://example.com/original.jpg',
-                filename: 'original.jpg'
-            }])
-        });
-        
-        // Try to save all memory with memoryFiles included
+        // Save file collection first directly to Redis
+        const { saveFileCollection } = await import('../../../../lib/fileUtils.js');
+        await saveFileCollection(contextId, null, [{
+            id: 'original',
+            url: 'https://example.com/original.jpg',
+            displayFilename: 'original.jpg'
+        }]);
+        
+        // Try to save all memory with memoryFiles included (should be ignored)
         await callPathway('sys_save_memory', {
             contextId,
             section: 'memoryAll',
@@ -473,20 +438,15 @@ test('Memory system: memoryFiles ignored in memoryAll save', async t => {
                 memoryFiles: JSON.stringify([{
                     id: 'new',
                     url: 'https://example.com/new.jpg',
-                    filename: 'new.jpg'
+                    displayFilename: 'new.jpg'
                 }])
             })
         });
         
-        // Verify original files are still there (not overwritten)
-        const files = await callPathway('sys_read_memory', {
-            contextId,
-            section: 'memoryFiles'
-        });
-        
-        const filesParsed = JSON.parse(files);
-        t.is(filesParsed.length, 1);
-        t.is(filesParsed[0].filename, 'original.jpg');
+        // Verify original files are still there (not overwritten - memoryFiles is ignored)
+        const files = await loadFileCollection(contextId, null, false);
+        t.is(files.length, 1);
+        t.is(files[0].displayFilename, 'original.jpg');
     } finally {
         await cleanup(contextId);
     }
@@ -509,11 +469,7 @@ test('generateFileMessageContent should find file by ID', async t => {
         });
         
         // Get the file ID from the collection
-        const saved = await callPathway('sys_read_memory', {
-            contextId,
-            section: 'memoryFiles'
-        });
-        const collection = extractFilesFromStored(saved);
+        const collection = await loadFileCollection(contextId, null, false);
         const fileId = collection[0].id;
         
         // Normalize by ID
@@ -522,8 +478,10 @@ test('generateFileMessageContent should find file by ID', async t => {
         t.truthy(result);
         t.is(result.type, 'image_url');
         t.is(result.url, 'https://example.com/test.pdf');
-        t.is(result.gcs, 'gs://bucket/test.pdf');
-        t.is(result.originalFilename, 'test.pdf');
+        t.is(result.gcs, 'gs://bucket/test.pdf'); 
+        // originalFilename is no longer returned in message content objects
+        t.truthy(result.url);
+        t.truthy(result.hash);
     } finally {
         await cleanup(contextId);
     }
@@ -575,12 +533,16 @@ test('generateFileMessageContent should find file by fuzzy filename match', asyn
         // Normalize by partial filename
         const result1 = await generateFileMessageContent('document', contextId);
         t.truthy(result1);
-        t.is(result1.originalFilename, 'document.pdf');
+        // originalFilename is no longer returned in message content objects
+        t.truthy(result1.url);
+        t.truthy(result1.hash);
         
         // Normalize by full filename
         const result2 = await generateFileMessageContent('image.jpg', contextId);
         t.truthy(result2);
-        t.is(result2.originalFilename, 'image.jpg');
+        // originalFilename is no longer returned in message content objects
+        t.truthy(result2.url);
+        t.truthy(result2.hash);
     } finally {
         await cleanup(contextId);
     }
@@ -598,11 +560,7 @@ test('generateFileMessageContent should detect image type', async t => {
             userMessage: 'Add image'
         });
         
-        const saved = await callPathway('sys_read_memory', {
-            contextId,
-            section: 'memoryFiles'
-        });
-        const collection = extractFilesFromStored(saved);
+        const collection = await loadFileCollection(contextId, null, false);
         const fileId = collection[0].id;
         
         const result = await generateFileMessageContent(fileId, contextId);
diff --git a/tests/integration/features/tools/writefile.test.js b/tests/integration/features/tools/writefile.test.js
index ea0a21a2..4ed5206a 100644
--- a/tests/integration/features/tools/writefile.test.js
+++ b/tests/integration/features/tools/writefile.test.js
@@ -4,6 +4,7 @@
 import test from 'ava';
 import serverFactory from '../../../../index.js';
 import { callPathway } from '../../../../lib/pathwayTools.js';
+import { loadFileCollection } from '../../../../lib/fileUtils.js';
 
 let testServer;
 
@@ -27,27 +28,16 @@ const createTestContext = () => {
     return contextId;
 };
 
-// Helper to extract files array from stored format (handles both old array format and new {version, files} format)
-const extractFilesFromStored = (stored) => {
-    if (!stored) return [];
-    const parsed = typeof stored === 'string' ? JSON.parse(stored) : stored;
-    // Handle new format: { version, files }
-    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed) && parsed.files) {
-        return Array.isArray(parsed.files) ? parsed.files : [];
-    }
-    // Handle old format: just an array
-    if (Array.isArray(parsed)) {
-        return parsed;
-    }
-    return [];
-};
 
 // Helper to clean up test data
 const cleanup = async (contextId, contextKey = null) => {
     try {
-        const { keyValueStorageClient } = await import('../../../../lib/keyValueStorageClient.js');
-        // Delete the key entirely instead of setting to empty array
-        await keyValueStorageClient.delete(`${contextId}-memoryFiles`);
+        const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+        const redisClient = await getRedisClient();
+        if (redisClient) {
+            const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+            await redisClient.del(contextMapKey);
+        }
     } catch (e) {
         // Ignore cleanup errors
     }
@@ -83,11 +73,7 @@ test('WriteFile: Write and upload text file', async t => {
         t.true(parsed.message.includes('written and uploaded successfully'));
         
         // Verify it was added to file collection
-        const saved = await callPathway('sys_read_memory', {
-            contextId,
-            section: 'memoryFiles'
-        });
-        const collection = extractFilesFromStored(saved);
+        const collection = await loadFileCollection(contextId, null, false);
         t.is(collection.length, 1);
         t.is(collection[0].filename, filename);
         t.is(collection[0].url, parsed.url);
@@ -131,11 +117,7 @@ test('WriteFile: Write JSON file with tags and notes', async t => {
         t.is(parsed.size, Buffer.byteLength(content, 'utf8'));
         
         // Verify it was added to file collection with metadata
-        const saved = await callPathway('sys_read_memory', {
-            contextId,
-            section: 'memoryFiles'
-        });
-        const collection = extractFilesFromStored(saved);
+        const collection = await loadFileCollection(contextId, null, false);
         t.is(collection.length, 1);
         t.is(collection[0].filename, filename);
         t.deepEqual(collection[0].tags, tags);
@@ -247,11 +229,7 @@ test('WriteFile: Different file types and MIME types', async t => {
         }
         
         // Verify all files were added
-        const saved = await callPathway('sys_read_memory', {
-            contextId,
-            section: 'memoryFiles'
-        });
-        const collection = extractFilesFromStored(saved);
+        const collection = await loadFileCollection(contextId, null, false);
         t.is(collection.length, successCount);
     } finally {
         await cleanup(contextId);
@@ -333,16 +311,13 @@ test('WriteFile: Duplicate content (same hash)', async t => {
         t.is(parsed2.success, true);
         t.is(parsed2.hash, firstHash); // Should have same hash
         
-        // Both files should be in collection with different filenames but same hash
-        const saved = await callPathway('sys_read_memory', {
-            contextId,
-            section: 'memoryFiles'
-        });
-        const collection = extractFilesFromStored(saved);
-        t.is(collection.length, 2);
-        t.true(collection.some(f => f.filename === filename1));
-        t.true(collection.some(f => f.filename === filename2));
-        t.is(collection[0].hash, collection[1].hash); // Same hash
+        // Both files with same hash should result in one entry (same content, CFH will find it)
+        // The second file will update the existing entry with the new filename
+        const collection = await loadFileCollection(contextId, null, false);
+        t.is(collection.length, 1); // Same hash = one entry
+        t.is(collection[0].hash, firstHash); // Same hash
+        // The filename should be from the most recent write
+        t.is(collection[0].filename, filename2);
     } finally {
         await cleanup(contextId);
     }
diff --git a/tests/unit/core/fileCollection.test.js b/tests/unit/core/fileCollection.test.js
index 24509689..fff7e1d5 100644
--- a/tests/unit/core/fileCollection.test.js
+++ b/tests/unit/core/fileCollection.test.js
@@ -26,10 +26,9 @@ test('extractFilesFromChatHistory should extract files from array content', t =>
     t.is(files.length, 2);
     t.is(files[0].url, 'https://example.com/image.jpg');
     t.is(files[0].gcs, 'gs://bucket/image.jpg');
-    t.is(files[0].filename, 'image.jpg');
+    // filename is no longer extracted from messages (displayFilename is set by CFH on upload)
     t.is(files[1].url, 'https://example.com/doc.pdf');
     t.is(files[1].gcs, 'gs://bucket/doc.pdf');
-    t.is(files[1].filename, 'doc.pdf');
 });
 
 test('extractFilesFromChatHistory should extract files from string JSON content', t => {
@@ -109,7 +108,7 @@ test('formatFilesForTemplate should format files correctly', t => {
             id: 'file-1',
             url: 'https://example.com/image.jpg',
             gcs: 'gs://bucket/image.jpg',
-            filename: 'image.jpg',
+            displayFilename: 'image.jpg',
             hash: 'abc123',
             addedDate: '2024-01-01T00:00:00Z',
             lastAccessed: '2024-01-02T00:00:00Z',
@@ -119,7 +118,7 @@ test('formatFilesForTemplate should format files correctly', t => {
         {
             id: 'file-2',
             url: 'https://example.com/doc.pdf',
-            filename: 'doc.pdf',
+            displayFilename: 'doc.pdf',
             hash: 'def456',
             addedDate: '2024-01-02T00:00:00Z',
             lastAccessed: '2024-01-03T00:00:00Z'
@@ -130,7 +129,7 @@ test('formatFilesForTemplate should format files correctly', t => {
     // Should not include header or notes
     t.false(result.includes('Hash | Filename | URL | Date Added | Notes'));
     t.false(result.includes('Test image'));
-    // Should include hash, filename, url, date, and tags
+    // Should include hash, displayFilename, url, date, and tags
     t.true(result.includes('def456 | doc.pdf | https://example.com/doc.pdf'));
     t.true(result.includes('abc123 | image.jpg | https://example.com/image.jpg'));
     t.true(result.includes('photo')); // tags should be included
@@ -151,7 +150,7 @@ test('formatFilesForTemplate should handle files without optional fields', t =>
         {
             id: 'file-1',
             url: 'https://example.com/image.jpg',
-            filename: 'image.jpg',
+            displayFilename: 'image.jpg',
             addedDate: '2024-01-01T00:00:00Z'
         }
     ];
@@ -159,7 +158,7 @@ test('formatFilesForTemplate should handle files without optional fields', t =>
     const result = formatFilesForTemplate(collection);
     // Should not include header
     t.false(result.includes('Hash | Filename | URL | Date Added | Notes'));
-    // Should include filename, url, and date even without hash or tags
+    // Should include displayFilename, url, and date even without hash or tags
     t.true(result.includes('image.jpg'));
     t.true(result.includes('https://example.com/image.jpg'));
     // Date should be included (may be 2023 or 2024 due to timezone conversion)
@@ -171,7 +170,7 @@ test('formatFilesForTemplate should handle files without optional fields', t =>
 test('formatFilesForTemplate should limit to 10 files and show note', t => {
     const collection = Array.from({ length: 15 }, (_, i) => ({
         id: `file-${i}`,
-        filename: `file${i}.txt`,
+        displayFilename: `file${i}.txt`,
         hash: `hash${i}`,
         url: `https://example.com/file${i}.txt`,
         addedDate: `2024-01-${String(i + 1).padStart(2, '0')}T00:00:00Z`,
@@ -243,12 +242,12 @@ test('extractFilesFromChatHistory should handle files without gcsUrl', t => {
     t.is(files[0].gcs, null);
 });
 
-test('extractFilesFromChatHistory should extract filename from various fields', t => {
+test('extractFilesFromChatHistory should extract files without filename (filename no longer extracted from messages)', t => {
     const testCases = [
-        { originalFilename: 'file1.jpg', expected: 'file1.jpg' },
-        { name: 'file2.jpg', expected: 'file2.jpg' },
-        { filename: 'file3.jpg', expected: 'file3.jpg' },
-        { url: 'https://example.com/file4.jpg', expected: null } // Will extract from URL
+        { originalFilename: 'file1.jpg' },
+        { name: 'file2.jpg' },
+        { filename: 'file3.jpg' },
+        { url: 'https://example.com/file4.jpg' }
     ];
     
     testCases.forEach((testCase, index) => {
@@ -262,9 +261,9 @@ test('extractFilesFromChatHistory should extract filename from various fields',
         }];
         
         const files = extractFilesFromChatHistory(chatHistory);
-        if (testCase.expected) {
-            t.is(files[0].filename, testCase.expected, `Test case ${index} failed`);
-        }
+        // Files should be extracted but without filename (displayFilename is set by CFH on upload)
+        t.is(files.length, 1, `Test case ${index} should extract file`);
+        t.is(files[0].url, testCase.url || 'https://example.com/test.jpg');
     });
 });
 

From 548f8b505ae5527a55a7cd77fbafa708cf256543 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Wed, 17 Dec 2025 22:50:44 -0700
Subject: [PATCH 08/27] CR feedback

---
 helper-apps/cortex-file-handler/src/index.js        |  1 -
 helper-apps/cortex-file-handler/src/redis.js        | 13 +------------
 helper-apps/cortex-file-handler/src/start.js        |  2 +-
 .../cortex-file-handler/tests/setRetention.test.js  |  1 -
 lib/fileUtils.js                                    |  4 ++--
 lib/redisSubscription.js                            |  2 +-
 tests/unit/core/util.test.js                        |  1 -
 7 files changed, 5 insertions(+), 19 deletions(-)

diff --git a/helper-apps/cortex-file-handler/src/index.js b/helper-apps/cortex-file-handler/src/index.js
index 7b5cb4a1..397a57e1 100644
--- a/helper-apps/cortex-file-handler/src/index.js
+++ b/helper-apps/cortex-file-handler/src/index.js
@@ -80,7 +80,6 @@ async function CortexFileHandler(context, req) {
     load,
     restore,
     setRetention,
-    retention,
     contextId,
   } = source;
   // Container parameter is ignored - always uses default container from env var
diff --git a/helper-apps/cortex-file-handler/src/redis.js b/helper-apps/cortex-file-handler/src/redis.js
index 94347790..363d8a51 100644
--- a/helper-apps/cortex-file-handler/src/redis.js
+++ b/helper-apps/cortex-file-handler/src/redis.js
@@ -25,17 +25,6 @@ export const getScopedHashKey = (hash, contextId = null) => {
   return `${hash}:ctx:${contextId}`;
 };
 
-const tryParseCtxKey = (key) => {
-  if (!key || typeof key !== "string") return null;
-  const marker = ":ctx:";
-  const idx = key.indexOf(marker);
-  if (idx === -1) return null;
-  const hash = key.slice(0, idx);
-  const contextId = key.slice(idx + marker.length);
-  if (!hash || !contextId) return null;
-  return { hash, contextId };
-};
-
 const legacyContainerKey = (hash, containerName) => {
   if (!hash || !containerName) return null;
   return `${hash}:${containerName}`;
@@ -316,7 +305,7 @@ const getFileStoreMap = async (hash, skipLazyCleanup = false, contextId = null)
     // Backwards compatibility for unscoped keys only:
     // If unscoped hash doesn't exist, fall back to legacy hash+container key (if still present).
     // SECURITY: Context-scoped lookups NEVER fall back - they must match exactly.
-    if (!value && hash && !contextId) {
+    if (!value && !contextId) {
       const baseHash = hash;
 
       // Only allow fallback for unscoped keys (not context-scoped)
diff --git a/helper-apps/cortex-file-handler/src/start.js b/helper-apps/cortex-file-handler/src/start.js
index 91e03d59..e4383468 100644
--- a/helper-apps/cortex-file-handler/src/start.js
+++ b/helper-apps/cortex-file-handler/src/start.js
@@ -6,7 +6,7 @@ import cors from "cors";
 import { readFileSync } from "fs";
 
 import { publicIpv4 } from "public-ip";
-import { AZURE_STORAGE_CONTAINER_NAME, getDefaultContainerName } from "./blobHandler.js";
+import { AZURE_STORAGE_CONTAINER_NAME } from "./blobHandler.js";
 import { sanitizeForLogging } from "./utils/logSecurity.js";
 
 // When running under tests we want all generated URLs to resolve to the
diff --git a/helper-apps/cortex-file-handler/tests/setRetention.test.js b/helper-apps/cortex-file-handler/tests/setRetention.test.js
index 2f43215f..22cdf418 100644
--- a/helper-apps/cortex-file-handler/tests/setRetention.test.js
+++ b/helper-apps/cortex-file-handler/tests/setRetention.test.js
@@ -2,7 +2,6 @@ import test from "ava";
 import axios from "axios";
 import FormData from "form-data";
 import fs from "fs";
-import os from "os";
 import path from "path";
 import { fileURLToPath } from "url";
 import { v4 as uuidv4 } from "uuid";
diff --git a/lib/fileUtils.js b/lib/fileUtils.js
index 6a13c6f3..68e5d2c5 100644
--- a/lib/fileUtils.js
+++ b/lib/fileUtils.js
@@ -746,7 +746,7 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
     const correctedFilename = ensureFilenameExtension(filename, mimeType);
 
     // If no hash, generate one from URL for storage key (needed for Redis hash map)
-    const storageHash = finalHash || (finalUrl ? await computeBufferHash(Buffer.from(finalUrl)) : null);
+    const storageHash = finalHash || await computeBufferHash(Buffer.from(finalUrl));
     
     // Create file entry (before locking to avoid recreating on retry)
     const fileEntry = {
@@ -790,7 +790,7 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
                     ...existingData, // Preserve CFH data (url, gcs, filename, etc.)
                     // Update Cortex metadata (use new ID if this is a new entry, otherwise keep existing)
                     id: existingData.id || fileEntry.id,
-                    url: finalUrl || existingData.url, // Use new URL if provided, otherwise keep existing
+                    url: finalUrl, // Use new URL (guaranteed to be truthy at this point)
                     gcs: finalGcs || existingData.gcs || null, // Use new GCS if provided, otherwise keep existing
                     // Preserve CFH's filename (managed by CFH), store user-provided filename as displayFilename
                     displayFilename: correctedFilename, // Store user-provided filename as displayFilename
diff --git a/lib/redisSubscription.js b/lib/redisSubscription.js
index 4a476db1..ad714f25 100644
--- a/lib/redisSubscription.js
+++ b/lib/redisSubscription.js
@@ -37,7 +37,7 @@ if (connectionString) {
 
     logger.info(`Using Redis subscription for channel(s) ${requestProgressChannel}, ${requestProgressSubscriptionsChannel}`);
     try {
-        subscriptionClient = connectionString && new Redis(connectionString, redisOptions);
+        subscriptionClient = new Redis(connectionString, redisOptions);
         if (subscriptionClient) {
             subscriptionClient.on('connect', () => {
                 logger.info('Redis subscription client connected successfully');
diff --git a/tests/unit/core/util.test.js b/tests/unit/core/util.test.js
index b7b7a9a7..5da46517 100644
--- a/tests/unit/core/util.test.js
+++ b/tests/unit/core/util.test.js
@@ -8,7 +8,6 @@ import os from 'os';
 import sinon from 'sinon';
 import { removeOldImageAndFileContent } from '../../../lib/util.js';
 import { computeFileHash, computeBufferHash, generateFileMessageContent, injectFileIntoChatHistory } from '../../../lib/fileUtils.js';
-import { axios } from '../../../lib/requestExecutor.js';
 
 // Test removeOldImageAndFileContent function
 

From 0a436e15890e7908eca1d3d4c83cadc082f3c550 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Wed, 17 Dec 2025 22:59:56 -0700
Subject: [PATCH 09/27] feat: add comprehensive file system documentation

- Introduced a new `FILE_SYSTEM_DOCUMENTATION.md` detailing the architecture, file handler service, utilities layer, and file collection system.
- Included sections on key concepts, API endpoints, and error handling to enhance understanding of file operations.
- Updated `README.md` to reference the new documentation and provide an overview of file system capabilities.
- Enhanced tests to validate file collection and metadata management functionalities.
---
 FILE_SYSTEM_DOCUMENTATION.md                  | 1197 +++++++++++++++++
 README.md                                     |   75 +-
 .../features/tools/fileCollection.test.js     |  147 +-
 .../features/tools/fileOperations.test.js     |    8 +-
 .../features/tools/writefile.test.js          |   12 +-
 tests/unit/core/fileCollection.test.js        |   70 +
 6 files changed, 1498 insertions(+), 11 deletions(-)
 create mode 100644 FILE_SYSTEM_DOCUMENTATION.md

diff --git a/FILE_SYSTEM_DOCUMENTATION.md b/FILE_SYSTEM_DOCUMENTATION.md
new file mode 100644
index 00000000..a13c2ade
--- /dev/null
+++ b/FILE_SYSTEM_DOCUMENTATION.md
@@ -0,0 +1,1197 @@
+# Cortex File System - Complete Documentation
+
+## Table of Contents
+1. [Architecture Overview](#architecture-overview)
+2. [File Handler Service](#file-handler-service)
+3. [Cortex File Utilities Layer](#cortex-file-utilities-layer)
+4. [File Collection System](#file-collection-system)
+5. [Tools Integration](#tools-integration)
+6. [Data Flow Diagrams](#data-flow-diagrams)
+7. [Storage Layers](#storage-layers)
+8. [Key Concepts](#key-concepts)
+9. [Complete Function Reference](#complete-function-reference)
+10. [Error Handling](#error-handling)
+
+---
+
+## Architecture Overview
+
+The Cortex file system is a multi-layered architecture that handles file uploads, storage, retrieval, and management:
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                    Cortex Application                        │
+│                                                              │
+│  ┌──────────────────────────────────────────────────────┐   │
+│  │         System Tools & Plugins                       │   │
+│  │  (WriteFile, EditFile, Image, FileCollection, etc.)  │   │
+│  └──────────────────┬─────────────────────────────────┘   │
+│                      │                                       │
+│  ┌───────────────────▼─────────────────────────────────┐   │
+│  │         lib/fileUtils.js                             │   │
+│  │  (Encapsulated file handler interactions)            │   │
+│  └───────────────────┬─────────────────────────────────┘   │
+│                      │                                       │
+│  ┌───────────────────▼─────────────────────────────────┐   │
+│  │         File Collection System                       │   │
+│  │  (Redis hash maps: FileStoreMap:ctx:<contextId>)    │   │
+│  └───────────────────┬─────────────────────────────────┘   │
+└───────────────────────┼───────────────────────────────────────┘
+                        │
+                        │ HTTP/HTTPS
+                        │
+┌───────────────────────▼───────────────────────────────────────┐
+│         Cortex File Handler Service                           │
+│  (External Azure Function - cortex-file-handler)              │
+│                                                                │
+│  ┌──────────────┐  ┌──────────────┐  ┌──────────────┐       │
+│  │   Azure Blob │  │   GCS        │  │    Redis     │       │
+│  │   Storage    │  │   Storage    │  │   Metadata   │       │
+│  └──────────────┘  └──────────────┘  └──────────────┘       │
+└────────────────────────────────────────────────────────────────┘
+```
+
+### Key Components
+
+1. **File Handler Service** (`cortex-file-handler`): External Azure Function that handles actual file storage
+2. **File Utilities** (`lib/fileUtils.js`): Cortex's abstraction layer over the file handler
+3. **File Collection System**: Redis-based metadata storage for user file collections
+4. **System Tools**: Pathways that use files (WriteFile, EditFile, Image, etc.)
+
+---
+
+## File Handler Service
+
+The file handler is an external Azure Function service that manages file storage and processing.
+
+### Configuration
+- **URL**: Configured via `WHISPER_MEDIA_API_URL` environment variable
+- **Storage Backends**: Azure Blob Storage (primary), Google Cloud Storage (optional), Local (fallback)
+
+### Key Features
+
+#### 1. Single Container Architecture
+- All files stored in a single Azure Blob Storage container
+- Files distinguished by blob index tags, not separate containers
+- No `container` parameter supported - always uses configured container
+
+#### 2. Retention Management
+- **Temporary** (default): Files tagged with `retention=temporary`, auto-deleted after 30 days
+- **Permanent**: Files tagged with `retention=permanent`, retained indefinitely
+- Retention changed via `setRetention` operation (updates blob tag, no file copying)
+
+#### 3. Context Scoping
+- **`contextId`**: Optional parameter for per-user/per-context file isolation
+- Redis keys: `<hash>:ctx:<contextId>` for context-scoped files
+- Falls back to unscoped keys if context-scoped not found
+- **Strongly recommended** for multi-tenant applications
+
+#### 4. Hash-Based Deduplication
+- Files identified by xxhash64 hash
+- Duplicate uploads return existing file URLs
+- Hash stored in Redis for fast lookups
+
+#### 5. Short-Lived URLs
+- All operations return `shortLivedUrl` (5-minute expiration, configurable)
+- Provides secure, time-limited access
+- Preferred for LLM file access
+
+### API Endpoints
+
+#### POST `/file-handler` - Upload File
+```javascript
+// FormData:
+{
+  file: <FileStream>,
+  hash: "abc123",           // Optional: for deduplication
+  contextId: "user-456",    // Optional: for scoping
+  requestId: "req-789"      // Optional: for tracking
+}
+
+// Response:
+{
+  url: "https://storage.../file.pdf?long-lived-sas",
+  shortLivedUrl: "https://storage.../file.pdf?short-lived-sas",
+  gcs: "gs://bucket/file.pdf",  // If GCS configured
+  hash: "abc123",
+  filename: "file.pdf"
+}
+```
+
+#### GET `/file-handler` - Retrieve/Process File
+```javascript
+// Query Parameters:
+{
+  hash: "abc123",                    // Check if file exists
+  checkHash: true,                    // Enable hash check
+  contextId: "user-456",              // Optional: for scoping
+  shortLivedMinutes: 5,               // Optional: URL expiration
+  fetch: "https://example.com/file",  // Download from URL
+  save: true                          // Save converted document
+}
+
+// Response (checkHash):
+{
+  url: "https://storage.../file.pdf",
+  shortLivedUrl: "https://storage.../file.pdf?short-lived",
+  gcs: "gs://bucket/file.pdf",
+  hash: "abc123",
+  filename: "file.pdf",
+  converted: {                        // If file was converted
+    url: "https://storage.../converted.csv",
+    gcs: "gs://bucket/converted.csv"
+  }
+}
+```
+
+#### DELETE `/file-handler` - Delete File
+```javascript
+// Query Parameters:
+{
+  hash: "abc123",           // Delete by hash
+  contextId: "user-456",    // Optional: for scoping
+  requestId: "req-789"      // Or delete all files for requestId
+}
+```
+
+#### POST/PUT `/file-handler` - Set Retention
+```javascript
+// Body:
+{
+  hash: "abc123",
+  retention: "permanent",   // or "temporary"
+  contextId: "user-456",   // Optional: for scoping
+  setRetention: true
+}
+
+// Response:
+{
+  hash: "abc123",
+  filename: "file.pdf",
+  retention: "permanent",
+  url: "https://storage.../file.pdf",  // Same URL (tag updated)
+  shortLivedUrl: "https://storage.../file.pdf?new-sas",
+  gcs: "gs://bucket/file.pdf"
+}
+```
+
+---
+
+## Cortex File Utilities Layer
+
+**Location**: `lib/fileUtils.js`
+
+This is Cortex's abstraction layer that encapsulates all file handler interactions. **No direct axios calls to the file handler should exist** - all go through these functions.
+
+### Core Functions
+
+#### URL Building
+```javascript
+buildFileHandlerUrl(baseUrl, params)
+```
+- Handles separator detection (`?` vs `&`)
+- Properly encodes all parameters
+- Skips null/undefined/empty values
+- **Used by all file handler operations**
+
+#### File Upload
+```javascript
+uploadFileToCloud(fileInput, mimeType, filename, pathwayResolver, contextId)
+```
+- **Input Types**: URL string, base64 string, or Buffer
+- **Process**:
+  1. Converts input to Buffer
+  2. Computes xxhash64 hash
+  3. Checks if file exists via `checkHashExists` (deduplication)
+  4. If exists, returns existing URLs
+  5. If not, uploads via file handler POST
+- **Returns**: `{url, gcs, hash}`
+- **ContextId**: Passed in formData body (not URL)
+
+#### File Retrieval
+```javascript
+checkHashExists(hash, fileHandlerUrl, pathwayResolver, contextId, shortLivedMinutes)
+```
+- Checks if file exists by hash
+- Returns short-lived URL (prefers converted version)
+- **Returns**: `{url, gcs, hash, filename}` or `null`
+- Makes single API call (optimized)
+
+```javascript
+fetchFileFromUrl(fileUrl, requestId, contextId, save)
+```
+- Downloads file from URL via file handler
+- Processes based on file type
+- **Used by**: `azureVideoTranslatePlugin`, `azureCognitivePlugin`
+
+#### File Deletion
+```javascript
+deleteFileByHash(hash, pathwayResolver, contextId)
+```
+- Deletes file from cloud storage
+- Handles 404 gracefully (file already deleted)
+- **Returns**: `true` if deleted, `false` if not found
+
+#### Retention Management
+```javascript
+setRetentionForHash(hash, retention, contextId, pathwayResolver)
+```
+- Sets file retention to `'temporary'` or `'permanent'`
+- Best-effort operation (logs warnings on failure)
+- **Used by**: `addFileToCollection` when `permanent=true`
+
+#### Short-Lived URL Resolution
+```javascript
+ensureShortLivedUrl(fileObject, fileHandlerUrl, contextId, shortLivedMinutes)
+```
+- Resolves file object to use short-lived URL
+- Updates GCS URL if converted version exists
+- **Used by**: Tools that send files to LLMs
+
+#### Media Chunks
+```javascript
+getMediaChunks(file, requestId, contextId)
+```
+- Gets chunked media file URLs
+- **Used by**: Media processing workflows
+
+#### Cleanup
+```javascript
+markCompletedForCleanUp(requestId, contextId)
+```
+- Marks request as completed for cleanup
+- **Used by**: `azureCognitivePlugin`
+
+---
+
+## File Collection System
+
+**Location**: `lib/fileUtils.js` + `pathways/system/entity/tools/sys_tool_file_collection.js`
+
+The file collection system stores file metadata in Redis hash maps using atomic operations for concurrent safety. Files are stored directly in Redis hash maps keyed by hash, with context-scoped isolation.
+
+### Storage Architecture
+
+```
+Redis Hash Maps
+└── FileStoreMap:ctx:<contextId>
+    └── Hash Map (hash → fileData JSON)
+        └── File Entry (JSON):
+            {
+              // CFH-managed fields (preserved from file handler)
+              url: "https://storage.../file.pdf",
+              gcs: "gs://bucket/file.pdf",
+              filename: "uuid-based-filename.pdf",  // CFH-managed
+              
+              // Cortex-managed fields (user metadata)
+              id: "timestamp-random",
+              displayFilename: "user-friendly-name.pdf",  // User-provided name
+              mimeType: "application/pdf",
+              tags: ["pdf", "report"],
+              notes: "Quarterly report",
+              hash: "abc123",
+              permanent: true,
+              addedDate: "2024-01-15T10:00:00.000Z",
+              lastAccessed: "2024-01-15T10:00:00.000Z"
+            }
+```
+
+### Key Features
+
+#### 1. Atomic Operations
+- Uses Redis hash map operations (HSET, HGET, HDEL) which are atomic
+- No version-based locking needed - Redis operations are thread-safe
+- Direct hash map access: `FileStoreMap:ctx:<contextId>` → `{hash: fileData}`
+
+#### 2. Caching
+- In-memory cache with 5-second TTL
+- Reduces Redis load for read operations
+- Cache invalidated on writes
+
+#### 3. Field Ownership
+- **CFH-managed fields**: `url`, `gcs`, `filename` (UUID-based, managed by file handler)
+- **Cortex-managed fields**: `id`, `displayFilename`, `tags`, `notes`, `mimeType`, `permanent`, `addedDate`, `lastAccessed`
+- When merging data, CFH fields are preserved, Cortex fields are updated
+
+### Core Functions
+
+#### Loading
+```javascript
+loadFileCollection(contextId, contextKey, useCache)
+```
+- Loads collection from Redis hash map `FileStoreMap:ctx:<contextId>`
+- Returns array of file entries (sorted by lastAccessed, most recent first)
+- Uses cache if available and fresh (5-second TTL)
+- Converts hash map entries to array format
+
+#### Saving
+```javascript
+saveFileCollection(contextId, contextKey, collection)
+```
+- Saves collection to Redis hash map (only updates changed entries)
+- Uses atomic HSET operations per file
+- Optimized to only write files that actually changed
+- Returns `true` if successful, `false` on error
+
+#### Metadata Updates
+```javascript
+updateFileMetadata(contextId, hash, metadata)
+```
+- Updates Cortex-managed metadata fields atomically
+- Preserves all CFH-managed fields
+- Updates only specified fields (displayFilename, tags, notes, mimeType, dates, permanent)
+- **Used for**: Updating lastAccessed, modifying tags/notes without full reload
+
+#### Adding Files
+```javascript
+addFileToCollection(contextId, contextKey, url, gcs, filename, tags, notes, hash, fileUrl, pathwayResolver, permanent)
+```
+- Adds file entry to collection via atomic HSET operation
+- If `fileUrl` provided, uploads file first via `uploadFileToCloud()`
+- If `permanent=true`, sets retention to permanent via `setRetentionForHash()`
+- Merges with existing CFH data if file with same hash already exists
+- Returns file entry object with `id`
+
+#### Syncing from Chat History
+```javascript
+syncFilesToCollection(chatHistory, contextId, contextKey)
+```
+- Extracts files from chat history messages
+- Adds/updates files in collection individually (atomic operations)
+- Checks for existing files by URL, GCS, or hash
+- Updates lastAccessed for existing files
+- **Used by**: `getAvailableFiles()` to sync files from conversation
+
+### File Entry Schema
+
+```javascript
+{
+  id: string,                    // Unique ID: "timestamp-random" (Cortex-managed)
+  url: string,                   // Azure Blob Storage URL (CFH-managed)
+  gcs: string | null,            // Google Cloud Storage URL (CFH-managed)
+  filename: string | null,       // CFH-managed filename (UUID-based) (CFH-managed)
+  displayFilename: string | null, // User-friendly filename (Cortex-managed)
+  mimeType: string | null,      // MIME type (Cortex-managed)
+  tags: string[],               // Searchable tags (Cortex-managed)
+  notes: string,                // User notes/description (Cortex-managed)
+  hash: string,                 // File hash for deduplication (used as Redis key)
+  permanent: boolean,           // Whether file is permanent (Cortex-managed)
+  addedDate: string,            // ISO timestamp when added (Cortex-managed)
+  lastAccessed: string          // ISO timestamp of last access (Cortex-managed)
+}
+```
+
+**Field Ownership Notes**:
+- `filename`: Managed by CFH, UUID-based storage filename
+- `displayFilename`: Managed by Cortex, user-provided friendly name
+- When displaying files, prefer `displayFilename` with fallback to `filename`
+
+---
+
+## Tools Integration
+
+### System Tools That Use Files
+
+#### 1. WriteFile (`sys_tool_writefile.js`)
+**Flow**:
+1. User provides content and filename
+2. Creates Buffer from content
+3. Calls `uploadFileToCloud()` with `contextId`
+4. Calls `addFileToCollection()` with `permanent=true`
+5. Returns file info with `fileId`
+
+**Key Code**:
+```javascript
+const uploadResult = await uploadFileToCloud(
+    fileBuffer, mimeType, filename, resolver, contextId
+);
+const fileEntry = await addFileToCollection(
+    contextId, contextKey, uploadResult.url, uploadResult.gcs,
+    filename, tags, notes, uploadResult.hash, null, resolver, true
+);
+```
+
+#### 2. EditFile (`sys_tool_editfile.js`)
+**Flow**:
+1. User provides file identifier and modification
+2. Resolves file via `resolveFileParameter()` → finds in collection
+3. Downloads file content via `axios.get(file.url)`
+4. Modifies content (line replacement or search/replace)
+5. Uploads modified file via `uploadFileToCloud()` (creates new hash)
+6. Updates collection entry atomically via `updateFileMetadata()` with new URL/hash
+7. Deletes old file version (if not permanent) via `deleteFileByHash()`
+
+**Key Code**:
+```javascript
+const foundFile = await resolveFileParameter(fileParam, contextId, contextKey);
+const oldHash = foundFile.hash;
+const uploadResult = await uploadFileToCloud(
+    fileBuffer, mimeType, filename, resolver, contextId
+);
+// Update file entry atomically (preserves CFH data, updates Cortex metadata)
+await updateFileMetadata(contextId, foundFile.hash, {
+    url: uploadResult.url,
+    gcs: uploadResult.gcs,
+    hash: uploadResult.hash
+});
+if (!foundFile.permanent) {
+    await deleteFileByHash(oldHash, resolver, contextId);
+}
+```
+
+#### 3. FileCollection (`sys_tool_file_collection.js`)
+**Tools**:
+- `AddFileToCollection`: Adds file to collection (with optional upload)
+- `SearchFileCollection`: Searches files by filename, tags, notes
+- `ListFileCollection`: Lists all files with filtering/sorting
+- `RemoveFileFromCollection`: Removes files (deletes from cloud if not permanent)
+
+**Key Code**:
+```javascript
+// Add file
+await addFileToCollection(contextId, contextKey, url, gcs, filename, tags, notes, hash, fileUrl, resolver, permanent);
+
+// Remove file (with permanent check)
+if (!fileInfo.permanent) {
+    await deleteFileByHash(fileInfo.hash, resolver, contextId);
+}
+```
+
+#### 4. Image Tools (`sys_tool_image.js`, `sys_tool_image_gemini.js`)
+**Flow**:
+1. Generates/modifies image
+2. Gets image URL
+3. Uploads via `uploadFileToCloud()`
+4. Adds to collection with `permanent=true`
+
+#### 5. ReadFile (`sys_tool_readfile.js`)
+**Flow**:
+1. Resolves file via `resolveFileParameter()` → finds in collection
+2. Downloads file content via `axios.get(file.url)`
+3. Validates file is text-based via `isTextMimeType()`
+4. Returns content with line/character range support
+
+#### 6. ViewImage (`sys_tool_view_image.js`)
+**Flow**:
+1. Finds file in collection
+2. Resolves to short-lived URL via `ensureShortLivedUrl()`
+3. Returns image URL for display
+
+#### 7. AnalyzeFile (`sys_tool_analyzefile.js`)
+**Flow**:
+1. Extracts files from chat history via `extractFilesFromChatHistory()`
+2. Generates file message content via `generateFileMessageContent()`
+3. Injects files into chat history via `injectFileIntoChatHistory()`
+4. Uses Gemini Vision model to analyze files
+
+### Plugins That Use Files
+
+#### 1. AzureVideoTranslatePlugin
+**Flow**:
+1. Receives video URL
+2. If not from Azure storage, uploads via `fetchFileFromUrl()`
+3. Uses uploaded URL for video translation
+
+**Key Code**:
+```javascript
+const response = await fetchFileFromUrl(videoUrl, this.requestId, contextId, false);
+const resultUrl = Array.isArray(response) ? response[0] : response.url;
+```
+
+#### 2. AzureCognitivePlugin
+**Flow**:
+1. Receives file for indexing
+2. If not text file, converts via `fetchFileFromUrl()` with `save=true`
+3. Uses converted text file for indexing
+4. Marks completed via `markCompletedForCleanUp()`
+
+**Key Code**:
+```javascript
+const data = await fetchFileFromUrl(file, requestId, contextId, true);
+url = Array.isArray(data) ? data[0] : data.url;
+```
+
+---
+
+## Data Flow Diagrams
+
+### File Upload Flow
+
+```
+User/LLM Request
+    │
+    ▼
+System Tool (WriteFile, Image, etc.)
+    │
+    ▼
+uploadFileToCloud()
+    │
+    ├─► Convert input to Buffer
+    ├─► Compute xxhash64 hash
+    ├─► checkHashExists() ──► File Handler GET /file-handler?checkHash=true
+    │                           │
+    │                           ├─► File exists? ──► Return existing URLs
+    │                           │
+    │                           └─► File not found ──► Continue
+    │
+    └─► Upload via POST ──► File Handler POST /file-handler
+        │                      │
+        │                      ├─► Store in Azure Blob Storage
+        │                      ├─► Store in GCS (if configured)
+        │                      ├─► Store metadata in Redis
+        │                      └─► Return {url, gcs, hash, shortLivedUrl}
+        │
+        └─► addFileToCollection()
+            │
+            ├─► If permanent=true ──► setRetentionForHash() ──► File Handler POST /file-handler?setRetention=true
+            │
+            └─► Save to Redis hash map (atomic operation)
+                │
+                └─► Redis HSET FileStoreMap:ctx:<contextId> <hash> <fileData>
+                    │
+                    ├─► Merge with existing CFH data (if hash exists)
+                    ├─► Preserve CFH fields (url, gcs, filename)
+                    └─► Update Cortex fields (displayFilename, tags, notes, etc.)
+```
+
+### File Retrieval Flow
+
+```
+User/LLM Request (e.g., "view file.pdf")
+    │
+    ▼
+System Tool (ViewImage, ReadFile, etc.)
+    │
+    ▼
+resolveFileParameter()
+    │
+    ├─► Find in collection via findFileInCollection()
+    │   │
+    │   └─► Matches by: ID, filename, hash, URL, or fuzzy filename
+    │
+    └─► ensureShortLivedUrl()
+        │
+        └─► checkHashExists() ──► File Handler GET /file-handler?checkHash=true&shortLivedMinutes=5
+            │                      │
+            │                      ├─► Check Redis for hash metadata
+            │                      ├─► Generate short-lived SAS token
+            │                      └─► Return {url, gcs, hash, filename, shortLivedUrl}
+            │
+            └─► Return file object with shortLivedUrl
+```
+
+### File Edit Flow
+
+```
+User/LLM Request (e.g., "edit file.txt, replace line 5")
+    │
+    ▼
+EditFile Tool
+    │
+    ├─► resolveFileParameter() ──► Find file in collection
+    │
+    ├─► Download file content ──► axios.get(file.url)
+    │
+    ├─► Modify content (line replacement or search/replace)
+    │
+    ├─► uploadFileToCloud() ──► Upload modified file
+    │   │
+    │   └─► Returns new {url, gcs, hash}
+    │
+    └─► updateFileMetadata() ──► Redis HSET (atomic update)
+        │
+        ├─► Preserve CFH fields (url, gcs, filename)
+        ├─► Update Cortex fields (url, gcs, hash)
+        └─► If update succeeds:
+            └─► Delete old file (if not permanent)
+                └─► deleteFileByHash() ──► File Handler DELETE /file-handler?hash=oldHash
+```
+
+### File Deletion Flow
+
+```
+User/LLM Request (e.g., "remove file.pdf from collection")
+    │
+    ▼
+RemoveFileFromCollection Tool
+    │
+    ├─► Load collection ──► findFileInCollection() for each fileId
+    │
+    ├─► Capture file info (hash, permanent) from collection
+    │
+    └─► Redis HDEL FileStoreMap:ctx:<contextId> <hash> (atomic deletion)
+        │
+        └─► Async deletion (fire and forget)
+            │
+            ├─► For each file:
+            │   │
+            │   ├─► If permanent=true ──► Skip deletion (keep in cloud)
+            │   │
+            │   └─► If permanent=false ──► deleteFileByHash()
+            │       │
+            │       └─► File Handler DELETE /file-handler?hash=hash&contextId=contextId
+            │           │
+            │           ├─► Delete from Azure Blob Storage
+            │           ├─► Delete from GCS (if configured)
+            │           └─► Remove from Redis metadata
+```
+
+---
+
+## Storage Layers
+
+### Layer 1: Cloud Storage (File Handler)
+
+#### Azure Blob Storage (Primary)
+- **Container**: Single container (configured via `AZURE_STORAGE_CONTAINER_NAME`)
+- **Naming**: UUID-based filenames
+- **Organization**: By `requestId` folders
+- **Access**: SAS tokens (long-lived and short-lived)
+- **Tags**: Blob index tags for retention (`retention=temporary` or `retention=permanent`)
+- **Lifecycle**: Azure automatically deletes `retention=temporary` files after 30 days
+
+#### Google Cloud Storage (Optional)
+- **Enabled**: If `GCP_SERVICE_ACCOUNT_KEY` configured
+- **URL Format**: `gs://bucket/path`
+- **Usage**: Media file chunks, converted files
+- **No short-lived URLs**: GCS URLs are permanent (no SAS equivalent)
+
+#### Local Storage (Fallback)
+- **Used**: If Azure not configured
+- **Served**: Via HTTP on configured port
+
+### Layer 2: Redis Metadata (File Handler)
+
+**Purpose**: Fast hash lookups, file metadata caching
+
+**Key Format**:
+- Unscoped: `<hash>`
+- Context-scoped: `<hash>:ctx:<contextId>`
+- Legacy (migrated): `<hash>:<containerName>` (auto-migrated on read)
+
+**Data Stored**:
+```javascript
+{
+  url: "https://storage.../file.pdf?long-lived-sas",
+  shortLivedUrl: "https://storage.../file.pdf?short-lived-sas",
+  gcs: "gs://bucket/file.pdf",
+  hash: "abc123",
+  filename: "file.pdf",
+  timestamp: "2024-01-15T10:00:00.000Z",
+  converted: {
+    url: "https://storage.../converted.csv",
+    gcs: "gs://bucket/converted.csv"
+  }
+}
+```
+
+### Layer 3: File Collection (Cortex Redis Hash Maps)
+
+**Purpose**: User-facing file collections with metadata
+
+**Storage**: Redis hash maps (`FileStoreMap:ctx:<contextId>`)
+
+**Format**:
+```javascript
+// Redis Hash Map Structure:
+// Key: FileStoreMap:ctx:<contextId>
+// Value: Hash map where each entry is {hash: fileDataJSON}
+
+// Example hash map entry:
+{
+  "abc123": JSON.stringify({
+    // CFH-managed fields
+    url: "https://storage.../file.pdf",
+    gcs: "gs://bucket/file.pdf",
+    filename: "uuid-based-name.pdf",
+    
+    // Cortex-managed fields
+    id: "1736966400000-abc123",
+    displayFilename: "user-friendly-name.pdf",
+    mimeType: "application/pdf",
+    tags: ["pdf", "report"],
+    notes: "Quarterly report",
+    hash: "abc123",
+    permanent: true,
+    addedDate: "2024-01-15T10:00:00.000Z",
+    lastAccessed: "2024-01-15T10:00:00.000Z"
+  })
+}
+```
+
+**Features**:
+- Atomic operations (Redis HSET/HDEL/HGET are thread-safe)
+- In-memory caching (5-second TTL)
+- Direct hash map access (no versioning needed)
+- Context-scoped isolation (`FileStoreMap:ctx:<contextId>`)
+
+---
+
+## Key Concepts
+
+### 1. Context Scoping (`contextId`)
+
+**Purpose**: Per-user/per-context file isolation
+
+**Usage**:
+- **Strongly recommended** for all file operations
+- Passed to all file handler functions
+- Stored in Redis with scoped keys: `<hash>:ctx:<contextId>`
+
+**Benefits**:
+- Prevents hash collisions between users
+- Enables per-user file management
+- Supports multi-tenant applications
+
+**Example**:
+```javascript
+// Upload with contextId
+await uploadFileToCloud(fileBuffer, mimeType, filename, resolver, "user-123");
+
+// Check hash with contextId
+await checkHashExists(hash, fileHandlerUrl, null, "user-123");
+
+// Delete with contextId
+await deleteFileByHash(hash, resolver, "user-123");
+```
+
+### 2. Permanent Files (`permanent` flag)
+
+**Purpose**: Indicate files that should be kept indefinitely
+
+**Storage**:
+- Stored in file collection entry: `permanent: true`
+- Sets blob index tag: `retention=permanent`
+- Prevents deletion from cloud storage
+
+**Usage**:
+```javascript
+// Add permanent file
+await addFileToCollection(
+    contextId, contextKey, url, gcs, filename, tags, notes, hash,
+    null, resolver, true  // permanent=true
+);
+
+// Check before deletion
+if (!file.permanent) {
+    await deleteFileByHash(file.hash, resolver, contextId);
+}
+```
+
+**Behavior**:
+- Permanent files are **not deleted** from cloud storage when removed from collection
+- Retention set via `setRetentionForHash()` (best-effort)
+- Default: `permanent=false` (temporary, 30-day retention)
+
+### 3. Hash Deduplication
+
+**Purpose**: Avoid storing duplicate files
+
+**Process**:
+1. Compute xxhash64 hash of file content
+2. Check if hash exists via `checkHashExists()`
+3. If exists, return existing URLs (no upload)
+4. If not, upload and store hash
+
+**Benefits**:
+- Saves storage space
+- Faster uploads (skip if duplicate)
+- Consistent file references
+
+### 4. Short-Lived URLs
+
+**Purpose**: Secure, time-limited file access
+
+**Features**:
+- 5-minute expiration (configurable)
+- Always included in file handler responses
+- Preferred for LLM file access
+- Automatically generated on `checkHash` operations
+
+**Usage**:
+```javascript
+// Resolve to short-lived URL
+const fileWithShortLivedUrl = await ensureShortLivedUrl(
+    fileObject, fileHandlerUrl, contextId, 5  // 5 minutes
+);
+// fileWithShortLivedUrl.url is now short-lived URL
+```
+
+### 5. Atomic Operations
+
+**Purpose**: Ensure thread-safe collection modifications
+
+**Process**:
+- Redis hash map operations (HSET, HDEL, HGET) are atomic
+- No version-based locking needed
+- Direct hash map updates per file (not full collection replacement)
+
+**Functions**:
+- `addFileToCollection()`: Atomic HSET operation
+- `updateFileMetadata()`: Atomic HSET operation (updates single file)
+- `loadFileCollection()`: Atomic HGETALL operation
+- File removal: Atomic HDEL operation
+
+**Benefits**:
+- No version conflicts (each file updated independently)
+- Faster operations (no retry loops)
+- Simpler code (no locking logic needed)
+
+---
+
+## Complete Function Reference
+
+### File Handler Operations
+
+#### `buildFileHandlerUrl(baseUrl, params)`
+Builds file handler URL with query parameters.
+- **Parameters**:
+  - `baseUrl`: File handler service URL
+  - `params`: Object with query parameters (null/undefined skipped)
+- **Returns**: Complete URL with encoded parameters
+- **Used by**: All file handler operations
+
+#### `fetchFileFromUrl(fileUrl, requestId, contextId, save)`
+Downloads and processes file from URL.
+- **Parameters**:
+  - `fileUrl`: URL to fetch
+  - `requestId`: Request ID for tracking
+  - `contextId`: Optional context ID
+  - `save`: Whether to save converted file (default: false)
+- **Returns**: Response data (object or array)
+- **Used by**: `azureVideoTranslatePlugin`, `azureCognitivePlugin`
+
+#### `uploadFileToCloud(fileInput, mimeType, filename, pathwayResolver, contextId)`
+Uploads file to cloud storage with deduplication.
+- **Parameters**:
+  - `fileInput`: URL string, base64 string, or Buffer
+  - `mimeType`: MIME type (optional)
+  - `filename`: Filename (optional, inferred if not provided)
+  - `pathwayResolver`: Optional resolver for logging
+  - `contextId`: Optional context ID for scoping
+- **Returns**: `{url, gcs, hash}`
+- **Process**:
+  1. Converts input to Buffer
+  2. Computes hash
+  3. Checks if exists (deduplication)
+  4. Uploads if not exists
+- **Used by**: All tools that upload files
+
+#### `checkHashExists(hash, fileHandlerUrl, pathwayResolver, contextId, shortLivedMinutes)`
+Checks if file exists by hash.
+- **Parameters**:
+  - `hash`: File hash
+  - `fileHandlerUrl`: File handler URL
+  - `pathwayResolver`: Optional resolver for logging
+  - `contextId`: Optional context ID
+  - `shortLivedMinutes`: URL expiration (default: 5)
+- **Returns**: `{url, gcs, hash, filename}` or `null`
+- **Used by**: Upload deduplication, file resolution
+
+#### `deleteFileByHash(hash, pathwayResolver, contextId)`
+Deletes file from cloud storage.
+- **Parameters**:
+  - `hash`: File hash
+  - `pathwayResolver`: Optional resolver for logging
+  - `contextId`: Optional context ID
+- **Returns**: `true` if deleted, `false` if not found
+- **Handles**: 404 gracefully (file already deleted)
+
+#### `setRetentionForHash(hash, retention, contextId, pathwayResolver)`
+Sets file retention (temporary or permanent).
+- **Parameters**:
+  - `hash`: File hash
+  - `retention`: `'temporary'` or `'permanent'`
+  - `contextId`: Optional context ID
+  - `pathwayResolver`: Optional resolver for logging
+- **Returns**: Response data or `null`
+- **Used by**: `addFileToCollection` when `permanent=true`
+
+#### `ensureShortLivedUrl(fileObject, fileHandlerUrl, contextId, shortLivedMinutes)`
+Resolves file to use short-lived URL.
+- **Parameters**:
+  - `fileObject`: File object with `hash` and `url`
+  - `fileHandlerUrl`: File handler URL
+  - `contextId`: Optional context ID
+  - `shortLivedMinutes`: URL expiration (default: 5)
+- **Returns**: File object with `url` updated to short-lived URL
+- **Used by**: Tools that send files to LLMs
+
+#### `getMediaChunks(file, requestId, contextId)`
+Gets chunked media file URLs.
+- **Parameters**:
+  - `file`: File URL
+  - `requestId`: Request ID
+  - `contextId`: Optional context ID
+- **Returns**: Array of chunk URLs
+
+#### `markCompletedForCleanUp(requestId, contextId)`
+Marks request as completed for cleanup.
+- **Parameters**:
+  - `requestId`: Request ID
+  - `contextId`: Optional context ID
+- **Returns**: Response data or `null`
+
+### File Collection Operations
+
+#### `loadFileCollection(contextId, contextKey, useCache)`
+Loads file collection from Redis hash map.
+- **Parameters**:
+  - `contextId`: Context ID (required)
+  - `contextKey`: Optional encryption key (unused, kept for compatibility)
+  - `useCache`: Whether to use cache (default: true)
+- **Returns**: Array of file entries (sorted by lastAccessed, most recent first)
+- **Process**:
+  1. Checks in-memory cache (5-second TTL)
+  2. Loads from Redis hash map `FileStoreMap:ctx:<contextId>`
+  3. Converts hash map entries to array format
+  4. Updates cache
+- **Used by**: All file collection operations
+
+#### `saveFileCollection(contextId, contextKey, collection)`
+Saves file collection to Redis hash map (optimized - only updates changed entries).
+- **Parameters**:
+  - `contextId`: Context ID
+  - `contextKey`: Optional encryption key (unused, kept for compatibility)
+  - `collection`: Array of file entries
+- **Returns**: `true` if successful, `false` on error
+- **Process**:
+  1. Compares each file with current state
+  2. Only updates files that changed (optimized)
+  3. Uses atomic HSET operations per file
+  4. Preserves CFH-managed fields, updates Cortex-managed fields
+- **Used by**: Tools that need to save multiple file changes
+
+#### `updateFileMetadata(contextId, hash, metadata)`
+Updates Cortex-managed metadata fields atomically.
+- **Parameters**:
+  - `contextId`: Context ID (required)
+  - `hash`: File hash (used as Redis key)
+  - `metadata`: Object with fields to update (displayFilename, tags, notes, mimeType, addedDate, lastAccessed, permanent)
+- **Returns**: `true` if successful, `false` on error
+- **Process**:
+  1. Loads existing file data from Redis
+  2. Merges metadata (preserves CFH fields, updates Cortex fields)
+  3. Writes back via atomic HSET
+  4. Invalidates cache
+- **Used by**: Search operations (updates lastAccessed), EditFile (updates URL/hash)
+
+#### `addFileToCollection(contextId, contextKey, url, gcs, filename, tags, notes, hash, fileUrl, pathwayResolver, permanent)`
+Adds file to collection via atomic operation.
+- **Parameters**:
+  - `contextId`: Context ID (required)
+  - `contextKey`: Optional encryption key (unused, kept for compatibility)
+  - `url`: Azure URL (optional if fileUrl provided)
+  - `gcs`: GCS URL (optional)
+  - `filename`: User-friendly filename (required)
+  - `tags`: Array of tags (optional)
+  - `notes`: Notes string (optional)
+  - `hash`: File hash (optional, computed if not provided)
+  - `fileUrl`: URL to upload (optional, uploads if provided)
+  - `pathwayResolver`: Optional resolver for logging
+  - `permanent`: Whether file is permanent (default: false)
+- **Returns**: File entry object with `id`
+- **Process**:
+  1. If `fileUrl` provided, uploads file first via `uploadFileToCloud()`
+  2. If `permanent=true`, sets retention to permanent via `setRetentionForHash()`
+  3. Creates file entry with `displayFilename` (user-friendly name)
+  4. Writes to Redis hash map via atomic HSET
+  5. Merges with existing CFH data if hash already exists
+- **Used by**: WriteFile, Image tools, FileCollection tool
+
+#### `syncFilesToCollection(chatHistory, contextId, contextKey)`
+Syncs files from chat history to file collection.
+- **Parameters**:
+  - `chatHistory`: Chat history array to scan
+  - `contextId`: Context ID (required)
+  - `contextKey`: Optional encryption key
+- **Returns**: Updated file collection array
+- **Process**:
+  1. Extracts files from chat history via `extractFilesFromChatHistory()`
+  2. Checks for existing files by URL, GCS, or hash
+  3. Adds new files or updates lastAccessed for existing files
+  4. Uses atomic operations per file
+- **Used by**: `getAvailableFiles()` to sync files from conversation
+
+### File Resolution
+
+#### `resolveFileParameter(fileParam, contextId, contextKey, options)`
+Resolves file parameter to file URL.
+- **Parameters**:
+  - `fileParam`: File ID, filename, URL, or hash
+  - `contextId`: Context ID (required)
+  - `contextKey`: Optional encryption key
+  - `options`: Optional options object with `preferGcs` boolean
+- **Returns**: File URL string (Azure or GCS) or `null` if not found
+- **Matching** (via `findFileInCollection()`):
+  - Exact ID match
+  - Exact hash match
+  - Exact URL match (Azure or GCS)
+  - Exact filename match (case-insensitive, basename comparison)
+  - Fuzzy filename match (contains, minimum 4 characters)
+- **Used by**: ReadFile, EditFile, and other tools that need file URLs
+
+#### `findFileInCollection(fileParam, collection)`
+Finds file in collection array.
+- **Parameters**:
+  - `fileParam`: File identifier
+  - `collection`: Collection array
+- **Returns**: File entry or `null`
+- **Used by**: `resolveFileParameter`
+
+#### `generateFileMessageContent(fileParam, contextId, contextKey)`
+Generates file content for LLM messages.
+- **Parameters**:
+  - `fileParam`: File identifier (ID, filename, URL, or hash)
+  - `contextId`: Context ID (required)
+  - `contextKey`: Optional encryption key
+- **Returns**: File content object with `type`, `url`, `gcs`, `hash` or `null`
+- **Process**:
+  1. Finds file in collection via `findFileInCollection()`
+  2. Resolves to short-lived URL via `ensureShortLivedUrl()`
+  3. Returns OpenAI-compatible format: `{type: 'image_url', url, gcs, hash}`
+- **Used by**: AnalyzeFile tool to inject files into chat history
+
+#### `extractFilesFromChatHistory(chatHistory)`
+Extracts file metadata from chat history messages.
+- **Parameters**:
+  - `chatHistory`: Chat history array to scan
+- **Returns**: Array of file metadata objects `{url, gcs, hash, type}`
+- **Process**:
+  1. Scans all messages for file content objects
+  2. Extracts from `image_url`, `file`, or direct URL objects
+  3. Returns normalized format
+- **Used by**: `syncFilesToCollection()`, `getAvailableFiles()`
+
+#### `getAvailableFiles(chatHistory, contextId, contextKey)`
+Gets formatted list of available files for templates.
+- **Parameters**:
+  - `chatHistory`: Chat history to scan
+  - `contextId`: Context ID (required)
+  - `contextKey`: Optional encryption key
+- **Returns**: Formatted string of available files (last 10 most recent)
+- **Process**:
+  1. Syncs files from chat history via `syncFilesToCollection()`
+  2. Formats files via `formatFilesForTemplate()`
+  3. Returns compact one-line format per file
+- **Used by**: Template rendering to show available files
+
+### Utility Functions
+
+#### `computeFileHash(filePath)`
+Computes xxhash64 hash of file.
+- **Returns**: Hash string (hex)
+
+#### `computeBufferHash(buffer)`
+Computes xxhash64 hash of buffer.
+- **Returns**: Hash string (hex)
+
+#### `extractFilenameFromUrl(url, gcs)`
+Extracts filename from URL (prefers GCS).
+- **Returns**: Filename string
+
+#### `ensureFilenameExtension(filename, mimeType)`
+Ensures filename has correct extension based on MIME type.
+- **Returns**: Filename with correct extension
+
+#### `determineMimeTypeFromUrl(url, gcs, filename)`
+Determines MIME type from URL or filename.
+- **Returns**: MIME type string
+
+#### `isTextMimeType(mimeType)`
+Checks if MIME type is text-based.
+- **Parameters**:
+  - `mimeType`: MIME type string to check
+- **Returns**: Boolean (true if text-based)
+- **Supports**: All `text/*` types, plus application types like JSON, JavaScript, XML, YAML, Python, etc.
+- **Used by**: ReadFile, EditFile to validate file types
+
+#### `getMimeTypeFromFilename(filenameOrPath, defaultMimeType)`
+Gets MIME type from filename or path.
+- **Parameters**:
+  - `filenameOrPath`: Filename or full file path
+  - `defaultMimeType`: Optional default (default: 'application/octet-stream')
+- **Returns**: MIME type string
+- **Used by**: File upload, file type detection
+
+#### `getMimeTypeFromExtension(extension, defaultMimeType)`
+Gets MIME type from file extension.
+- **Parameters**:
+  - `extension`: File extension (with or without leading dot)
+  - `defaultMimeType`: Optional default (default: 'application/octet-stream')
+- **Returns**: MIME type string
+
+---
+
+## Error Handling
+
+### File Handler Errors
+
+**Network Errors**:
+- Handled gracefully in all functions
+- Logged via `pathwayResolver` or `logger`
+- Non-critical operations return `null` instead of throwing
+
+**404 Errors**:
+- Treated as "file not found" (not an error)
+- `deleteFileByHash` returns `false` on 404
+- `checkHashExists` returns `null` on 404
+
+**Timeout Errors**:
+- Upload: 30 seconds
+- Check hash: 10 seconds
+- Fetch file: 60 seconds
+- Set retention: 15 seconds
+
+### File Collection Errors
+
+**Missing ContextId**:
+- File collection operations require `contextId`
+- Returns `null` or throws error if missing
+
+**Concurrent Modifications**:
+- Prevented by atomic Redis operations (HSET, HDEL are thread-safe)
+- No version conflicts (each file updated independently)
+
+**Invalid File Data**:
+- Invalid JSON entries are skipped during load
+- Missing required fields are handled gracefully
+
+### Best Practices
+
+1. **Always pass `contextId`** when available (strongly recommended for multi-tenant)
+2. **Use atomic operations** - `addFileToCollection()`, `updateFileMetadata()` are thread-safe
+3. **Check `permanent` flag** before deleting files from cloud storage
+4. **Handle errors gracefully** - don't throw on non-critical failures
+5. **Use short-lived URLs** for LLM file access (via `ensureShortLivedUrl()`)
+6. **Check for existing files** before uploading (automatic in `uploadFileToCloud`)
+7. **Preserve CFH fields** - when updating metadata, preserve `url`, `gcs`, `filename` from file handler
+8. **Use `displayFilename`** for user-facing displays (fallback to `filename` if not set)
+
+---
+
+## Summary
+
+The Cortex file system provides:
+
+✅ **Encapsulated file handler interactions** - No direct axios calls
+✅ **Hash-based deduplication** - Avoids duplicate storage
+✅ **Context scoping** - Per-user file isolation via `FileStoreMap:ctx:<contextId>`
+✅ **Permanent file support** - Indefinite retention
+✅ **Atomic operations** - Thread-safe collection modifications via Redis hash maps
+✅ **Short-lived URLs** - Secure file access (5-minute expiration)
+✅ **Comprehensive error handling** - Graceful failure handling
+✅ **Single API call optimization** - Efficient file resolution
+✅ **Field ownership separation** - CFH-managed vs Cortex-managed fields
+✅ **Chat history integration** - Automatic file syncing from conversations
+
+All file operations flow through `lib/fileUtils.js`, ensuring consistency, maintainability, and proper error handling throughout the system.
+
+### Architecture Highlights
+
+- **File Handler Service**: External Azure Function managing cloud storage
+- **File Utilities Layer**: Abstraction over file handler (no direct API calls)
+- **File Collection System**: Redis hash maps for user file metadata
+- **Atomic Operations**: Thread-safe via Redis HSET/HDEL/HGET operations
+- **Context Isolation**: Per-context hash maps for multi-tenant support
+
diff --git a/README.md b/README.md
index cb9bce74..4c5896f0 100644
--- a/README.md
+++ b/README.md
@@ -700,6 +700,16 @@ Each model configuration can include:
     "maxImageSize": 5242880,
     "supportsStreaming": true,
     "supportsVision": true,
+    "emulateOpenAIChatModel": "gpt-4o",
+    "emulateOpenAICompletionModel": "gpt-3.5-turbo",
+    "restStreaming": {
+        "inputParameters": {
+            "stream": false
+        },
+        "timeout": 120,
+        "enableDuplicateRequests": false,
+        "geminiSafetySettings": []
+    },
     "geminiSafetySettings": [
         {
             "category": "HARM_CATEGORY",
@@ -709,6 +719,50 @@ Each model configuration can include:
 }
 ```
 
+**REST Endpoint Emulation**: To expose a model through OpenAI-compatible REST endpoints (`/v1/chat/completions` or `/v1/completions`), add one of these properties:
+
+- `emulateOpenAIChatModel`: Exposes the model as a chat completion model (e.g., `"gpt-4o"`, `"gpt-5"`, `"claude-4-sonnet"`)
+- `emulateOpenAICompletionModel`: Exposes the model as a text completion model (e.g., `"gpt-3.5-turbo"`, `"ollama-completion"`)
+
+When `enableRestEndpoints` is `true`, Cortex automatically:
+1. Generates REST streaming pathways for models with `emulateOpenAIChatModel` or `emulateOpenAICompletionModel`
+2. Exposes them through `/v1/chat/completions` or `/v1/completions` endpoints
+3. Makes them available via the `/v1/models` endpoint
+
+**Optional `restStreaming` Configuration**: You can customize the generated REST pathways with:
+- `inputParameters`: Additional input parameters for the REST endpoint
+- `timeout`: Request timeout in seconds
+- `enableDuplicateRequests`: Enable duplicate request handling
+- `geminiSafetySettings`: Gemini-specific safety settings (for Gemini models)
+
+**Example**:
+```json
+{
+    "oai-gpt4o": {
+        "type": "OPENAI-VISION",
+        "emulateOpenAIChatModel": "gpt-4o",
+        "restStreaming": {
+            "inputParameters": {
+                "stream": false
+            },
+            "timeout": 120
+        },
+        "url": "https://api.openai.com/v1/chat/completions",
+        "headers": {
+            "Authorization": "Bearer {{OPENAI_API_KEY}}",
+            "Content-Type": "application/json"
+        },
+        "params": {
+            "model": "gpt-4o"
+        },
+        "maxTokenLength": 131072,
+        "supportsStreaming": true
+    }
+}
+```
+
+This configuration will make the model available as `gpt-4o` through the `/v1/chat/completions` endpoint when `enableRestEndpoints` is `true`.
+
 **Rate Limiting**: The `requestsPerSecond` parameter controls the rate limiting for each model endpoint. If not specified, Cortex defaults to **100 requests per second** per endpoint. This rate limiting is implemented using the Bottleneck library with a token bucket algorithm that includes:
 - Minimum time between requests (`minTime`)
 - Maximum concurrent requests (`maxConcurrent`)
@@ -719,9 +773,11 @@ Each model configuration can include:
 
 Cortex provides OpenAI-compatible REST endpoints that allow you to use various models through a standardized interface. When `enableRestEndpoints` is set to `true`, Cortex exposes the following endpoints:
 
-- `/v1/models`: List available models
-- `/v1/chat/completions`: Chat completion endpoint
-- `/v1/completions`: Text completion endpoint
+- `/v1/models`: List available models (includes all models with `emulateOpenAIChatModel` or `emulateOpenAICompletionModel`)
+- `/v1/chat/completions`: Chat completion endpoint (for models with `emulateOpenAIChatModel`)
+- `/v1/completions`: Text completion endpoint (for models with `emulateOpenAICompletionModel`)
+
+**Model Exposure**: To expose a model through these endpoints, add `emulateOpenAIChatModel` or `emulateOpenAICompletionModel` to your model configuration (see [Model Configuration](#model-configuration) above). Cortex automatically generates REST streaming pathways for these models.
 
 This means you can use Cortex with any client library or tool that supports the OpenAI API format. For example:
 
@@ -870,6 +926,8 @@ Extends Cortex with several file processing capabilities:
 - Progress reporting for file operations
 - Cleanup and deletion management
 
+For comprehensive documentation on the Cortex file system architecture, see [FILE_SYSTEM_DOCUMENTATION.md](FILE_SYSTEM_DOCUMENTATION.md).
+
 Each helper app can be deployed independently using Docker:
 ```sh
 # Build the Docker image
@@ -882,6 +940,17 @@ docker tag [app-name] [registry-url]/cortex/[app-name]
 docker push [registry-url]/cortex/[app-name]
 ```
 
+## Documentation
+
+### File System
+For detailed documentation on Cortex's file system architecture, including file upload, storage, retrieval, and management, see [FILE_SYSTEM_DOCUMENTATION.md](FILE_SYSTEM_DOCUMENTATION.md). This document covers:
+- File handler service integration
+- File collection system
+- Storage layers (Azure Blob Storage, GCS, Redis)
+- System tools that use files
+- Complete function reference
+- Best practices and error handling
+
 ## Troubleshooting
 If you encounter any issues while using Cortex, there are a few things you can do. First, check the Cortex documentation for any common errors and their solutions. If that does not help, you can also open an issue on the Cortex GitHub repository.
 
diff --git a/tests/integration/features/tools/fileCollection.test.js b/tests/integration/features/tools/fileCollection.test.js
index 67b2c573..bf933aa9 100644
--- a/tests/integration/features/tools/fileCollection.test.js
+++ b/tests/integration/features/tools/fileCollection.test.js
@@ -217,7 +217,8 @@ test('File collection: Remove single file', async t => {
         });
         const listParsed = JSON.parse(listResult);
         t.is(listParsed.totalFiles, 1);
-        t.false(listParsed.files.some(f => f.filename === 'file1.jpg'));
+        // Check displayFilename with fallback to filename
+        t.false(listParsed.files.some(f => (f.displayFilename || f.filename) === 'file1.jpg'));
     } finally {
         await cleanup(contextId);
     }
@@ -840,3 +841,147 @@ test('resolveFileParameter: Handle contextKey for encrypted collections', async
         await cleanup(contextId);
     }
 });
+
+test('File collection: Update file metadata', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add a file first
+        const addResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/original.pdf',
+            filename: 'original.pdf',
+            tags: ['initial'],
+            notes: 'Initial notes',
+            userMessage: 'Add file'
+        });
+        
+        const addParsed = JSON.parse(addResult);
+        t.is(addParsed.success, true);
+        const fileId = addParsed.fileId;
+        
+        // Get the hash from the collection
+        const collection = await loadFileCollection(contextId, null, false);
+        const file = collection.find(f => f.id === fileId);
+        t.truthy(file);
+        const hash = file.hash;
+        
+        // Update metadata using updateFileMetadata
+        const { updateFileMetadata } = await import('../../../../lib/fileUtils.js');
+        const success = await updateFileMetadata(contextId, hash, {
+            displayFilename: 'renamed.pdf',
+            tags: ['updated', 'document'],
+            notes: 'Updated notes',
+            permanent: true
+        });
+        
+        t.is(success, true);
+        
+        // Verify metadata was updated
+        const updatedCollection = await loadFileCollection(contextId, null, false);
+        const updatedFile = updatedCollection.find(f => f.id === fileId);
+        t.truthy(updatedFile);
+        t.is(updatedFile.displayFilename, 'renamed.pdf');
+        t.deepEqual(updatedFile.tags, ['updated', 'document']);
+        t.is(updatedFile.notes, 'Updated notes');
+        t.is(updatedFile.permanent, true);
+        
+        // Verify CFH fields were preserved
+        t.is(updatedFile.url, 'https://example.com/original.pdf');
+        t.is(updatedFile.hash, hash);
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: Permanent files not deleted on remove', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add a permanent file
+        const addResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/permanent.pdf',
+            filename: 'permanent.pdf',
+            userMessage: 'Add permanent file'
+        });
+        
+        const addParsed = JSON.parse(addResult);
+        t.is(addParsed.success, true);
+        const fileId = addParsed.fileId;
+        
+        // Mark as permanent
+        const collection = await loadFileCollection(contextId, null, false);
+        const file = collection.find(f => f.id === fileId);
+        const { updateFileMetadata } = await import('../../../../lib/fileUtils.js');
+        await updateFileMetadata(contextId, file.hash, { permanent: true });
+        
+        // Remove from collection
+        const removeResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            fileIds: [fileId],
+            userMessage: 'Remove permanent file'
+        });
+        
+        const removeParsed = JSON.parse(removeResult);
+        t.is(removeParsed.success, true);
+        t.is(removeParsed.removedCount, 1);
+        // Message should indicate permanent files are not deleted from cloud
+        t.true(removeParsed.message.includes('permanent') || removeParsed.message.includes('Cloud storage cleanup'));
+        
+        // Verify file was removed from collection
+        const listResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            userMessage: 'List files'
+        });
+        const listParsed = JSON.parse(listResult);
+        t.is(listParsed.totalFiles, 0);
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: Sync files from chat history', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        const { syncFilesToCollection } = await import('../../../../lib/fileUtils.js');
+        
+        // Create chat history with files
+        const chatHistory = [
+            {
+                role: 'user',
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: { url: 'https://example.com/synced1.jpg' },
+                        gcs: 'gs://bucket/synced1.jpg',
+                        hash: 'hash1'
+                    },
+                    {
+                        type: 'file',
+                        url: 'https://example.com/synced2.pdf',
+                        gcs: 'gs://bucket/synced2.pdf',
+                        hash: 'hash2'
+                    }
+                ]
+            }
+        ];
+        
+        // Sync files to collection
+        await syncFilesToCollection(chatHistory, contextId, null);
+        
+        // Verify files were added
+        const collection = await loadFileCollection(contextId, null, false);
+        t.is(collection.length, 2);
+        t.true(collection.some(f => f.url === 'https://example.com/synced1.jpg'));
+        t.true(collection.some(f => f.url === 'https://example.com/synced2.pdf'));
+        
+        // Sync again (should update lastAccessed, not duplicate)
+        await syncFilesToCollection(chatHistory, contextId, null);
+        const collection2 = await loadFileCollection(contextId, null, false);
+        t.is(collection2.length, 2); // Should still be 2, not 4
+    } finally {
+        await cleanup(contextId);
+    }
+});
diff --git a/tests/integration/features/tools/fileOperations.test.js b/tests/integration/features/tools/fileOperations.test.js
index b8d57e1b..b6ae39b9 100644
--- a/tests/integration/features/tools/fileOperations.test.js
+++ b/tests/integration/features/tools/fileOperations.test.js
@@ -30,8 +30,12 @@ const createTestContext = () => {
 // Helper to clean up test data
 const cleanup = async (contextId, contextKey = null) => {
     try {
-        const { keyValueStorageClient } = await import('../../../../lib/keyValueStorageClient.js');
-        await keyValueStorageClient.delete(`${contextId}-memoryFiles`);
+        const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+        const redisClient = await getRedisClient();
+        if (redisClient) {
+            const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+            await redisClient.del(contextMapKey);
+        }
     } catch (e) {
         // Ignore cleanup errors
     }
diff --git a/tests/integration/features/tools/writefile.test.js b/tests/integration/features/tools/writefile.test.js
index 4ed5206a..fc7cb21c 100644
--- a/tests/integration/features/tools/writefile.test.js
+++ b/tests/integration/features/tools/writefile.test.js
@@ -75,7 +75,8 @@ test('WriteFile: Write and upload text file', async t => {
         // Verify it was added to file collection
         const collection = await loadFileCollection(contextId, null, false);
         t.is(collection.length, 1);
-        t.is(collection[0].filename, filename);
+        // Use displayFilename (user-friendly name) with fallback to filename (CFH-managed)
+        t.is(collection[0].displayFilename || collection[0].filename, filename);
         t.is(collection[0].url, parsed.url);
         t.truthy(collection[0].hash);
     } finally {
@@ -119,7 +120,8 @@ test('WriteFile: Write JSON file with tags and notes', async t => {
         // Verify it was added to file collection with metadata
         const collection = await loadFileCollection(contextId, null, false);
         t.is(collection.length, 1);
-        t.is(collection[0].filename, filename);
+        // Use displayFilename (user-friendly name) with fallback to filename (CFH-managed)
+        t.is(collection[0].displayFilename || collection[0].filename, filename);
         t.deepEqual(collection[0].tags, tags);
         t.is(collection[0].notes, notes);
     } finally {
@@ -312,12 +314,12 @@ test('WriteFile: Duplicate content (same hash)', async t => {
         t.is(parsed2.hash, firstHash); // Should have same hash
         
         // Both files with same hash should result in one entry (same content, CFH will find it)
-        // The second file will update the existing entry with the new filename
+        // The second file will update the existing entry with the new displayFilename
         const collection = await loadFileCollection(contextId, null, false);
         t.is(collection.length, 1); // Same hash = one entry
         t.is(collection[0].hash, firstHash); // Same hash
-        // The filename should be from the most recent write
-        t.is(collection[0].filename, filename2);
+        // The displayFilename should be from the most recent write
+        t.is(collection[0].displayFilename || collection[0].filename, filename2);
     } finally {
         await cleanup(contextId);
     }
diff --git a/tests/unit/core/fileCollection.test.js b/tests/unit/core/fileCollection.test.js
index fff7e1d5..19c2eefd 100644
--- a/tests/unit/core/fileCollection.test.js
+++ b/tests/unit/core/fileCollection.test.js
@@ -341,3 +341,73 @@ test('ensureFilenameExtension should normalize extensions (jpeg->jpg, markdown->
     t.is(ensureFilenameExtension('doc.markdown', 'text/markdown'), 'doc.md');
 });
 
+// Test MIME type utilities
+test('getMimeTypeFromFilename should detect MIME types from filenames', async t => {
+    const { getMimeTypeFromFilename } = await import('../../../lib/fileUtils.js');
+    
+    t.is(getMimeTypeFromFilename('test.pdf'), 'application/pdf');
+    t.is(getMimeTypeFromFilename('image.jpg'), 'image/jpeg');
+    t.is(getMimeTypeFromFilename('script.js'), 'application/javascript');
+    t.is(getMimeTypeFromFilename('readme.md'), 'text/markdown');
+    t.is(getMimeTypeFromFilename('data.json'), 'application/json');
+    t.is(getMimeTypeFromFilename('page.html'), 'text/html');
+    t.is(getMimeTypeFromFilename('data.csv'), 'text/csv');
+    // .xyz files may have a specific MIME type from the library, so we check it's not empty
+    const xyzMime = getMimeTypeFromFilename('unknown.xyz');
+    t.truthy(xyzMime);
+    t.not(xyzMime, '');
+    t.is(getMimeTypeFromFilename('noextension'), 'application/octet-stream');
+});
+
+test('getMimeTypeFromFilename should handle paths', async t => {
+    const { getMimeTypeFromFilename } = await import('../../../lib/fileUtils.js');
+    
+    t.is(getMimeTypeFromFilename('/path/to/file.pdf'), 'application/pdf');
+    t.is(getMimeTypeFromFilename('folder/subfolder/image.png'), 'image/png');
+    t.is(getMimeTypeFromFilename('C:\\Windows\\file.txt'), 'text/plain');
+});
+
+test('getMimeTypeFromExtension should detect MIME types from extensions', async t => {
+    const { getMimeTypeFromExtension } = await import('../../../lib/fileUtils.js');
+    
+    t.is(getMimeTypeFromExtension('.pdf'), 'application/pdf');
+    t.is(getMimeTypeFromExtension('pdf'), 'application/pdf');
+    t.is(getMimeTypeFromExtension('.jpg'), 'image/jpeg');
+    t.is(getMimeTypeFromExtension('js'), 'application/javascript');
+    t.is(getMimeTypeFromExtension('.md'), 'text/markdown');
+    t.is(getMimeTypeFromExtension('.json'), 'application/json');
+    // .xyz files may have a specific MIME type from the library, so we check it's not empty
+    const xyzMime = getMimeTypeFromExtension('.xyz');
+    t.truthy(xyzMime);
+    t.not(xyzMime, '');
+});
+
+test('isTextMimeType should identify text MIME types', async t => {
+    const { isTextMimeType } = await import('../../../lib/fileUtils.js');
+    
+    // Text types
+    t.true(isTextMimeType('text/plain'));
+    t.true(isTextMimeType('text/html'));
+    t.true(isTextMimeType('text/markdown'));
+    t.true(isTextMimeType('text/csv'));
+    t.true(isTextMimeType('text/javascript'));
+    t.true(isTextMimeType('application/json'));
+    t.true(isTextMimeType('application/javascript'));
+    t.true(isTextMimeType('application/xml'));
+    t.true(isTextMimeType('application/x-sh'));
+    t.true(isTextMimeType('application/x-python'));
+    
+    // Non-text types
+    t.false(isTextMimeType('image/jpeg'));
+    t.false(isTextMimeType('image/png'));
+    t.false(isTextMimeType('application/pdf'));
+    t.false(isTextMimeType('application/octet-stream'));
+    t.false(isTextMimeType('video/mp4'));
+    t.false(isTextMimeType('audio/mpeg'));
+    
+    // Edge cases
+    t.false(isTextMimeType(null));
+    t.false(isTextMimeType(undefined));
+    t.false(isTextMimeType(''));
+});
+

From f71cf53d6e2f87e28059a832ceb294d4fada4739 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Wed, 17 Dec 2025 23:26:34 -0700
Subject: [PATCH 10/27] fix: update generateFileMessageContent to pass
 contextId for short-lived URL resolution

- Modified the `generateFileMessageContent` function to pass `contextId` when ensuring short-lived URLs, improving file retrieval accuracy within the correct context scope.
---
 lib/fileUtils.js | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/fileUtils.js b/lib/fileUtils.js
index 68e5d2c5..4d5c4573 100644
--- a/lib/fileUtils.js
+++ b/lib/fileUtils.js
@@ -1321,9 +1321,8 @@ async function generateFileMessageContent(fileParam, contextId, contextKey = nul
     }
 
     // Resolve to short-lived URL if possible
-    // Note: contextId is not available in this function, so we pass null
-    // This is acceptable as short-lived URLs work without contextId
-    const fileWithShortLivedUrl = await ensureShortLivedUrl(foundFile, MEDIA_API_URL, null);
+    // Pass contextId to ensure files are found in the correct context scope
+    const fileWithShortLivedUrl = await ensureShortLivedUrl(foundFile, MEDIA_API_URL, contextId);
 
     return {
         type: 'image_url',

From 0b2392c96781d2b318b4d1514d107220967b756a Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Thu, 18 Dec 2025 16:45:22 -0700
Subject: [PATCH 11/27] feat: enhance file collection management with
 inCollection support

- Updated file handling functions to incorporate inCollection metadata, allowing for flexible filtering of files based on chat IDs.
- Modified loadFileCollection to cache raw file data, enabling efficient retrieval and filtering by inCollection status.
- Enhanced updateFileMetadata to support updates to inCollection, including normalization of values for backward compatibility.
- Improved tests to validate the functionality of inCollection updates and ensure accurate file retrieval based on context.
---
 helper-apps/cortex-file-handler/src/index.js  |  10 +-
 lib/fileUtils.js                              | 220 +++++++++++++-----
 .../entity/files/sys_update_file_metadata.js  |  10 +-
 .../system/entity/tools/sys_tool_editfile.js  |  34 ++-
 .../entity/tools/sys_tool_file_collection.js  |   3 +-
 .../system/entity/tools/sys_tool_readfile.js  |   3 +-
 .../features/tools/fileCollection.test.js     |  74 ++++++
 7 files changed, 282 insertions(+), 72 deletions(-)

diff --git a/helper-apps/cortex-file-handler/src/index.js b/helper-apps/cortex-file-handler/src/index.js
index 397a57e1..7aded91c 100644
--- a/helper-apps/cortex-file-handler/src/index.js
+++ b/helper-apps/cortex-file-handler/src/index.js
@@ -282,10 +282,10 @@ async function CortexFileHandler(context, req) {
       }
 
       // Check if file already exists (using hash or URL as the key)
-      // For hash lookups, use raw hash; for URL lookups, use URL as key (unscoped only)
+      // Always respect contextId if provided, even for URL-based lookups
       const exists = hash 
         ? await getFileStoreMap(hash, false, resolvedContextId)
-        : await getFileStoreMap(remoteUrl, false, null); // URL lookups are unscoped
+        : await getFileStoreMap(remoteUrl, false, resolvedContextId);
       if (exists) {
         context.res = {
           status: 200,
@@ -295,7 +295,7 @@ async function CortexFileHandler(context, req) {
         if (hash) {
           await setFileStoreMap(hash, exists, resolvedContextId);
         } else {
-          await setFileStoreMap(remoteUrl, exists, null); // URL lookups are unscoped
+          await setFileStoreMap(remoteUrl, exists, resolvedContextId);
         }
         return;
       }
@@ -318,11 +318,11 @@ async function CortexFileHandler(context, req) {
       res.permanent = false;
 
       //Update Redis (using hash or URL as the key)
-      // Container parameter is ignored - always uses default container from env var
+      // Always respect contextId if provided, even for URL-based lookups
       if (hash) {
         await setFileStoreMap(hash, res, resolvedContextId);
       } else {
-        await setFileStoreMap(remoteUrl, res, null); // URL lookups are unscoped
+        await setFileStoreMap(remoteUrl, res, resolvedContextId);
       }
 
       // Return the file URL
diff --git a/lib/fileUtils.js b/lib/fileUtils.js
index 4d5c4573..e17b8814 100644
--- a/lib/fileUtils.js
+++ b/lib/fileUtils.js
@@ -345,6 +345,8 @@ function extractFileMetadataFromContent(contentObj) {
 }
 
 // Cache for file collections during a request lifecycle
+// Stores raw parsed file data (all files from Redis) to support flexible filtering
+// Structure: { rawFiles: Array<parsed file data>, timestamp: number }
 const fileCollectionCache = new Map();
 const CACHE_TTL = 5000; // 5 seconds
 
@@ -441,30 +443,119 @@ function extractFilesFromChatHistory(chatHistory) {
     return extractedFiles;
 }
 
+/**
+ * Check if a file should be included in the collection based on inCollection metadata
+ * Supports both boolean (backward compat) and array format
+ * @param {boolean|Array<string>|undefined} inCollection - inCollection metadata value
+ * @param {string|null} chatId - Optional chat ID to filter by (if null, only global files are included)
+ * @returns {boolean} True if file should be included
+ */
+function isFileInCollection(inCollection, chatId = null) {
+    // If not set, file is not in collection
+    if (inCollection === undefined || inCollection === null || inCollection === false) {
+        return false;
+    }
+    
+    // Backward compatibility: boolean true means global
+    if (inCollection === true) {
+        return true;
+    }
+    
+    // Array format: check if it includes '*' (global) or the specific chatId
+    if (Array.isArray(inCollection)) {
+        // If no chatId specified, only include global files
+        if (chatId === null) {
+            return inCollection.includes('*');
+        }
+        // Include if global or matches specific chatId
+        return inCollection.includes('*') || inCollection.includes(chatId);
+    }
+    
+    // Unknown format, exclude
+    return false;
+}
+
 /**
  * Load file collection from memory system or cache
  * @param {string} contextId - Context ID for the file collection
  * @param {string} contextKey - Optional context key for encryption
  * @param {boolean} useCache - Whether to check cache first (default: true)
+ * @param {string|null} chatId - Optional chat ID to filter files by (if provided, only includes files with '*' or this chatId in inCollection)
  * @returns {Promise<Array>} File collection array
  */
-async function loadFileCollection(contextId, contextKey = null, useCache = true) {
+/**
+ * Parse raw Redis hash map data into file objects (without filtering)
+ * @param {Object} allFiles - Redis HGETALL result {hash: fileDataStr}
+ * @returns {Array} Array of parsed file data objects (includes inCollection metadata)
+ */
+function parseRawFileData(allFiles) {
+    return Object.entries(allFiles).map(([hash, fileDataStr]) => {
+        try {
+            const fileData = JSON.parse(fileDataStr);
+            // Return parsed file data with hash and inCollection preserved for filtering
+            return {
+                id: fileData.id || `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
+                url: fileData.url,
+                gcs: fileData.gcs || null,
+                displayFilename: fileData.displayFilename || fileData.filename || null,
+                mimeType: fileData.mimeType || null,
+                tags: fileData.tags || [],
+                notes: fileData.notes || '',
+                hash: hash,
+                permanent: fileData.permanent || false,
+                addedDate: fileData.addedDate || fileData.timestamp || new Date().toISOString(),
+                lastAccessed: fileData.lastAccessed || fileData.timestamp || new Date().toISOString(),
+                // Preserve inCollection for filtering
+                inCollection: fileData.inCollection
+            };
+        } catch (e) {
+            // Skip invalid entries
+            return null;
+        }
+    }).filter(Boolean);
+}
+
+/**
+ * Filter and format file collection based on inCollection and chatId
+ * @param {Array} rawFiles - Array of parsed file data objects
+ * @param {string|null} chatId - Optional chat ID to filter by
+ * @returns {Array} Filtered and sorted file collection
+ */
+function filterAndFormatFileCollection(rawFiles, chatId = null) {
+    // Filter by inCollection and optional chatId
+    const filtered = rawFiles.filter(file => isFileInCollection(file.inCollection, chatId));
+    
+    // Remove inCollection from output (internal metadata)
+    const formatted = filtered.map(({ inCollection, ...file }) => file);
+    
+    // Sort by lastAccessed (most recent first)
+    formatted.sort((a, b) => {
+        const aDate = new Date(a.lastAccessed || a.addedDate || 0);
+        const bDate = new Date(b.lastAccessed || b.addedDate || 0);
+        return bDate - aDate;
+    });
+    
+    return formatted;
+}
+
+async function loadFileCollection(contextId, contextKey = null, useCache = true, chatId = null) {
     if (!contextId) {
         return [];
     }
 
     const cacheKey = getCollectionCacheKey(contextId, contextKey);
 
-    // Check cache first
+    // Check cache first - cache stores raw parsed file data, so we can filter by chatId from cache
     if (useCache && fileCollectionCache.has(cacheKey)) {
         const cached = fileCollectionCache.get(cacheKey);
         if (Date.now() - cached.timestamp < CACHE_TTL) {
-            return cached.collection;
+            // Apply filtering to cached raw data
+            return filterAndFormatFileCollection(cached.rawFiles, chatId);
         }
     }
 
     // Load from context-scoped Redis hash map (FileStoreMap:ctx:<contextId>)
-    let files = [];
+    let rawFiles = [];
     
     try {
         const redisClient = await getRedisClient();
@@ -473,60 +564,61 @@ async function loadFileCollection(contextId, contextKey = null, useCache = true)
             const contextMapKey = `FileStoreMap:ctx:${contextId}`;
             const allFiles = await redisClient.hgetall(contextMapKey);
             
-            // Convert hash map entries to file collection array
-            // Each entry is {hash: fileData}
-            files = Object.entries(allFiles).map(([hash, fileDataStr]) => {
-                try {
-                    const fileData = JSON.parse(fileDataStr);
-                    // Extract file collection metadata (tags, notes, etc.) if present
-                    // Otherwise create minimal entry from CFH data
-                    // Use displayFilename (user-friendly name) instead of filename (CFH-managed)
-                    // Fallback to filename if displayFilename is not set (for files uploaded before displayFilename was added)
-                    return {
-                        id: fileData.id || `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
-                        url: fileData.url,
-                        gcs: fileData.gcs || null,
-                        displayFilename: fileData.displayFilename || fileData.filename || null,
-                        mimeType: fileData.mimeType || null,
-                        tags: fileData.tags || [],
-                        notes: fileData.notes || '',
-                        hash: hash,
-                        permanent: fileData.permanent || false,
-                        addedDate: fileData.addedDate || fileData.timestamp || new Date().toISOString(),
-                        lastAccessed: fileData.lastAccessed || fileData.timestamp || new Date().toISOString()
-                    };
-                } catch (e) {
-                    // Skip invalid entries
-                    return null;
-                }
-            }).filter(Boolean);
-            
-            // Sort by lastAccessed (most recent first)
-            files.sort((a, b) => {
-                const aDate = new Date(a.lastAccessed || a.addedDate || 0);
-                const bDate = new Date(b.lastAccessed || b.addedDate || 0);
-                return bDate - aDate;
-            });
+            // Parse raw file data (preserves inCollection metadata for filtering)
+            rawFiles = parseRawFileData(allFiles);
         }
     } catch (e) {
         // Collection doesn't exist yet or error reading, start with empty array
-        files = [];
+        rawFiles = [];
     }
 
-    // Update cache
-    fileCollectionCache.set(cacheKey, {
-        collection: files,
-        timestamp: Date.now()
-    });
+    // Update cache with raw file data (supports any filtering on retrieval)
+    if (useCache) {
+        fileCollectionCache.set(cacheKey, {
+            rawFiles: rawFiles,
+            timestamp: Date.now()
+        });
+    }
 
-    return files;
+    // Filter and format for return
+    return filterAndFormatFileCollection(rawFiles, chatId);
+}
+
+/**
+ * Normalize inCollection value to array format
+ * @param {boolean|Array<string>|undefined} inCollection - inCollection value to normalize
+ * @returns {Array<string>|undefined} Normalized array or undefined if false/null
+ */
+function normalizeInCollection(inCollection) {
+    // If explicitly false or null, return undefined (file not in collection)
+    if (inCollection === false || inCollection === null) {
+        return undefined;
+    }
+    
+    // If undefined, return undefined (preserve existing state)
+    if (inCollection === undefined) {
+        return undefined;
+    }
+    
+    // Boolean true means global
+    if (inCollection === true) {
+        return ['*'];
+    }
+    
+    // Already an array, return as-is
+    if (Array.isArray(inCollection)) {
+        return inCollection;
+    }
+    
+    // Unknown format, default to global
+    return ['*'];
 }
 
 /**
  * Update file metadata in Redis hash map (direct atomic operation)
  * @param {string} contextId - Context ID
  * @param {string} hash - File hash
- * @param {Object} metadata - Metadata to update (displayFilename, id, tags, notes, mimeType, addedDate, lastAccessed, permanent)
+ * @param {Object} metadata - Metadata to update (displayFilename, id, tags, notes, mimeType, addedDate, lastAccessed, permanent, inCollection)
  * Note: Does NOT update CFH core fields (url, gcs, hash, filename) - those are managed by CFH
  * @returns {Promise<boolean>} True if successful
  */
@@ -558,6 +650,12 @@ async function updateFileMetadata(contextId, hash, metadata) {
         // Only update Cortex-managed fields, preserve CFH fields (url, gcs, hash, filename)
         const fileData = {
             ...existingData, // Preserve all CFH data (url, gcs, hash, filename, etc.)
+            // Handle inCollection: normalize if provided, otherwise preserve existing or default to global
+            inCollection: metadata.inCollection !== undefined
+                ? normalizeInCollection(metadata.inCollection)
+                : (existingData.inCollection !== undefined
+                    ? normalizeInCollection(existingData.inCollection)
+                    : ['*']),
             // Update only Cortex-managed metadata fields
             ...(metadata.displayFilename !== undefined && { displayFilename: metadata.displayFilename }),
             ...(metadata.id !== undefined && { id: metadata.id }),
@@ -569,6 +667,11 @@ async function updateFileMetadata(contextId, hash, metadata) {
             ...(metadata.permanent !== undefined && { permanent: metadata.permanent })
         };
         
+        // Remove inCollection if it's undefined (file not in collection)
+        if (fileData.inCollection === undefined) {
+            delete fileData.inCollection;
+        }
+        
         // Write back to hash map (atomic operation)
         await redisClient.hset(contextMapKey, hash, JSON.stringify(fileData));
         
@@ -648,6 +751,7 @@ async function saveFileCollection(contextId, contextKey, collection) {
                 
                 // Merge CFH data with Cortex metadata
                 // Preserve all CFH fields (url, gcs, filename, displayFilename, etc.)
+                // Mark as inCollection: true (chat files that should appear in file collection)
                 const fileData = {
                     ...existingData, // Preserve all CFH data first
                     id: file.id,
@@ -660,7 +764,8 @@ async function saveFileCollection(contextId, contextKey, collection) {
                     mimeType: file.mimeType || existingData.mimeType || null,
                     addedDate: file.addedDate || existingData.timestamp || new Date().toISOString(),
                     lastAccessed: file.lastAccessed || new Date().toISOString(),
-                    permanent: file.permanent !== undefined ? file.permanent : (existingData.permanent || false)
+                    permanent: file.permanent !== undefined ? file.permanent : (existingData.permanent || false),
+                    inCollection: ['*'] // Mark as global chat file (available to all chats)
                 };
                 
                 // Write back to hash map (atomic operation)
@@ -671,11 +776,8 @@ async function saveFileCollection(contextId, contextKey, collection) {
         // Note: We don't remove files from hash map when removed from collection
         // CFH manages file lifecycle, and files might still exist in storage
 
-        // Update cache
-        fileCollectionCache.set(cacheKey, {
-            collection,
-            timestamp: Date.now()
-        });
+        // Invalidate cache (will be repopulated on next loadFileCollection call with fresh Redis data)
+        fileCollectionCache.delete(cacheKey);
         
         return true;
     } catch (e) {
@@ -786,6 +888,7 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
                 
                 // Merge CFH data with Cortex metadata
                 // If file already exists with same hash, update metadata but keep the existing entry
+                // Mark as inCollection: true (chat files that should appear in file collection)
                 const fileData = {
                     ...existingData, // Preserve CFH data (url, gcs, filename, etc.)
                     // Update Cortex metadata (use new ID if this is a new entry, otherwise keep existing)
@@ -797,6 +900,7 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
                     tags: fileEntry.tags.length > 0 ? fileEntry.tags : (existingData.tags || []), // Merge tags if new ones provided
                     notes: fileEntry.notes || existingData.notes || '', // Keep existing notes if new ones empty
                     mimeType: fileEntry.mimeType || existingData.mimeType || null,
+                    inCollection: ['*'], // Mark as global chat file (available to all chats)
                     addedDate: existingData.addedDate || fileEntry.addedDate, // Keep earliest addedDate
                     lastAccessed: new Date().toISOString(), // Always update lastAccessed
                     permanent: fileEntry.permanent !== undefined ? fileEntry.permanent : (existingData.permanent || false),
@@ -1013,7 +1117,8 @@ async function syncFilesToCollection(chatHistory, contextId, contextKey = null)
                         tags: existingData.tags || [],
                         notes: existingData.notes || '',
                         addedDate: existingData.addedDate || existingData.timestamp || new Date().toISOString(),
-                        lastAccessed: new Date().toISOString()
+                        lastAccessed: new Date().toISOString(),
+                        inCollection: ['*'] // Mark as global chat file (available to all chats)
                     };
                     
                     await redisClient.hset(contextMapKey, file.hash, JSON.stringify(fileData));
@@ -1031,7 +1136,8 @@ async function syncFilesToCollection(chatHistory, contextId, contextKey = null)
                         hash: file.hash,
                         permanent: false,
                         addedDate: new Date().toISOString(),
-                        lastAccessed: new Date().toISOString()
+                        lastAccessed: new Date().toISOString(),
+                        inCollection: ['*'] // Mark as global chat file (available to all chats)
                     };
                     
                     await redisClient.hset(contextMapKey, file.hash, JSON.stringify(fileData));
@@ -1259,7 +1365,7 @@ export async function resolveFileParameter(fileParam, contextId, contextKey = nu
     }
 
     const trimmed = fileParam.trim();
-    const { preferGcs = false } = options;
+    const { preferGcs = false, useCache = true } = options;
 
     // If no contextId, can't look up in collection - return null
     if (!contextId) {
@@ -1268,7 +1374,8 @@ export async function resolveFileParameter(fileParam, contextId, contextKey = nu
 
     try {
         // Load file collection and find the file
-        const collection = await loadFileCollection(contextId, contextKey, true);
+        // useCache can be set to false to bypass cache (e.g., after file edits)
+        const collection = await loadFileCollection(contextId, contextKey, useCache);
         const foundFile = findFileInCollection(trimmed, collection);
         
         if (foundFile) {
@@ -1931,6 +2038,7 @@ export {
     getMimeTypeFromFilename,
     getMimeTypeFromExtension,
     isTextMimeType,
+    isFileInCollection,
     // Exported for testing
     extractFilenameFromUrl,
     ensureFilenameExtension,
diff --git a/pathways/system/entity/files/sys_update_file_metadata.js b/pathways/system/entity/files/sys_update_file_metadata.js
index 27caa099..d384c731 100644
--- a/pathways/system/entity/files/sys_update_file_metadata.js
+++ b/pathways/system/entity/files/sys_update_file_metadata.js
@@ -12,13 +12,14 @@ export default {
         tags: { type: 'array', items: { type: 'string' } }, // Optional - no default
         notes: { type: 'string' }, // Optional - no default
         mimeType: { type: 'string' }, // Optional - no default
-        permanent: { type: 'boolean' } // Optional - no default
+        permanent: { type: 'boolean' }, // Optional - no default
+        inCollection: { type: 'array', items: { type: 'string' } } // Optional - array of chat IDs, or can be boolean true/false (normalized to ['*'] or removed)
     },
     model: 'oai-gpt4o',
     isMutation: true, // Declaratively mark this as a Mutation
 
     resolver: async (_parent, args, _contextValue, _info) => {
-        const { contextId, hash, displayFilename, tags, notes, mimeType, permanent } = args;
+        const { contextId, hash, displayFilename, tags, notes, mimeType, permanent, inCollection } = args;
         
         // Validate required parameters
         if (!contextId || !hash) {
@@ -46,6 +47,11 @@ export default {
             if (permanent !== undefined && permanent !== null) {
                 metadata.permanent = Boolean(permanent);
             }
+            // inCollection can be: boolean true/false, or array of chat IDs (e.g., ['*'] for global, ['chat-123'] for specific chat)
+            // Will be normalized by updateFileMetadata: true -> ['*'], false -> undefined (removed), array -> as-is
+            if (inCollection !== undefined && inCollection !== null) {
+                metadata.inCollection = inCollection;
+            }
             
             // Update metadata (only Cortex-managed fields)
             const success = await updateFileMetadata(contextId, hash, metadata);
diff --git a/pathways/system/entity/tools/sys_tool_editfile.js b/pathways/system/entity/tools/sys_tool_editfile.js
index bb59129f..c7caf99b 100644
--- a/pathways/system/entity/tools/sys_tool_editfile.js
+++ b/pathways/system/entity/tools/sys_tool_editfile.js
@@ -2,7 +2,7 @@
 // Entity tool that modifies existing files by replacing line ranges or exact string matches
 import logger from '../../../../lib/logger.js';
 import { axios } from '../../../../lib/requestExecutor.js';
-import { uploadFileToCloud, findFileInCollection, loadFileCollection, saveFileCollection, getMimeTypeFromFilename, resolveFileParameter, deleteFileByHash, isTextMimeType, updateFileMetadata } from '../../../../lib/fileUtils.js';
+import { uploadFileToCloud, findFileInCollection, loadFileCollection, saveFileCollection, getMimeTypeFromFilename, resolveFileParameter, deleteFileByHash, isTextMimeType, updateFileMetadata, getCollectionCacheKey } from '../../../../lib/fileUtils.js';
 
 export default {
     prompt: [],
@@ -293,7 +293,8 @@ export default {
             }
 
             // Determine MIME type from filename using utility function
-            const filename = foundFile.filename || 'modified.txt';
+            // Use displayFilename (user-friendly) if available, otherwise fall back to filename (CFH-managed)
+            const filename = foundFile.displayFilename || foundFile.filename || 'modified.txt';
             let mimeType = getMimeTypeFromFilename(filename, 'text/plain');
             
             // Add charset=utf-8 for text-based MIME types
@@ -351,12 +352,14 @@ export default {
                         url: uploadResult.url,
                         gcs: uploadResult.gcs || null,
                         hash: uploadResult.hash,
-                        filename: uploadResult.filename || filename, // Use CFH filename if available, otherwise preserve
+                        filename: uploadResult.filename || fileToUpdate.filename || filename, // Use CFH filename if available, otherwise preserve
                         // Cortex-managed metadata
                         id: fileToUpdate.id, // Keep same ID
+                        displayFilename: fileToUpdate.displayFilename || filename, // Preserve user-friendly filename
                         tags: fileToUpdate.tags || [],
                         notes: fileToUpdate.notes || '',
                         mimeType: fileToUpdate.mimeType || mimeType || null,
+                        inCollection: ['*'], // Mark as global chat file (available to all chats)
                         addedDate: fileToUpdate.addedDate, // Keep original added date
                         lastAccessed: new Date().toISOString(),
                         permanent: fileToUpdate.permanent || false
@@ -369,8 +372,6 @@ export default {
                     if (oldHashToDelete && oldHashToDelete !== uploadResult.hash) {
                         await redisClient.hdel(contextMapKey, oldHashToDelete);
                     }
-                    
-                    // Cache will expire naturally (5 second TTL) or can be invalidated by reloading collection
                 }
             } else if (fileToUpdate.hash) {
                 // Same hash, just update Cortex metadata (filename, lastAccessed)
@@ -398,8 +399,20 @@ export default {
             }
             
             // Get the updated file info for the result
+            // Use useCache: false to ensure we get fresh data after Redis write
             const updatedCollection = await loadFileCollection(contextId, contextKey, false);
             const updatedFile = updatedCollection.find(f => f.id === fileIdToUpdate);
+            
+            if (!updatedFile) {
+                logger.warn(`File with ID "${fileIdToUpdate}" not found in updated collection. This may indicate a timing issue.`);
+                // Fall back to using uploadResult data directly
+                const fallbackFile = {
+                    id: fileIdToUpdate,
+                    url: uploadResult.url,
+                    hash: uploadResult.hash
+                };
+                logger.info(`Using fallback file data: ${JSON.stringify(fallbackFile)}`);
+            }
 
             // Build result message
             let message;
@@ -416,13 +429,20 @@ export default {
             const result = {
                 success: true,
                 filename: filename,
-                fileId: updatedFile.id,
-                url: uploadResult.url,
+                fileId: updatedFile?.id || fileIdToUpdate,
+                url: uploadResult.url, // Always use the new URL from upload
                 gcs: uploadResult.gcs || null,
                 hash: uploadResult.hash || null,
                 ...modificationInfo,
                 message: message
             };
+            
+            // Log for debugging
+            if (!updatedFile) {
+                logger.warn(`EditFile: Could not find updated file in collection, but upload succeeded. Using uploadResult URL: ${uploadResult.url}`);
+            } else {
+                logger.info(`EditFile: Successfully updated file. New URL: ${uploadResult.url}, New hash: ${uploadResult.hash}`);
+            }
 
             resolver.tool = JSON.stringify({ toolUsed: toolName });
             return JSON.stringify(result);
diff --git a/pathways/system/entity/tools/sys_tool_file_collection.js b/pathways/system/entity/tools/sys_tool_file_collection.js
index 52403160..fb78ac18 100644
--- a/pathways/system/entity/tools/sys_tool_file_collection.js
+++ b/pathways/system/entity/tools/sys_tool_file_collection.js
@@ -416,7 +416,8 @@ export default {
                 // List collection (read-only, no locking needed)
                 const { tags: filterTags = [], sortBy = 'date', limit = 50 } = args;
                 
-                const collection = await loadFileCollection(contextId, contextKey, true);
+                // Use useCache: false to ensure we get the latest file data (important after edits)
+                const collection = await loadFileCollection(contextId, contextKey, false);
                 let results = collection;
 
                 // Filter by tags if provided
diff --git a/pathways/system/entity/tools/sys_tool_readfile.js b/pathways/system/entity/tools/sys_tool_readfile.js
index 40983ade..ed8cce58 100644
--- a/pathways/system/entity/tools/sys_tool_readfile.js
+++ b/pathways/system/entity/tools/sys_tool_readfile.js
@@ -112,7 +112,8 @@ export default {
                     resolver.tool = JSON.stringify({ toolUsed: "ReadFile" });
                     return JSON.stringify(errorResult);
                 }
-                const resolvedUrl = await resolveFileParameter(file, contextId, contextKey);
+                // Use useCache: false to ensure we get the latest file data (important after edits)
+                const resolvedUrl = await resolveFileParameter(file, contextId, contextKey, { useCache: false });
                 if (!resolvedUrl) {
                     const errorResult = {
                         success: false,
diff --git a/tests/integration/features/tools/fileCollection.test.js b/tests/integration/features/tools/fileCollection.test.js
index bf933aa9..cc1bcbc4 100644
--- a/tests/integration/features/tools/fileCollection.test.js
+++ b/tests/integration/features/tools/fileCollection.test.js
@@ -894,6 +894,80 @@ test('File collection: Update file metadata', async t => {
     }
 });
 
+test('updateFileMetadata should allow updating inCollection', async (t) => {
+    const contextId = `test-${Date.now()}`;
+    
+    try {
+        // Add a file to collection
+        const addResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/test-incollection.pdf',
+            filename: 'test-incollection.pdf',
+            userMessage: 'Add file'
+        });
+        
+        const addParsed = JSON.parse(addResult);
+        t.is(addParsed.success, true);
+        const fileId = addParsed.fileId;
+        
+        // Get the hash from the collection
+        const collection = await loadFileCollection(contextId, null, false);
+        const file = collection.find(f => f.id === fileId);
+        t.truthy(file);
+        const hash = file.hash;
+        
+        // Verify file is in collection (should be global by default)
+        t.truthy(file);
+        
+        // Update inCollection to a specific chat
+        const { updateFileMetadata } = await import('../../../../lib/fileUtils.js');
+        const success1 = await updateFileMetadata(contextId, hash, {
+            inCollection: ['chat-123']
+        });
+        t.is(success1, true);
+        
+        // Verify file is now only in chat-123 (not global)
+        const collection1 = await loadFileCollection(contextId, null, false);
+        const file1 = collection1.find(f => f.id === fileId);
+        // Should not appear in global collection
+        t.falsy(file1);
+        
+        // Should appear when filtering by chat-123
+        const collection2 = await loadFileCollection(contextId, null, false, 'chat-123');
+        const file2 = collection2.find(f => f.id === fileId);
+        t.truthy(file2);
+        
+        // Update inCollection back to global
+        const success2 = await updateFileMetadata(contextId, hash, {
+            inCollection: ['*']
+        });
+        t.is(success2, true);
+        
+        // Verify file is back in global collection
+        const collection3 = await loadFileCollection(contextId, null, false);
+        const file3 = collection3.find(f => f.id === fileId);
+        t.truthy(file3);
+        
+        // Update inCollection to false (remove from collection)
+        const success3 = await updateFileMetadata(contextId, hash, {
+            inCollection: false
+        });
+        t.is(success3, true);
+        
+        // Verify file is no longer in collection
+        const collection4 = await loadFileCollection(contextId, null, false);
+        const file4 = collection4.find(f => f.id === fileId);
+        t.falsy(file4);
+        
+        // Also not in chat-specific collection
+        const collection5 = await loadFileCollection(contextId, null, false, 'chat-123');
+        const file5 = collection5.find(f => f.id === fileId);
+        t.falsy(file5);
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
 test('File collection: Permanent files not deleted on remove', async t => {
     const contextId = createTestContext();
     

From 21d00f23b505216cd65502f7a8b05a259d5076a1 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Thu, 18 Dec 2025 17:06:51 -0700
Subject: [PATCH 12/27] refactor: update file collection handling and memory
 normalization

- Revised the GraphQL pathway for reading file collections to clarify that file collections are now stored in Redis hash maps.
- Updated memory normalization logic to filter out deprecated memoryFiles, ensuring only valid sections are processed and stored in the upgraded memory format.
---
 .../system/entity/files/sys_read_file_collection.js |  3 ++-
 pathways/system/entity/memory/sys_memory_manager.js | 13 +++++++++----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/pathways/system/entity/files/sys_read_file_collection.js b/pathways/system/entity/files/sys_read_file_collection.js
index b99e61e8..f7b8e791 100644
--- a/pathways/system/entity/files/sys_read_file_collection.js
+++ b/pathways/system/entity/files/sys_read_file_collection.js
@@ -1,5 +1,6 @@
 // sys_read_file_collection.js
-// GraphQL pathway for reading file collections (replaces sys_read_memory with section: "memoryFiles")
+// GraphQL pathway for reading file collections
+// File collections are stored in Redis hash maps (FileStoreMap:ctx:<contextId>
 // Returns file collection as JSON array string for backward compatibility with Labeeb
 
 import { loadFileCollection } from '../../../../lib/fileUtils.js';
diff --git a/pathways/system/entity/memory/sys_memory_manager.js b/pathways/system/entity/memory/sys_memory_manager.js
index 2c5866cc..ce5b4ee4 100644
--- a/pathways/system/entity/memory/sys_memory_manager.js
+++ b/pathways/system/entity/memory/sys_memory_manager.js
@@ -52,18 +52,23 @@ export default {
                     await callPathway('sys_save_memory', { ...args, aiMemory: AI_MEMORY_DEFAULTS });
                 } else {
                     // Upgrade memory to current version
-                    const normalizePromises = Object.keys(parsedMemory).map(async (section) => {
+                    // Filter out memoryFiles (deprecated - file collections are now stored separately in Redis hash maps)
+                    const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser', 'memoryContext', 'memoryVersion'];
+                    const sectionsToNormalize = Object.keys(parsedMemory).filter(section => validSections.includes(section));
+                    
+                    const normalizePromises = sectionsToNormalize.map(async (section) => {
                         const normalized = await normalizeMemoryFormat(args, parsedMemory[section]);
                         return [section, normalized];
                     });
                     
                     const normalizedResults = await Promise.all(normalizePromises);
+                    const upgradedMemory = {};
                     normalizedResults.forEach(([section, normalized]) => {
-                        parsedMemory[section] = normalized;
+                        upgradedMemory[section] = normalized;
                     });
                     
-                    parsedMemory.memoryVersion = MEMORY_VERSION;
-                    await callPathway('sys_save_memory', { ...args, aiMemory: JSON.stringify(parsedMemory) });
+                    upgradedMemory.memoryVersion = MEMORY_VERSION;
+                    await callPathway('sys_save_memory', { ...args, aiMemory: JSON.stringify(upgradedMemory) });
                 }
             }
 

From a7ea03cd2f3acc35b57b8cb4799b95178f5813e6 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Thu, 18 Dec 2025 17:18:44 -0700
Subject: [PATCH 13/27] refactor: improve file existence checks and cleanup
 logic in Redis handler

- Enhanced the `getFileStoreMap` function to clarify the logic for checking file existence in primary and GCS backup storage, ensuring accurate cleanup decisions.
- Updated the `removeFromFileStoreMap` function to handle scoped hash formats, allowing for more flexible removal of entries from both unscoped and context-scoped maps.
- Improved logging messages to provide clearer context regarding file operations and potential issues during cleanup.
---
 helper-apps/cortex-file-handler/src/redis.js  | 77 +++++++++++++------
 .../features/tools/fileCollection.test.js     | 17 ++--
 2 files changed, 61 insertions(+), 33 deletions(-)

diff --git a/helper-apps/cortex-file-handler/src/redis.js b/helper-apps/cortex-file-handler/src/redis.js
index 363d8a51..f2d6796b 100644
--- a/helper-apps/cortex-file-handler/src/redis.js
+++ b/helper-apps/cortex-file-handler/src/redis.js
@@ -344,36 +344,34 @@ const getFileStoreMap = async (hash, skipLazyCleanup = false, contextId = null)
             const storageService = new StorageService();
 
             let shouldRemove = false;
+            let primaryExists = false;
+            let gcsExists = false;
 
             // Check primary storage
             if (parsedValue?.url) {
-              const exists = await storageService.fileExists(parsedValue.url);
-              if (!exists) {
+              primaryExists = await storageService.fileExists(parsedValue.url);
+              if (!primaryExists) {
                 console.log(
-                  `Lazy cleanup: Primary storage file missing for key ${key}: ${parsedValue.url}`,
+                  `Lazy cleanup: Primary storage file missing for hash ${hash}: ${parsedValue.url}`,
                 );
-                shouldRemove = true;
               }
             }
 
-            // Check GCS backup if primary is missing
-            if (
-              shouldRemove &&
-              parsedValue?.gcs &&
-              storageService.backupProvider
-            ) {
-              const gcsExists = await storageService.fileExists(
-                parsedValue.gcs,
-              );
+            // Check GCS backup if available
+            if (parsedValue?.gcs && storageService.backupProvider) {
+              gcsExists = await storageService.fileExists(parsedValue.gcs);
               if (gcsExists) {
-                // GCS backup exists, so don't remove the entry
-                shouldRemove = false;
                 console.log(
-                  `Lazy cleanup: GCS backup found for key ${key}, keeping entry`,
+                  `Lazy cleanup: GCS backup found for hash ${hash}, keeping entry`,
                 );
               }
             }
 
+            // Only remove if both primary and backup are missing
+            if (!primaryExists && !gcsExists) {
+              shouldRemove = true;
+            }
+
             // Remove stale entry if both primary and backup are missing
             // Need to extract contextId from the key if it was scoped
             if (shouldRemove) {
@@ -387,7 +385,7 @@ const getFileStoreMap = async (hash, skipLazyCleanup = false, contextId = null)
               return null; // Return null since file no longer exists
             }
           } catch (error) {
-            console.log(`Lazy cleanup error for key ${key}: ${error.message}`);
+            console.log(`Lazy cleanup error for hash ${hash}: ${error.message}`);
             // If cleanup fails, return the original value to avoid breaking functionality
           }
         }
@@ -408,21 +406,51 @@ const getFileStoreMap = async (hash, skipLazyCleanup = false, contextId = null)
 // Function to remove key from "FileStoreMap" hash map
 // If contextId is provided, removes from context-scoped map
 // Otherwise removes from unscoped map
-// Hash is always the raw hash (no scoping in the key itself)
+// Hash can be either raw hash or scoped key format (hash:ctx:contextId)
+// If scoped format is provided, extracts base hash and removes both scoped and legacy keys
 const removeFromFileStoreMap = async (hash, contextId = null) => {
   try {
     if (!hash) {
       return;
     }
     
+    // Extract base hash if hash is in scoped format (hash:ctx:contextId)
+    let baseHash = hash;
+    let extractedContextId = contextId;
+    if (String(hash).includes(":ctx:")) {
+      const parts = String(hash).split(":ctx:");
+      baseHash = parts[0];
+      if (parts.length > 1 && !extractedContextId) {
+        extractedContextId = parts[1];
+      }
+    }
+    
     let result = 0;
-    if (contextId) {
-      // Remove from context-scoped map
+    
+    // First, try to delete from unscoped map (in case scoped key was stored there)
+    if (!contextId) {
+      // Remove from unscoped map (including scoped key format if present)
+      result = await client.hdel("FileStoreMap", hash);
+      // Also try removing with base hash if hash was scoped
+      if (hash !== baseHash) {
+        const baseResult = await client.hdel("FileStoreMap", baseHash);
+        if (baseResult > 0) {
+          result = baseResult;
+        }
+      }
+    }
+    
+    // Also try to delete from context-scoped map if we extracted a contextId
+    if (extractedContextId) {
+      const contextMapKey = `FileStoreMap:ctx:${extractedContextId}`;
+      const contextResult = await client.hdel(contextMapKey, baseHash);
+      if (contextResult > 0) {
+        result = contextResult;
+      }
+    } else if (contextId) {
+      // If contextId was provided explicitly, delete from context-scoped map
       const contextMapKey = `FileStoreMap:ctx:${contextId}`;
       result = await client.hdel(contextMapKey, hash);
-    } else {
-      // Remove from unscoped map
-      result = await client.hdel("FileStoreMap", hash);
     }
     if (result > 0) {
       console.log(`The hash ${hash} was removed successfully`);
@@ -430,7 +458,6 @@ const removeFromFileStoreMap = async (hash, contextId = null) => {
 
     // Always try to clean up legacy container-scoped entry as well.
     // This ensures we don't leave orphaned legacy keys behind.
-    const baseHash = hash;
     // Only attempt legacy cleanup if baseHash doesn't contain a colon (not already scoped)
     if (!String(baseHash).includes(":")) {
       const defaultContainerName = getDefaultContainerName();
@@ -444,7 +471,7 @@ const removeFromFileStoreMap = async (hash, contextId = null) => {
     }
 
     if (result === 0) {
-      console.log(`The key ${key} does not exist (may have been migrated or already deleted)`);
+      console.log(`The hash ${hash} does not exist (may have been migrated or already deleted)`);
     }
   } catch (error) {
     console.error(`Error removing key from FileStoreMap: ${error}`);
diff --git a/tests/integration/features/tools/fileCollection.test.js b/tests/integration/features/tools/fileCollection.test.js
index cc1bcbc4..acf55a3b 100644
--- a/tests/integration/features/tools/fileCollection.test.js
+++ b/tests/integration/features/tools/fileCollection.test.js
@@ -352,11 +352,11 @@ test('File collection: List with filters and sorting', async t => {
     }
 });
 
-test('Memory system: file collections excluded from memoryAll', async t => {
+test('Memory system: file collections excluded from memoryAll (memoryFiles deprecated)', async t => {
     const contextId = createTestContext();
     
     try {
-        // Save a file collection directly to Redis
+        // Save a file collection directly to Redis (file collections are stored separately, not in memory system)
         const { saveFileCollection } = await import('../../../../lib/fileUtils.js');
         await saveFileCollection(contextId, null, [{
             id: 'test-1',
@@ -371,7 +371,7 @@ test('Memory system: file collections excluded from memoryAll', async t => {
             aiMemory: 'Test memory content'
         });
         
-        // Read all memory - should not include file collections
+        // Read all memory - should not include file collections (memoryFiles section is deprecated and not returned)
         const allMemory = await callPathway('sys_read_memory', {
             contextId,
             section: 'memoryAll'
@@ -379,7 +379,7 @@ test('Memory system: file collections excluded from memoryAll', async t => {
         
         const parsed = JSON.parse(allMemory);
         t.truthy(parsed.memorySelf);
-        t.falsy(parsed.memoryFiles);
+        t.falsy(parsed.memoryFiles); // memoryFiles is deprecated - file collections are stored in Redis hash maps
         
         // But should be accessible via loadFileCollection
         const files = await loadFileCollection(contextId, null, false);
@@ -418,11 +418,11 @@ test('Memory system: file collections not cleared by memoryAll clear', async t =
     }
 });
 
-test('Memory system: file collections ignored in memoryAll save', async t => {
+test('Memory system: file collections ignored in memoryAll save (memoryFiles deprecated)', async t => {
     const contextId = createTestContext();
     
     try {
-        // Save file collection first directly to Redis
+        // Save file collection first directly to Redis (file collections are stored separately, not in memory system)
         const { saveFileCollection } = await import('../../../../lib/fileUtils.js');
         await saveFileCollection(contextId, null, [{
             id: 'original',
@@ -430,7 +430,8 @@ test('Memory system: file collections ignored in memoryAll save', async t => {
             displayFilename: 'original.jpg'
         }]);
         
-        // Try to save all memory with memoryFiles included (should be ignored)
+        // Try to save all memory with memoryFiles included (should be ignored - memoryFiles is deprecated)
+        // File collections are now stored in Redis hash maps (FileStoreMap:ctx:<contextId>), not in memory system
         await callPathway('sys_save_memory', {
             contextId,
             section: 'memoryAll',
@@ -444,7 +445,7 @@ test('Memory system: file collections ignored in memoryAll save', async t => {
             })
         });
         
-        // Verify original files are still there (not overwritten - memoryFiles is ignored)
+        // Verify original files are still there (not overwritten - memoryFiles section is ignored by sys_save_memory)
         const files = await loadFileCollection(contextId, null, false);
         t.is(files.length, 1);
         t.is(files[0].displayFilename, 'original.jpg');

From 2be19224c2cf247479a297b41696bce5cbd11ab3 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Thu, 18 Dec 2025 17:19:04 -0700
Subject: [PATCH 14/27] chore: update version to 2.7.0 in package.json and
 package-lock.json

- Bumped the version of @aj-archipelago/cortex-file-handler from 2.6.4 to 2.7.0 in both package.json and package-lock.json to reflect the latest changes and improvements.
---
 helper-apps/cortex-file-handler/package-lock.json | 4 ++--
 helper-apps/cortex-file-handler/package.json      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/helper-apps/cortex-file-handler/package-lock.json b/helper-apps/cortex-file-handler/package-lock.json
index 21423467..ac4a2558 100644
--- a/helper-apps/cortex-file-handler/package-lock.json
+++ b/helper-apps/cortex-file-handler/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@aj-archipelago/cortex-file-handler",
-  "version": "2.6.3",
+  "version": "2.7.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@aj-archipelago/cortex-file-handler",
-      "version": "2.6.3",
+      "version": "2.7.0",
       "dependencies": {
         "@azure/storage-blob": "^12.13.0",
         "@distube/ytdl-core": "^4.14.3",
diff --git a/helper-apps/cortex-file-handler/package.json b/helper-apps/cortex-file-handler/package.json
index f6de72d1..39ce23de 100644
--- a/helper-apps/cortex-file-handler/package.json
+++ b/helper-apps/cortex-file-handler/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex-file-handler",
-  "version": "2.6.4",
+  "version": "2.7.0",
   "description": "File handling service for Cortex - handles file uploads, media chunking, and document processing",
   "type": "module",
   "main": "src/index.js",

From 35d5a26f74794188c310fbe6d4469850389d5d32 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Thu, 18 Dec 2025 18:02:14 -0700
Subject: [PATCH 15/27] refactor: streamline Azure container setup in file
 handler

- Removed the setup for Azure test environment from the GitHub Actions workflow, consolidating GCS and Azure tests into a single step.
- Updated the Azure container setup script to always create default test containers and deduplicate container names from the environment variable, improving clarity and efficiency in container management.
---
 .../workflows/cortex-file-handler-test.yml    |  8 +------
 .../scripts/setup-azure-container.js          | 23 +++++++++++--------
 2 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/cortex-file-handler-test.yml b/.github/workflows/cortex-file-handler-test.yml
index 7c3cf79e..fa11598a 100644
--- a/.github/workflows/cortex-file-handler-test.yml
+++ b/.github/workflows/cortex-file-handler-test.yml
@@ -46,15 +46,9 @@ jobs:
     - name: Setup GCS test environment
       run: cp .env.test.gcs.ci .env.test.gcs
     
-    - name: Setup Azure test environment
-      run: cp .env.test.azure.ci .env.test.azure
-    
-    - name: Run GCS tests
+    - name: Run GCS tests (includes Azure tests)
       run: npm run test:gcs
     
-    - name: Run Azure tests
-      run: npm run test:azure
-    
     - name: Run tests
       run: npm test
       env:
diff --git a/helper-apps/cortex-file-handler/scripts/setup-azure-container.js b/helper-apps/cortex-file-handler/scripts/setup-azure-container.js
index ded70922..d99611bf 100644
--- a/helper-apps/cortex-file-handler/scripts/setup-azure-container.js
+++ b/helper-apps/cortex-file-handler/scripts/setup-azure-container.js
@@ -9,21 +9,24 @@ async function createContainers() {
     if (!connectionString) {
       throw new Error("AZURE_STORAGE_CONNECTION_STRING environment variable is required");
     }
-    
-    if (!containerNames) {
-      throw new Error("AZURE_STORAGE_CONTAINER_NAME environment variable is required");
-    }
 
     const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString);
     
-    // Parse container names from environment variable
-    const containers = containerNames.split(',').map(name => name.trim());
-    console.log(`Creating containers: ${containers.join(', ')}`);
+    // Always create test containers that are used in tests
+    const testContainers = ["default", "test-container"];
+    
+    // Also create containers from environment variable if provided
+    const envContainers = containerNames 
+      ? containerNames.split(',').map(name => name.trim()).filter(name => name)
+      : [];
+    
+    // Combine and deduplicate container names
+    const allContainers = [...new Set([...testContainers, ...envContainers])];
+    
+    console.log(`Creating containers: ${allContainers.join(', ')}`);
 
     // Create each container
-    for (const containerName of containers) {
-      if (!containerName) continue; // Skip empty names
-      
+    for (const containerName of allContainers) {
       try {
         const containerClient = blobServiceClient.getContainerClient(containerName);
         await containerClient.create();

From ddf0466c411d8442828b44059879aa2347302a33 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Thu, 18 Dec 2025 22:24:27 -0700
Subject: [PATCH 16/27] refactor: enhance Azure container name retrieval logic

- Updated the `getContainerName` function to improve handling of environment variable values, ensuring a default value of "cortextempfiles" is returned if the variable is not set, empty, or contains the string "undefined".
- Added fallback logic to handle legacy comma-separated values and ensure robustness in container name retrieval.
- Increased timeout settings in tests for image content processing to accommodate longer processing times.
---
 helper-apps/cortex-file-handler/src/constants.js       | 10 +++++++++-
 .../rest/message_content_compliance.test.js            |  3 ++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/helper-apps/cortex-file-handler/src/constants.js b/helper-apps/cortex-file-handler/src/constants.js
index c1d14f0b..00df7b02 100644
--- a/helper-apps/cortex-file-handler/src/constants.js
+++ b/helper-apps/cortex-file-handler/src/constants.js
@@ -136,7 +136,12 @@ export const AZURITE_ACCOUNT_NAME = "devstoreaccount1";
 // Get single container name from environment variable
 // CFH operates on a single Azure container and single GCS bucket
 export const getContainerName = () => {
-  const envValue = process.env.AZURE_STORAGE_CONTAINER_NAME || "cortextempfiles";
+  const envValue = process.env.AZURE_STORAGE_CONTAINER_NAME;
+  
+  // Default to cortextempfiles if not set, empty, or the string "undefined"
+  if (!envValue || (typeof envValue === 'string' && envValue.trim() === "") || envValue === "undefined") {
+    return "cortextempfiles";
+  }
   
   // Handle legacy comma-separated values (take the last one)
   if (envValue.includes(",")) {
@@ -151,6 +156,8 @@ export const getContainerName = () => {
       );
       return containerName;
     }
+    // If all containers were empty after splitting, fall back to default
+    return "cortextempfiles";
   }
   
   return envValue;
@@ -161,5 +168,6 @@ export const getDefaultContainerName = () => {
   return getContainerName();
 };
 
+// Export constant - evaluated at module load time, but getContainerName() handles defaults
 export const AZURE_STORAGE_CONTAINER_NAME = getContainerName();
 export const GCS_BUCKETNAME = process.env.GCS_BUCKETNAME || "cortextempfiles";
diff --git a/tests/integration/rest/message_content_compliance.test.js b/tests/integration/rest/message_content_compliance.test.js
index 40786ed0..ae5ff194 100644
--- a/tests/integration/rest/message_content_compliance.test.js
+++ b/tests/integration/rest/message_content_compliance.test.js
@@ -612,6 +612,7 @@ test('POST /chat/completions - complex conversation with all content variations'
 test('POST /chat/completions - user message with image content part', async (t) => {
   // Spec: User messages can have image content parts
   // Note: This test may timeout if image URL validation fails, but it tests the content structure
+  t.timeout(30000); // Increase timeout to 30s for image processing
   const response = await got.post(`${API_BASE}/chat/completions`, {
     json: {
       model: 'gpt-4.1',
@@ -634,7 +635,7 @@ test('POST /chat/completions - user message with image content part', async (t)
     responseType: 'json',
     throwHttpErrors: false,
     timeout: {
-      request: 5000
+      request: 30000 // Increase timeout to 30s for image processing
     }
   });
 

From 510af41687b3947b6d9ca553ad55420727303f28 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Thu, 18 Dec 2025 22:54:43 -0700
Subject: [PATCH 17/27] refactor: improve Azure blob tag handling for local
 emulator support

- Added checks to handle scenarios where Azurite (local emulator) may not support blob tags, ensuring that tag updates are skipped gracefully without throwing errors.
- Enhanced logging to provide clear warnings when tag updates fail in test environments, allowing operations to continue smoothly.
- Updated the `updateBlobTags` method to account for potential failures in tag operations, improving robustness in both local and production environments.
---
 .../services/storage/AzureStorageProvider.js  | 27 +++++++++++++++++--
 .../src/services/storage/StorageService.js    |  9 ++++++-
 2 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js b/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js
index e93c9ba3..fd6dc871 100644
--- a/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js
+++ b/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js
@@ -434,6 +434,11 @@ export class AzureStorageProvider extends StorageProvider {
     const { containerClient } = await this.getBlobClient();
     const blockBlobClient = containerClient.getBlockBlobClient(blobName);
     
+    // Check if we're using Azurite (local emulator) which may not fully support blob tags
+    const isAzurite = this.connectionString?.includes('UseDevelopmentStorage=true') || 
+                       this.connectionString?.includes('127.0.0.1') ||
+                       this.connectionString?.includes('localhost');
+    
     // Get current tags first (may return empty object if no tags exist)
     let currentTags = {};
     try {
@@ -443,7 +448,12 @@ export class AzureStorageProvider extends StorageProvider {
         currentTags = tagsResponse.tags || tagsResponse;
       }
     } catch (error) {
-      // If getTags fails (e.g., no tags exist), start with empty object
+      // If getTags fails (e.g., no tags exist or Azurite doesn't support it), start with empty object
+      if (isAzurite) {
+        // Azurite may not support blob tags - this is expected, so we'll skip tag updates
+        console.log(`[Azurite] Blob tags not supported, skipping tag update for ${blobName}`);
+        return;
+      }
       currentTags = {};
     }
     
@@ -453,6 +463,19 @@ export class AzureStorageProvider extends StorageProvider {
       retention: retention
     };
     
-    await blockBlobClient.setTags(updatedTags);
+    try {
+      await blockBlobClient.setTags(updatedTags);
+    } catch (error) {
+      // If setTags fails (e.g., Azurite doesn't support it), log but don't throw
+      // This allows the operation to continue even if tags aren't supported
+      // In test environments, we'll be lenient and not throw errors for tag operations
+      const isTestEnv = process.env.NODE_ENV === 'test' || isAzurite;
+      if (isTestEnv) {
+        console.log(`[Test/Azurite] Blob tags not supported, skipping tag update for ${blobName}: ${error.message}`);
+        return;
+      }
+      // For real Azure in production, re-throw the error as it's unexpected
+      throw error;
+    }
   }
 }
diff --git a/helper-apps/cortex-file-handler/src/services/storage/StorageService.js b/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
index 3c038703..5077c4e6 100644
--- a/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
+++ b/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
@@ -309,8 +309,15 @@ export class StorageService {
     }
 
     // Update blob index tag
+    // Note: This may fail in Azurite (local emulator) which doesn't fully support blob tags
+    // We'll continue with the operation even if tag update fails
     context.log?.(`Updating blob index tag for ${blobName} to ${retention}`);
-    await provider.updateBlobTags(blobName, retention);
+    try {
+      await provider.updateBlobTags(blobName, retention);
+    } catch (error) {
+      // Log warning but continue - blob tags may not be supported in test environments (e.g., Azurite)
+      context.log?.(`Warning: Failed to update blob tags for ${blobName}: ${error.message}. Continuing with operation.`);
+    }
 
     // Generate new short-lived URL
     const { containerClient } = await provider.getBlobClient();

From ac99712e8e332f7766262b8feaf6ab53dd2e192e Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Thu, 18 Dec 2025 23:23:42 -0700
Subject: [PATCH 18/27] refactor: improve blob tag handling and URL generation
 in StorageService

- Added support for checking if the provider can handle blob tag operations, enhancing compatibility with local emulators like Azurite.
- Streamlined the extraction of blob names and generation of short-lived URLs, ensuring operations continue smoothly even when tag updates fail.
- Improved logging for better visibility into tag update failures and fallback mechanisms for providers that do not support blob tags.
---
 .../src/services/storage/StorageService.js    | 104 ++++++++++--------
 1 file changed, 60 insertions(+), 44 deletions(-)

diff --git a/helper-apps/cortex-file-handler/src/services/storage/StorageService.js b/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
index 5077c4e6..7edb1d16 100644
--- a/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
+++ b/helper-apps/cortex-file-handler/src/services/storage/StorageService.js
@@ -302,54 +302,70 @@ export class StorageService {
     // Always use primary provider - single container only
     const provider = this.primaryProvider;
     
-    // Extract blob name from URL
-    const blobName = provider.extractBlobNameFromUrl(hashResult.url);
-    if (!blobName) {
-      throw new Error(`Could not extract blob name from URL: ${hashResult.url}`);
-    }
+    // Check if provider supports blob tag operations (Azure only)
+    const supportsBlobTags = typeof provider.extractBlobNameFromUrl === 'function' && 
+                             typeof provider.updateBlobTags === 'function' &&
+                             typeof provider.getBlobClient === 'function' &&
+                             typeof provider.generateShortLivedSASToken === 'function';
+    
+    let shortLivedUrl = hashResult.shortLivedUrl || hashResult.url;
+    let convertedResult = hashResult.converted || null;
+
+    if (supportsBlobTags) {
+      // Extract blob name from URL
+      const blobName = provider.extractBlobNameFromUrl(hashResult.url);
+      if (!blobName) {
+        throw new Error(`Could not extract blob name from URL: ${hashResult.url}`);
+      }
 
-    // Update blob index tag
-    // Note: This may fail in Azurite (local emulator) which doesn't fully support blob tags
-    // We'll continue with the operation even if tag update fails
-    context.log?.(`Updating blob index tag for ${blobName} to ${retention}`);
-    try {
-      await provider.updateBlobTags(blobName, retention);
-    } catch (error) {
-      // Log warning but continue - blob tags may not be supported in test environments (e.g., Azurite)
-      context.log?.(`Warning: Failed to update blob tags for ${blobName}: ${error.message}. Continuing with operation.`);
-    }
-
-    // Generate new short-lived URL
-    const { containerClient } = await provider.getBlobClient();
-    const shortLivedSasToken = provider.generateShortLivedSASToken(containerClient, blobName, 5);
-    const urlObj = new URL(hashResult.url);
-    const baseUrl = `${urlObj.protocol}//${urlObj.host}${urlObj.pathname}`;
-    const shortLivedUrl = `${baseUrl}?${shortLivedSasToken}`;
-
-    // Handle converted file if it exists
-    let convertedResult = null;
-    if (hashResult.converted?.url) {
-      context.log?.(`Updating blob index tag for converted file to ${retention}`);
-      const convertedBlobName = provider.extractBlobNameFromUrl(hashResult.converted.url);
-      if (convertedBlobName) {
-        try {
-          await provider.updateBlobTags(convertedBlobName, retention);
-          const convertedUrlObj = new URL(hashResult.converted.url);
-          const convertedBaseUrl = `${convertedUrlObj.protocol}//${convertedUrlObj.host}${convertedUrlObj.pathname}`;
-          const convertedShortLivedSasToken = provider.generateShortLivedSASToken(containerClient, convertedBlobName, 5);
-          const convertedShortLivedUrl = `${convertedBaseUrl}?${convertedShortLivedSasToken}`;
-          convertedResult = {
-            url: hashResult.converted.url,
-            shortLivedUrl: convertedShortLivedUrl,
-            gcs: hashResult.converted.gcs
-          };
-        } catch (error) {
-          context.log?.(`Warning: Failed to update converted file tag: ${error.message}`);
+      // Update blob index tag
+      // Note: This may fail in Azurite (local emulator) which doesn't fully support blob tags
+      // We'll continue with the operation even if tag update fails
+      context.log?.(`Updating blob index tag for ${blobName} to ${retention}`);
+      try {
+        await provider.updateBlobTags(blobName, retention);
+      } catch (error) {
+        // Log warning but continue - blob tags may not be supported in test environments (e.g., Azurite)
+        context.log?.(`Warning: Failed to update blob tags for ${blobName}: ${error.message}. Continuing with operation.`);
+      }
+
+      // Generate new short-lived URL
+      const { containerClient } = await provider.getBlobClient();
+      const shortLivedSasToken = provider.generateShortLivedSASToken(containerClient, blobName, 5);
+      const urlObj = new URL(hashResult.url);
+      const baseUrl = `${urlObj.protocol}//${urlObj.host}${urlObj.pathname}`;
+      shortLivedUrl = `${baseUrl}?${shortLivedSasToken}`;
+
+      // Handle converted file if it exists
+      if (hashResult.converted?.url) {
+        context.log?.(`Updating blob index tag for converted file to ${retention}`);
+        const convertedBlobName = provider.extractBlobNameFromUrl(hashResult.converted.url);
+        if (convertedBlobName) {
+          try {
+            await provider.updateBlobTags(convertedBlobName, retention);
+            const convertedUrlObj = new URL(hashResult.converted.url);
+            const convertedBaseUrl = `${convertedUrlObj.protocol}//${convertedUrlObj.host}${convertedUrlObj.pathname}`;
+            const convertedShortLivedSasToken = provider.generateShortLivedSASToken(containerClient, convertedBlobName, 5);
+            const convertedShortLivedUrl = `${convertedBaseUrl}?${convertedShortLivedSasToken}`;
+            convertedResult = {
+              url: hashResult.converted.url,
+              shortLivedUrl: convertedShortLivedUrl,
+              gcs: hashResult.converted.gcs
+            };
+          } catch (error) {
+            context.log?.(`Warning: Failed to update converted file tag: ${error.message}`);
+            convertedResult = hashResult.converted;
+          }
+        } else {
           convertedResult = hashResult.converted;
         }
-      } else {
-        convertedResult = hashResult.converted;
       }
+    } else {
+      // For providers that don't support blob tags (e.g., LocalStorageProvider),
+      // just use the existing URLs - retention is tracked in Redis only
+      context.log?.(`Provider does not support blob tags, updating Redis only`);
+      shortLivedUrl = hashResult.shortLivedUrl || hashResult.url;
+      convertedResult = hashResult.converted || null;
     }
 
     // Update Redis with new information (including shortLivedUrl and permanent flag)

From 21093728052b843b96473e242723f1b5f7800b5c Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Fri, 19 Dec 2025 09:20:41 -0700
Subject: [PATCH 19/27] feat: implement encryption and decryption for file
 metadata in Redis

- Added functions to write and read file data with encryption for sensitive fields (tags and notes) using a context key.
- Updated existing file handling functions to support optional context key for encryption and decryption during metadata updates.
- Enhanced tests to validate encryption functionality, ensuring sensitive data is securely stored and can be decrypted correctly.
- Ensured core fields remain unencrypted for accessibility while maintaining data integrity.
---
 lib/fileUtils.js                              | 243 ++++++++++-----
 .../entity/files/sys_update_file_metadata.js  |   7 +-
 .../system/entity/tools/sys_tool_editfile.js  |   8 +-
 .../entity/tools/sys_tool_file_collection.js  |   4 +-
 .../features/tools/fileCollection.test.js     | 291 ++++++++++++++++++
 5 files changed, 465 insertions(+), 88 deletions(-)

diff --git a/lib/fileUtils.js b/lib/fileUtils.js
index e17b8814..91e802ef 100644
--- a/lib/fileUtils.js
+++ b/lib/fileUtils.js
@@ -13,6 +13,7 @@ import path from 'path';
 import FormData from 'form-data';
 import xxhash from 'xxhash-wasm';
 import mime from 'mime-types';
+import { encrypt, decrypt } from './crypto.js';
 
 const pipeline = promisify(stream.pipeline);
 const MEDIA_API_URL = config.get('whisperMediaApiUrl');
@@ -483,35 +484,154 @@ function isFileInCollection(inCollection, chatId = null) {
  * @param {string|null} chatId - Optional chat ID to filter files by (if provided, only includes files with '*' or this chatId in inCollection)
  * @returns {Promise<Array>} File collection array
  */
+/**
+ * Write file data to Redis with encryption of sensitive fields
+ * Follows the same pattern as setvWithDoubleEncryption - skips encryption for empty values
+ * @param {Object} redisClient - Redis client
+ * @param {string} contextMapKey - Redis hash map key
+ * @param {string} hash - File hash (key in hash map)
+ * @param {Object} fileData - File data object
+ * @param {string} contextKey - Optional context key for encryption
+ */
+async function writeFileDataToRedis(redisClient, contextMapKey, hash, fileData, contextKey = null) {
+    const dataToStore = { ...fileData };
+    
+    // Encrypt sensitive fields if contextKey is provided (same pattern as memory encryption)
+    if (contextKey && contextKey.trim() !== '') {
+        // Encrypt tags (array of strings) - skip if empty (consistent with memory encryption)
+        if (dataToStore.tags && Array.isArray(dataToStore.tags) && dataToStore.tags.length > 0) {
+            try {
+                const tagsJson = JSON.stringify(dataToStore.tags);
+                const encrypted = encrypt(tagsJson, contextKey);
+                if (encrypted !== null) {
+                    dataToStore.tags = encrypted;
+                }
+                // If encryption fails, continue with unencrypted (same pattern as memory)
+            } catch (error) {
+                logger.warn(`Failed to encrypt tags: ${error.message}`);
+            }
+        }
+        
+        // Encrypt notes (string) - skip if empty (consistent with memory encryption)
+        if (dataToStore.notes && typeof dataToStore.notes === 'string' && dataToStore.notes.trim() !== '') {
+            try {
+                const encrypted = encrypt(dataToStore.notes, contextKey);
+                if (encrypted !== null) {
+                    dataToStore.notes = encrypted;
+                }
+                // If encryption fails, continue with unencrypted (same pattern as memory)
+            } catch (error) {
+                logger.warn(`Failed to encrypt notes: ${error.message}`);
+            }
+        }
+    }
+    
+    await redisClient.hset(contextMapKey, hash, JSON.stringify(dataToStore));
+}
+
+/**
+ * Read file data from Redis with decryption of sensitive fields
+ * Follows the same pattern as getvWithDoubleDecryption - tries decrypt, falls back to original
+ * @param {string} dataStr - JSON string from Redis
+ * @param {string} contextKey - Optional context key for decryption
+ * @returns {Object|null} Parsed and decrypted file data, or null if invalid
+ */
+function readFileDataFromRedis(dataStr, contextKey = null) {
+    if (!dataStr) return null;
+    
+    try {
+        const fileData = JSON.parse(dataStr);
+        
+        // Decrypt sensitive fields if contextKey is provided (same pattern as memory decryption)
+        if (contextKey && contextKey.trim() !== '') {
+            // Decrypt tags (array of strings)
+            if (fileData.tags !== undefined && fileData.tags !== null) {
+                // If already an array, it's unencrypted legacy data - keep as-is
+                if (!Array.isArray(fileData.tags) && typeof fileData.tags === 'string') {
+                    // Try to decrypt (encrypted strings have ':' separator from IV)
+                    if (fileData.tags.includes(':')) {
+                        try {
+                            const decrypted = decrypt(fileData.tags, contextKey);
+                            if (decrypted !== null) {
+                                // Try to parse as JSON array, fallback to array with single string
+                                try {
+                                    fileData.tags = JSON.parse(decrypted);
+                                } catch (e) {
+                                    fileData.tags = [decrypted];
+                                }
+                            }
+                            // If decryption returns null, keep original (might be unencrypted legacy data)
+                        } catch (error) {
+                            // Decryption failed, keep as-is (unencrypted legacy data)
+                        }
+                    } else {
+                        // No ':' means not encrypted - try parsing as JSON, fallback to array
+                        try {
+                            fileData.tags = JSON.parse(fileData.tags);
+                        } catch (e) {
+                            fileData.tags = [fileData.tags];
+                        }
+                    }
+                }
+            } else {
+                fileData.tags = [];
+            }
+            
+            // Decrypt notes (string)
+            if (fileData.notes !== undefined && fileData.notes !== null) {
+                if (typeof fileData.notes === 'string' && fileData.notes.includes(':')) {
+                    // Try to decrypt
+                    try {
+                        const decrypted = decrypt(fileData.notes, contextKey);
+                        if (decrypted !== null) {
+                            fileData.notes = decrypted;
+                        }
+                        // If decryption returns null, keep original (might be unencrypted legacy data)
+                    } catch (error) {
+                        // Decryption failed, keep as-is (unencrypted legacy data)
+                    }
+                }
+                // If not encrypted (no ':'), keep as-is (legacy unencrypted data)
+            } else {
+                fileData.notes = '';
+            }
+        }
+        
+        return fileData;
+    } catch (e) {
+        return null;
+    }
+}
+
 /**
  * Parse raw Redis hash map data into file objects (without filtering)
  * @param {Object} allFiles - Redis HGETALL result {hash: fileDataStr}
+ * @param {string} contextKey - Optional context key for decryption
  * @returns {Array} Array of parsed file data objects (includes inCollection metadata)
  */
-function parseRawFileData(allFiles) {
+function parseRawFileData(allFiles, contextKey = null) {
     return Object.entries(allFiles).map(([hash, fileDataStr]) => {
-        try {
-            const fileData = JSON.parse(fileDataStr);
-            // Return parsed file data with hash and inCollection preserved for filtering
-            return {
-                id: fileData.id || `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
-                url: fileData.url,
-                gcs: fileData.gcs || null,
-                displayFilename: fileData.displayFilename || fileData.filename || null,
-                mimeType: fileData.mimeType || null,
-                tags: fileData.tags || [],
-                notes: fileData.notes || '',
-                hash: hash,
-                permanent: fileData.permanent || false,
-                addedDate: fileData.addedDate || fileData.timestamp || new Date().toISOString(),
-                lastAccessed: fileData.lastAccessed || fileData.timestamp || new Date().toISOString(),
-                // Preserve inCollection for filtering
-                inCollection: fileData.inCollection
-            };
-        } catch (e) {
-            // Skip invalid entries
+        const decryptedData = readFileDataFromRedis(fileDataStr, contextKey);
+        if (!decryptedData) {
             return null;
         }
+        
+        // Return parsed file data with hash and inCollection preserved for filtering
+        return {
+            id: decryptedData.id || `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
+            url: decryptedData.url,
+            gcs: decryptedData.gcs || null,
+            displayFilename: decryptedData.displayFilename || decryptedData.filename || null,
+            mimeType: decryptedData.mimeType || null,
+            tags: decryptedData.tags || [],
+            notes: decryptedData.notes || '',
+            hash: hash,
+            permanent: decryptedData.permanent || false,
+            addedDate: decryptedData.addedDate || decryptedData.timestamp || new Date().toISOString(),
+            lastAccessed: decryptedData.lastAccessed || decryptedData.timestamp || new Date().toISOString(),
+            // Preserve inCollection for filtering
+            inCollection: decryptedData.inCollection
+        };
     }).filter(Boolean);
 }
 
@@ -565,7 +685,8 @@ async function loadFileCollection(contextId, contextKey = null, useCache = true,
             const allFiles = await redisClient.hgetall(contextMapKey);
             
             // Parse raw file data (preserves inCollection metadata for filtering)
-            rawFiles = parseRawFileData(allFiles);
+            // Pass contextKey for decryption
+            rawFiles = parseRawFileData(allFiles, contextKey);
         }
     } catch (e) {
         // Collection doesn't exist yet or error reading, start with empty array
@@ -619,10 +740,11 @@ function normalizeInCollection(inCollection) {
  * @param {string} contextId - Context ID
  * @param {string} hash - File hash
  * @param {Object} metadata - Metadata to update (displayFilename, id, tags, notes, mimeType, addedDate, lastAccessed, permanent, inCollection)
+ * @param {string} contextKey - Optional context key for encryption
  * Note: Does NOT update CFH core fields (url, gcs, hash, filename) - those are managed by CFH
  * @returns {Promise<boolean>} True if successful
  */
-async function updateFileMetadata(contextId, hash, metadata) {
+async function updateFileMetadata(contextId, hash, metadata, contextKey = null) {
     if (!contextId || !hash) {
         return false;
     }
@@ -636,15 +758,7 @@ async function updateFileMetadata(contextId, hash, metadata) {
         const contextMapKey = `FileStoreMap:ctx:${contextId}`;
         // Get existing file data from CFH (if any)
         const existingDataStr = await redisClient.hget(contextMapKey, hash);
-        let existingData = {};
-        if (existingDataStr) {
-            try {
-                existingData = JSON.parse(existingDataStr);
-            } catch (e) {
-                // Invalid data, start fresh
-                existingData = {};
-            }
-        }
+        const existingData = readFileDataFromRedis(existingDataStr, contextKey) || {};
         
         // Merge CFH data with Cortex metadata
         // Only update Cortex-managed fields, preserve CFH fields (url, gcs, hash, filename)
@@ -672,8 +786,8 @@ async function updateFileMetadata(contextId, hash, metadata) {
             delete fileData.inCollection;
         }
         
-        // Write back to hash map (atomic operation)
-        await redisClient.hset(contextMapKey, hash, JSON.stringify(fileData));
+        // Write back to hash map (atomic operation) - encryption happens in helper
+        await writeFileDataToRedis(redisClient, contextMapKey, hash, fileData, contextKey);
         
         // Invalidate cache
         const cacheKey = getCollectionCacheKey(contextId, null);
@@ -723,8 +837,8 @@ async function saveFileCollection(contextId, contextKey, collection) {
             
             // Check if file actually changed
             if (currentDataStr) {
-                try {
-                    const currentData = JSON.parse(currentDataStr);
+                const currentData = readFileDataFromRedis(currentDataStr, contextKey);
+                if (currentData) {
                     // Compare metadata fields (ignore CFH fields like url, gcs, timestamp)
                     if (currentData.id === file.id &&
                         JSON.stringify(currentData.tags || []) === JSON.stringify(file.tags || []) &&
@@ -733,21 +847,12 @@ async function saveFileCollection(contextId, contextKey, collection) {
                         currentData.permanent === (file.permanent || false)) {
                         needsUpdate = false;
                     }
-                } catch (e) {
-                    // Invalid data, needs update
                 }
             }
             
             if (needsUpdate) {
                 // Get existing CFH data
-                let existingData = {};
-                if (currentDataStr) {
-                    try {
-                        existingData = JSON.parse(currentDataStr);
-                    } catch (e) {
-                        existingData = {};
-                    }
-                }
+                const existingData = readFileDataFromRedis(currentDataStr, contextKey) || {};
                 
                 // Merge CFH data with Cortex metadata
                 // Preserve all CFH fields (url, gcs, filename, displayFilename, etc.)
@@ -768,8 +873,8 @@ async function saveFileCollection(contextId, contextKey, collection) {
                     inCollection: ['*'] // Mark as global chat file (available to all chats)
                 };
                 
-                // Write back to hash map (atomic operation)
-                await redisClient.hset(contextMapKey, fileHash, JSON.stringify(fileData));
+                // Write back to hash map (atomic operation) - encryption happens in helper
+                await writeFileDataToRedis(redisClient, contextMapKey, fileHash, fileData, contextKey);
             }
         }
         
@@ -875,16 +980,7 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
                 const contextMapKey = `FileStoreMap:ctx:${contextId}`;
                 // Get existing file data from CFH (if any)
                 const existingDataStr = await redisClient.hget(contextMapKey, storageHash);
-                let existingData = {};
-                
-                if (existingDataStr) {
-                    try {
-                        existingData = JSON.parse(existingDataStr);
-                    } catch (e) {
-                        // Invalid data, start fresh
-                        existingData = {};
-                    }
-                }
+                const existingData = readFileDataFromRedis(existingDataStr, contextKey) || {};
                 
                 // Merge CFH data with Cortex metadata
                 // If file already exists with same hash, update metadata but keep the existing entry
@@ -907,8 +1003,8 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
                     hash: storageHash // Store the hash used as key (actual hash or generated from URL)
                 };
                 
-                // Write back to hash map (atomic operation) - same hash key, just update metadata
-                await redisClient.hset(contextMapKey, storageHash, JSON.stringify(fileData));
+                // Write back to hash map (atomic operation) - encryption happens in helper
+                await writeFileDataToRedis(redisClient, contextMapKey, storageHash, fileData, contextKey);
             }
         } catch (e) {
             // Log but don't fail - metadata update is best effort
@@ -1082,26 +1178,12 @@ async function syncFilesToCollection(chatHistory, contextId, contextKey = null)
                 // File not found in context-scoped map - check if CFH has it (context-scoped or unscoped)
                 // This handles the case where file was uploaded but not yet in this context's collection
                 const existingDataStr = await redisClient.hget(contextMapKey, file.hash);
-                let existingData = null;
-                
-                if (existingDataStr) {
-                    try {
-                        existingData = JSON.parse(existingDataStr);
-                    } catch (e) {
-                        // Invalid data, treat as new
-                    }
-                }
+                let existingData = readFileDataFromRedis(existingDataStr, contextKey);
                 
                 // Also check unscoped map (CFH might have written it there)
                 if (!existingData) {
                     const unscopedDataStr = await redisClient.hget("FileStoreMap", file.hash);
-                    if (unscopedDataStr) {
-                        try {
-                            existingData = JSON.parse(unscopedDataStr);
-                        } catch (e) {
-                            // Invalid data, treat as new
-                        }
-                    }
+                    existingData = readFileDataFromRedis(unscopedDataStr, contextKey);
                 }
                 
                 if (existingData) {
@@ -1121,7 +1203,8 @@ async function syncFilesToCollection(chatHistory, contextId, contextKey = null)
                         inCollection: ['*'] // Mark as global chat file (available to all chats)
                     };
                     
-                    await redisClient.hset(contextMapKey, file.hash, JSON.stringify(fileData));
+                    // Write to Redis - encryption happens in helper
+                    await writeFileDataToRedis(redisClient, contextMapKey, file.hash, fileData, contextKey);
                 } else {
                     // File doesn't exist in CFH - create minimal entry (file referenced in chat but not uploaded)
                     const mimeType = determineMimeTypeFromUrl(file.url, file.gcs, null);
@@ -1140,13 +1223,14 @@ async function syncFilesToCollection(chatHistory, contextId, contextKey = null)
                         inCollection: ['*'] // Mark as global chat file (available to all chats)
                     };
                     
-                    await redisClient.hset(contextMapKey, file.hash, JSON.stringify(fileData));
+                    // Write to Redis - encryption happens in helper
+                    await writeFileDataToRedis(redisClient, contextMapKey, file.hash, fileData, contextKey);
                 }
             } else if (file.hash) {
                 // File exists - update lastAccessed directly
                 await updateFileMetadata(contextId, file.hash, {
                     lastAccessed: new Date().toISOString()
-                });
+                }, contextKey);
             }
         }
         
@@ -2039,6 +2123,7 @@ export {
     getMimeTypeFromExtension,
     isTextMimeType,
     isFileInCollection,
+    writeFileDataToRedis,
     // Exported for testing
     extractFilenameFromUrl,
     ensureFilenameExtension,
diff --git a/pathways/system/entity/files/sys_update_file_metadata.js b/pathways/system/entity/files/sys_update_file_metadata.js
index d384c731..37e38d5c 100644
--- a/pathways/system/entity/files/sys_update_file_metadata.js
+++ b/pathways/system/entity/files/sys_update_file_metadata.js
@@ -13,13 +13,14 @@ export default {
         notes: { type: 'string' }, // Optional - no default
         mimeType: { type: 'string' }, // Optional - no default
         permanent: { type: 'boolean' }, // Optional - no default
-        inCollection: { type: 'array', items: { type: 'string' } } // Optional - array of chat IDs, or can be boolean true/false (normalized to ['*'] or removed)
+        inCollection: { type: 'array', items: { type: 'string' } }, // Optional - array of chat IDs, or can be boolean true/false (normalized to ['*'] or removed)
+        contextKey: `` // Optional - context key for encryption
     },
     model: 'oai-gpt4o',
     isMutation: true, // Declaratively mark this as a Mutation
 
     resolver: async (_parent, args, _contextValue, _info) => {
-        const { contextId, hash, displayFilename, tags, notes, mimeType, permanent, inCollection } = args;
+        const { contextId, hash, displayFilename, tags, notes, mimeType, permanent, inCollection, contextKey } = args;
         
         // Validate required parameters
         if (!contextId || !hash) {
@@ -54,7 +55,7 @@ export default {
             }
             
             // Update metadata (only Cortex-managed fields)
-            const success = await updateFileMetadata(contextId, hash, metadata);
+            const success = await updateFileMetadata(contextId, hash, metadata, contextKey);
             
             if (success) {
                 return JSON.stringify({ 
diff --git a/pathways/system/entity/tools/sys_tool_editfile.js b/pathways/system/entity/tools/sys_tool_editfile.js
index c7caf99b..11d35fc2 100644
--- a/pathways/system/entity/tools/sys_tool_editfile.js
+++ b/pathways/system/entity/tools/sys_tool_editfile.js
@@ -2,7 +2,7 @@
 // Entity tool that modifies existing files by replacing line ranges or exact string matches
 import logger from '../../../../lib/logger.js';
 import { axios } from '../../../../lib/requestExecutor.js';
-import { uploadFileToCloud, findFileInCollection, loadFileCollection, saveFileCollection, getMimeTypeFromFilename, resolveFileParameter, deleteFileByHash, isTextMimeType, updateFileMetadata, getCollectionCacheKey } from '../../../../lib/fileUtils.js';
+import { uploadFileToCloud, findFileInCollection, loadFileCollection, getMimeTypeFromFilename, resolveFileParameter, deleteFileByHash, isTextMimeType, updateFileMetadata, writeFileDataToRedis } from '../../../../lib/fileUtils.js';
 
 export default {
     prompt: [],
@@ -365,8 +365,8 @@ export default {
                         permanent: fileToUpdate.permanent || false
                     };
                     
-                    // Write new entry (atomic operation)
-                    await redisClient.hset(contextMapKey, uploadResult.hash, JSON.stringify(fileData));
+                    // Write new entry (atomic operation) - encryption happens in helper
+                    await writeFileDataToRedis(redisClient, contextMapKey, uploadResult.hash, fileData, contextKey);
                     
                     // If hash changed, remove old entry
                     if (oldHashToDelete && oldHashToDelete !== uploadResult.hash) {
@@ -378,7 +378,7 @@ export default {
                 await updateFileMetadata(contextId, fileToUpdate.hash, {
                     filename: filename,
                     lastAccessed: new Date().toISOString()
-                });
+                }, contextKey);
             }
 
             // Now it is safe to delete the old file version (after lock succeeds)
diff --git a/pathways/system/entity/tools/sys_tool_file_collection.js b/pathways/system/entity/tools/sys_tool_file_collection.js
index fb78ac18..72f3aa06 100644
--- a/pathways/system/entity/tools/sys_tool_file_collection.js
+++ b/pathways/system/entity/tools/sys_tool_file_collection.js
@@ -2,7 +2,7 @@
 // Tool pathway that manages user file collections (add, search, list files)
 // Uses Redis hash maps (FileStoreMap:ctx:<contextId>) for storage
 import logger from '../../../../lib/logger.js';
-import { addFileToCollection, loadFileCollection, saveFileCollection, findFileInCollection, deleteFileByHash, updateFileMetadata } from '../../../../lib/fileUtils.js';
+import { addFileToCollection, loadFileCollection, findFileInCollection, deleteFileByHash, updateFileMetadata } from '../../../../lib/fileUtils.js';
 
 export default {
     prompt: [],
@@ -230,7 +230,7 @@ export default {
                         // Update lastAccessed directly (atomic operation)
                         await updateFileMetadata(contextId, file.hash, {
                             lastAccessed: now
-                        });
+                        }, contextKey);
                     }
                 }
                 
diff --git a/tests/integration/features/tools/fileCollection.test.js b/tests/integration/features/tools/fileCollection.test.js
index acf55a3b..681eac0f 100644
--- a/tests/integration/features/tools/fileCollection.test.js
+++ b/tests/integration/features/tools/fileCollection.test.js
@@ -1060,3 +1060,294 @@ test('File collection: Sync files from chat history', async t => {
         await cleanup(contextId);
     }
 });
+
+// ============================================
+// File Collection Encryption Tests
+// ============================================
+
+test('File collection encryption: Encrypt tags and notes with contextKey', async t => {
+    const contextId = createTestContext();
+    const contextKey = '1234567890123456789012345678901234567890123456789012345678901234'; // 64 hex chars
+    
+    try {
+        // Add file with tags and notes
+        const result = await callPathway('sys_tool_file_collection', {
+            contextId,
+            contextKey,
+            url: 'https://example.com/encrypted.pdf',
+            filename: 'encrypted.pdf',
+            tags: ['sensitive', 'private', 'confidential'],
+            notes: 'This is sensitive information that should be encrypted',
+            userMessage: 'Add encrypted file'
+        });
+        
+        const parsed = JSON.parse(result);
+        t.is(parsed.success, true);
+        
+        // Verify data is encrypted in Redis
+        const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+        const redisClient = await getRedisClient();
+        const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+        const collection = await loadFileCollection(contextId, contextKey, false);
+        const file = collection.find(f => f.id === parsed.fileId);
+        t.truthy(file);
+        
+        // Get raw data from Redis (should be encrypted)
+        const rawDataStr = await redisClient.hget(contextMapKey, file.hash);
+        const rawData = JSON.parse(rawDataStr);
+        
+        // Verify tags and notes are encrypted (encrypted strings contain ':')
+        t.true(typeof rawData.tags === 'string', 'Tags should be encrypted string');
+        t.true(rawData.tags.includes(':'), 'Encrypted tags should contain IV separator');
+        t.true(typeof rawData.notes === 'string', 'Notes should be encrypted string');
+        t.true(rawData.notes.includes(':'), 'Encrypted notes should contain IV separator');
+        
+        // Verify core fields are NOT encrypted
+        t.is(rawData.url, 'https://example.com/encrypted.pdf', 'URL should not be encrypted');
+        t.is(rawData.displayFilename, 'encrypted.pdf', 'displayFilename should not be encrypted');
+        
+        // Verify decryption works correctly
+        t.deepEqual(file.tags, ['sensitive', 'private', 'confidential'], 'Tags should be decrypted correctly');
+        t.is(file.notes, 'This is sensitive information that should be encrypted', 'Notes should be decrypted correctly');
+    } finally {
+        await cleanup(contextId, contextKey);
+    }
+});
+
+test('File collection encryption: Empty tags and notes are not encrypted', async t => {
+    const contextId = createTestContext();
+    const contextKey = '1234567890123456789012345678901234567890123456789012345678901234'; // 64 hex chars
+    
+    try {
+        // Add file with empty tags and notes
+        const result = await callPathway('sys_tool_file_collection', {
+            contextId,
+            contextKey,
+            url: 'https://example.com/empty.pdf',
+            filename: 'empty.pdf',
+            tags: [],
+            notes: '',
+            userMessage: 'Add file with empty metadata'
+        });
+        
+        const parsed = JSON.parse(result);
+        t.is(parsed.success, true);
+        
+        // Verify empty values are not encrypted in Redis
+        const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+        const redisClient = await getRedisClient();
+        const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+        const collection = await loadFileCollection(contextId, contextKey, false);
+        const file = collection.find(f => f.id === parsed.fileId);
+        t.truthy(file);
+        
+        const rawDataStr = await redisClient.hget(contextMapKey, file.hash);
+        const rawData = JSON.parse(rawDataStr);
+        
+        // Empty tags should be array (not encrypted)
+        t.true(Array.isArray(rawData.tags), 'Empty tags should remain as array');
+        t.is(rawData.tags.length, 0, 'Empty tags array should be empty');
+        
+        // Empty notes should be empty string (not encrypted)
+        t.is(rawData.notes, '', 'Empty notes should remain as empty string');
+        t.false(rawData.notes.includes(':'), 'Empty notes should not be encrypted');
+    } finally {
+        await cleanup(contextId, contextKey);
+    }
+});
+
+test('File collection encryption: Decryption fails with wrong contextKey', async t => {
+    const contextId = createTestContext();
+    const contextKey = '1234567890123456789012345678901234567890123456789012345678901234'; // 64 hex chars
+    const wrongKey = '0000000000000000000000000000000000000000000000000000000000000000'; // 64 hex chars
+    
+    try {
+        // Add file with contextKey
+        const result = await callPathway('sys_tool_file_collection', {
+            contextId,
+            contextKey,
+            url: 'https://example.com/wrong-key.pdf',
+            filename: 'wrong-key.pdf',
+            tags: ['secret'],
+            notes: 'Secret notes',
+            userMessage: 'Add file'
+        });
+        
+        const parsed = JSON.parse(result);
+        t.is(parsed.success, true);
+        
+        // Try to load with wrong key
+        const collection = await loadFileCollection(contextId, wrongKey, false);
+        const file = collection.find(f => f.id === parsed.fileId);
+        t.truthy(file);
+        
+        // With wrong key, tags and notes should be encrypted strings (not decrypted)
+        // The fallback should keep them as-is, but they'll be encrypted strings
+        const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+        const redisClient = await getRedisClient();
+        const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+        const rawDataStr = await redisClient.hget(contextMapKey, file.hash);
+        const rawData = JSON.parse(rawDataStr);
+        
+        // When decryption fails, readFileDataFromRedis keeps the original encrypted string
+        // So file.tags and file.notes will be encrypted strings, not the original values
+        t.true(typeof file.tags === 'string' || Array.isArray(file.tags), 'Tags should be string or array');
+        if (typeof file.tags === 'string') {
+            t.true(file.tags.includes(':'), 'Tags should remain encrypted with wrong key');
+        }
+        
+        t.true(typeof file.notes === 'string', 'Notes should be string');
+        if (file.notes.includes(':')) {
+            t.true(file.notes.includes(':'), 'Notes should remain encrypted with wrong key');
+        }
+    } finally {
+        await cleanup(contextId, contextKey);
+    }
+});
+
+test('File collection encryption: Migration from unencrypted to encrypted', async t => {
+    const contextId = createTestContext();
+    const contextKey = '1234567890123456789012345678901234567890123456789012345678901234'; // 64 hex chars
+    
+    try {
+        // First, add file without contextKey (unencrypted)
+        const result1 = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/migration.pdf',
+            filename: 'migration.pdf',
+            tags: ['unencrypted'],
+            notes: 'Unencrypted notes',
+            userMessage: 'Add unencrypted file'
+        });
+        
+        const parsed1 = JSON.parse(result1);
+        t.is(parsed1.success, true);
+        
+        // Verify it's unencrypted in Redis
+        const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+        const redisClient = await getRedisClient();
+        const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+        const collection1 = await loadFileCollection(contextId, null, false);
+        const file1 = collection1.find(f => f.id === parsed1.fileId);
+        t.truthy(file1);
+        
+        const rawDataStr1 = await redisClient.hget(contextMapKey, file1.hash);
+        const rawData1 = JSON.parse(rawDataStr1);
+        
+        // Unencrypted data should have tags as array and notes as string
+        t.true(Array.isArray(rawData1.tags), 'Unencrypted tags should be array');
+        t.is(typeof rawData1.notes, 'string', 'Unencrypted notes should be string');
+        t.false(rawData1.notes.includes(':'), 'Unencrypted notes should not contain IV separator');
+        
+        // Now update with contextKey (should encrypt on next write)
+        await callPathway('sys_update_file_metadata', {
+            contextId,
+            contextKey,
+            hash: file1.hash,
+            tags: ['encrypted'],
+            notes: 'Encrypted notes'
+        });
+        
+        // Verify it's now encrypted
+        const rawDataStr2 = await redisClient.hget(contextMapKey, file1.hash);
+        const rawData2 = JSON.parse(rawDataStr2);
+        
+        t.true(typeof rawData2.tags === 'string', 'Tags should now be encrypted string');
+        t.true(rawData2.tags.includes(':'), 'Encrypted tags should contain IV separator');
+        t.true(typeof rawData2.notes === 'string', 'Notes should now be encrypted string');
+        t.true(rawData2.notes.includes(':'), 'Encrypted notes should contain IV separator');
+        
+        // Verify decryption works
+        const collection2 = await loadFileCollection(contextId, contextKey, false);
+        const file2 = collection2.find(f => f.id === parsed1.fileId);
+        t.deepEqual(file2.tags, ['encrypted'], 'Tags should be decrypted correctly');
+        t.is(file2.notes, 'Encrypted notes', 'Notes should be decrypted correctly');
+    } finally {
+        await cleanup(contextId, contextKey);
+    }
+});
+
+test('File collection encryption: Core fields are never encrypted', async t => {
+    const contextId = createTestContext();
+    const contextKey = '1234567890123456789012345678901234567890123456789012345678901234'; // 64 hex chars
+    
+    try {
+        // Add file with all fields
+        const result = await callPathway('sys_tool_file_collection', {
+            contextId,
+            contextKey,
+            url: 'https://example.com/core-fields.pdf',
+            filename: 'core-fields.pdf',
+            tags: ['test'],
+            notes: 'Test notes',
+            userMessage: 'Add file'
+        });
+        
+        const parsed = JSON.parse(result);
+        t.is(parsed.success, true);
+        
+        // Verify core fields are NOT encrypted
+        const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+        const redisClient = await getRedisClient();
+        const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+        const collection = await loadFileCollection(contextId, contextKey, false);
+        const file = collection.find(f => f.id === parsed.fileId);
+        t.truthy(file);
+        
+        const rawDataStr = await redisClient.hget(contextMapKey, file.hash);
+        const rawData = JSON.parse(rawDataStr);
+        
+        // Core fields should never be encrypted
+        t.is(rawData.url, 'https://example.com/core-fields.pdf', 'URL should not be encrypted');
+        t.is(rawData.displayFilename, 'core-fields.pdf', 'displayFilename should not be encrypted');
+        t.truthy(rawData.id, 'ID should not be encrypted');
+        t.truthy(rawData.hash, 'Hash should not be encrypted');
+        t.truthy(rawData.mimeType || rawData.mimeType === null, 'mimeType should not be encrypted');
+        t.truthy(rawData.addedDate, 'addedDate should not be encrypted');
+        t.truthy(rawData.lastAccessed, 'lastAccessed should not be encrypted');
+        t.is(typeof rawData.permanent, 'boolean', 'permanent should not be encrypted');
+    } finally {
+        await cleanup(contextId, contextKey);
+    }
+});
+
+test('File collection encryption: Works without contextKey (no encryption)', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add file without contextKey
+        const result = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/no-encryption.pdf',
+            filename: 'no-encryption.pdf',
+            tags: ['public'],
+            notes: 'Public notes',
+            userMessage: 'Add unencrypted file'
+        });
+        
+        const parsed = JSON.parse(result);
+        t.is(parsed.success, true);
+        
+        // Verify data is NOT encrypted in Redis
+        const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+        const redisClient = await getRedisClient();
+        const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+        const collection = await loadFileCollection(contextId, null, false);
+        const file = collection.find(f => f.id === parsed.fileId);
+        t.truthy(file);
+        
+        const rawDataStr = await redisClient.hget(contextMapKey, file.hash);
+        const rawData = JSON.parse(rawDataStr);
+        
+        // Without contextKey, tags and notes should be unencrypted
+        t.true(Array.isArray(rawData.tags), 'Tags should be array when not encrypted');
+        t.is(typeof rawData.notes, 'string', 'Notes should be string when not encrypted');
+        t.false(rawData.notes.includes(':'), 'Unencrypted notes should not contain IV separator');
+        
+        // Verify values are correct
+        t.deepEqual(file.tags, ['public'], 'Tags should be readable');
+        t.is(file.notes, 'Public notes', 'Notes should be readable');
+    } finally {
+        await cleanup(contextId);
+    }
+});

From f2d78299122f21d8ce43de63810fb948c98d47ae Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Fri, 19 Dec 2025 09:50:07 -0700
Subject: [PATCH 20/27] feat: enhance encryption and decryption functions to
 support AES-256-GCM

- Updated the encryption function to use AES-256-GCM with a 12-byte IV and added support for generating an authentication tag.
- Modified the decryption function to handle both the new GCM format (iv:tag:encrypted) and the legacy CBC format (iv:encrypted) for backward compatibility.
- Improved error handling and logging for decryption failures, ensuring better visibility into issues with invalid message formats.
---
 lib/crypto.js | 49 ++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 38 insertions(+), 11 deletions(-)

diff --git a/lib/crypto.js b/lib/crypto.js
index fa5f6ead..97eb882c 100644
--- a/lib/crypto.js
+++ b/lib/crypto.js
@@ -14,23 +14,31 @@ function getMessagePreview(message, maxLength = 50) {
     }
 }
 
-// Encryption function
+// Encryption function using AES-256-GCM (AEAD mode)
+// Format: iv:tag:encrypted (all hex-encoded)
 function encrypt(text, key) {
     if (!key) { return text; }
     try {
         key = tryBufferKey(key);
-        let iv = crypto.randomBytes(16);
-        let cipher = crypto.createCipheriv('aes-256-cbc', key, iv);
+        // GCM requires 12-byte IV (96 bits) for best performance
+        let iv = crypto.randomBytes(12);
+        let cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
         let encrypted = cipher.update(text, 'utf8', 'hex');
         encrypted += cipher.final('hex');
-        return iv.toString('hex') + ':' + encrypted;
+        // Get authentication tag (16 bytes by default for GCM)
+        let tag = cipher.getAuthTag();
+        // Return format: iv:tag:encrypted (all hex)
+        return iv.toString('hex') + ':' + tag.toString('hex') + ':' + encrypted;
     } catch (error) {
         logger.error(`Encryption failed: ${error.message}`);
         return null;
     }
 }
 
-// Decryption function
+// Decryption function using AES-256-GCM (AEAD mode)
+// Supports both old CBC format (for migration) and new GCM format
+// Old format: iv:encrypted (CBC, no tag)
+// New format: iv:tag:encrypted (GCM with authentication)
 function decrypt(message, key) {
     if (!key) { return message; }
     try {
@@ -50,12 +58,31 @@ function decrypt(message, key) {
         
         key = tryBufferKey(key);
         let parts = message.split(':');
-        let iv = Buffer.from(parts.shift(), 'hex');
-        let encrypted = parts.join(':');
-        let decipher = crypto.createDecipheriv('aes-256-cbc', key, iv);
-        let decrypted = decipher.update(encrypted, 'hex', 'utf8');
-        decrypted += decipher.final('utf8');
-        return decrypted;
+        
+        // Detect format: GCM has 3 parts (iv:tag:encrypted), CBC has 2 parts (iv:encrypted)
+        if (parts.length === 3) {
+            // New GCM format: iv:tag:encrypted
+            let iv = Buffer.from(parts[0], 'hex');
+            let tag = Buffer.from(parts[1], 'hex');
+            let encrypted = parts[2];
+            
+            let decipher = crypto.createDecipheriv('aes-256-gcm', key, iv);
+            decipher.setAuthTag(tag);
+            let decrypted = decipher.update(encrypted, 'hex', 'utf8');
+            decrypted += decipher.final('utf8');
+            return decrypted;
+        } else if (parts.length === 2) {
+            // Legacy CBC format: iv:encrypted (for backward compatibility during migration)
+            let iv = Buffer.from(parts[0], 'hex');
+            let encrypted = parts[1];
+            
+            let decipher = crypto.createDecipheriv('aes-256-cbc', key, iv);
+            let decrypted = decipher.update(encrypted, 'hex', 'utf8');
+            decrypted += decipher.final('utf8');
+            return decrypted;
+        } else {
+            throw new Error('Invalid encrypted message format');
+        }
     } catch (error) {
         const preview = getMessagePreview(message);
         logger.error(`Decryption failed: ${error.message} (preview: ${preview})`);

From ec6ab37f35a21555b274911f11af8bb5f3d04e40 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Fri, 19 Dec 2025 11:31:46 -0700
Subject: [PATCH 21/27] feat: add MIME type handling and enhance logging
 security

- Introduced MIME type detection for uploaded files, storing the type for better file handling and context detection.
- Enhanced logging by redacting sensitive information such as SAS tokens and context IDs to improve security.
- Updated sanitization logic to ensure context IDs are specifically redacted during logging, maintaining data privacy.
---
 helper-apps/cortex-file-handler/src/blobHandler.js   | 10 ++++++++++
 helper-apps/cortex-file-handler/src/index.js         | 12 +++++++-----
 .../cortex-file-handler/src/utils/logSecurity.js     |  9 +++++++--
 3 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/helper-apps/cortex-file-handler/src/blobHandler.js b/helper-apps/cortex-file-handler/src/blobHandler.js
index 6be2dc36..eeae97a4 100644
--- a/helper-apps/cortex-file-handler/src/blobHandler.js
+++ b/helper-apps/cortex-file-handler/src/blobHandler.js
@@ -9,6 +9,7 @@ import { PassThrough } from "stream";
 import { Storage } from "@google-cloud/storage";
 import { BlobServiceClient } from "@azure/storage-blob";
 import axios from "axios";
+import mime from "mime-types";
 
 import {
   sanitizeFilename,
@@ -492,6 +493,11 @@ function uploadBlob(
               };
               if (hash) result.hash = hash;
               
+              // Store MIME type from upload (used by Cortex for file type detection)
+              if (contentType) {
+                result.mimeType = contentType;
+              }
+              
               // Extract contextId from form fields if present
               if (fields && fields.contextId) {
                 result.contextId = fields.contextId;
@@ -833,6 +839,10 @@ async function uploadFile(
       result.hash = hash;
     }
     
+    // Store MIME type determined from filename (used by Cortex for file type detection)
+    const mimeType = mime.lookup(uploadName) || 'application/octet-stream';
+    result.mimeType = mimeType;
+    
     // Extract contextId from form fields if present (only available for multipart uploads)
     if (fields && fields.contextId) {
       result.contextId = fields.contextId;
diff --git a/helper-apps/cortex-file-handler/src/index.js b/helper-apps/cortex-file-handler/src/index.js
index 7aded91c..d00db8d5 100644
--- a/helper-apps/cortex-file-handler/src/index.js
+++ b/helper-apps/cortex-file-handler/src/index.js
@@ -19,7 +19,7 @@ import { FileConversionService } from "./services/FileConversionService.js";
 import { StorageService } from "./services/storage/StorageService.js";
 import { uploadBlob } from "./blobHandler.js";
 import { generateShortId } from "./utils/filenameUtils.js";
-import { redactContextId } from "./utils/logSecurity.js";
+import { redactContextId, redactSasToken, sanitizeForLogging } from "./utils/logSecurity.js";
 
 // Hybrid cleanup approach:
 // 1. Lazy cleanup: Check file existence when cache entries are accessed (in getFileStoreMap)
@@ -117,7 +117,7 @@ async function CortexFileHandler(context, req) {
                 : "upload";
 
   context.log(
-    `Processing ${req.method} request - ${requestId ? `requestId: ${requestId}, ` : ""}${uri ? `uri: ${uri}, ` : ""}${hash ? `hash: ${hash}, ` : ""}${resolvedContextId ? `contextId: ${redactContextId(resolvedContextId)}, ` : ""}operation: ${operation}`,
+    `Processing ${req.method} request - ${requestId ? `requestId: ${requestId}, ` : ""}${uri ? `uri: ${redactSasToken(uri)}, ` : ""}${hash ? `hash: ${hash}, ` : ""}${resolvedContextId ? `contextId: ${redactContextId(resolvedContextId)}, ` : ""}operation: ${operation}`,
   );
 
   // Trigger lightweight age-based cleanup (runs every 100 requests)
@@ -268,7 +268,7 @@ async function CortexFileHandler(context, req) {
 
   const remoteUrl = shouldFetchRemote;
   if (req.method.toLowerCase() === "get" && remoteUrl) {
-    context.log(`Remote file: ${remoteUrl}`);
+    context.log(`Remote file: ${redactSasToken(remoteUrl)}`);
     let filename;
     try {
       // Validate URL format and accessibility
@@ -381,7 +381,7 @@ async function CortexFileHandler(context, req) {
       context.log(`File exists in map: ${hash}${resolvedContextId ? ` (contextId: ${redactContextId(resolvedContextId)})` : ""}`);
 
       // Log the URL retrieved from Redis before checking existence
-      context.log(`Checking existence of URL from Redis: ${hashResult?.url}`);
+      context.log(`Checking existence of URL from Redis: ${redactSasToken(hashResult?.url || '')}`);
 
       try {
         // Check primary storage first
@@ -858,9 +858,11 @@ async function CortexFileHandler(context, req) {
     return;
   }
 
+  // Sanitize result before logging to redact SAS tokens and contextIds
+  const sanitizedResult = sanitizeForLogging(result);
   console.log(
     "result:",
-    result
+    sanitizedResult
       .map((item) =>
         typeof item === "object" ? JSON.stringify(item, null, 2) : item,
       )
diff --git a/helper-apps/cortex-file-handler/src/utils/logSecurity.js b/helper-apps/cortex-file-handler/src/utils/logSecurity.js
index ce4e84b9..198ae97f 100644
--- a/helper-apps/cortex-file-handler/src/utils/logSecurity.js
+++ b/helper-apps/cortex-file-handler/src/utils/logSecurity.js
@@ -40,7 +40,7 @@ export function redactSasToken(url) {
 }
 
 /**
- * Recursively sanitizes an object by redacting SAS tokens from URLs
+ * Recursively sanitizes an object by redacting SAS tokens from URLs and contextIds
  * @param {any} obj - The object to sanitize
  * @returns {any} - Sanitized copy of the object
  */
@@ -58,7 +58,12 @@ export function sanitizeForLogging(obj) {
   if (typeof obj === 'object') {
     const sanitized = {};
     for (const [key, value] of Object.entries(obj)) {
-      sanitized[key] = sanitizeForLogging(value);
+      // Redact contextId fields specifically
+      if (key === 'contextId' && typeof value === 'string') {
+        sanitized[key] = redactContextId(value);
+      } else {
+        sanitized[key] = sanitizeForLogging(value);
+      }
     }
     return sanitized;
   }

From dc5007311e6ab70de6f077678630df45d27649d9 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Fri, 19 Dec 2025 12:41:55 -0700
Subject: [PATCH 22/27] Bug fixes for crypto and youtube handling

---
 lib/crypto.js                                 |  26 ++-
 lib/fileUtils.js                              |  70 +++++++
 .../entity/tools/sys_tool_analyzefile.js      |  49 +++--
 pathways/transcribe_gemini.js                 |  38 +---
 .../features/tools/fileCollection.test.js     | 181 ++++++++++++++++++
 .../graphql/features/tools/call_tools.test.js |   1 +
 tests/unit/core/crypto.test.js                |  41 ++++
 7 files changed, 351 insertions(+), 55 deletions(-)

diff --git a/lib/crypto.js b/lib/crypto.js
index 97eb882c..659b8963 100644
--- a/lib/crypto.js
+++ b/lib/crypto.js
@@ -59,10 +59,23 @@ function decrypt(message, key) {
         key = tryBufferKey(key);
         let parts = message.split(':');
         
+        // Helper to check if a string is valid hex IV (correct length and hex characters only)
+        function isValidHexIV(hexStr, expectedBytes) {
+            const expectedHexLength = expectedBytes * 2;
+            return hexStr.length === expectedHexLength && /^[0-9a-fA-F]+$/.test(hexStr);
+        }
+        
         // Detect format: GCM has 3 parts (iv:tag:encrypted), CBC has 2 parts (iv:encrypted)
+        // Validate IV before attempting decryption to avoid treating plain text as encrypted
         if (parts.length === 3) {
             // New GCM format: iv:tag:encrypted
-            let iv = Buffer.from(parts[0], 'hex');
+            let ivHex = parts[0];
+            // If IV doesn't look like encrypted data (24 hex chars for 12-byte IV), return as-is
+            if (!isValidHexIV(ivHex, 12)) {
+                return message;
+            }
+            
+            let iv = Buffer.from(ivHex, 'hex');
             let tag = Buffer.from(parts[1], 'hex');
             let encrypted = parts[2];
             
@@ -73,7 +86,13 @@ function decrypt(message, key) {
             return decrypted;
         } else if (parts.length === 2) {
             // Legacy CBC format: iv:encrypted (for backward compatibility during migration)
-            let iv = Buffer.from(parts[0], 'hex');
+            let ivHex = parts[0];
+            // If IV doesn't look like encrypted data (32 hex chars for 16-byte IV), return as-is
+            if (!isValidHexIV(ivHex, 16)) {
+                return message;
+            }
+            
+            let iv = Buffer.from(ivHex, 'hex');
             let encrypted = parts[1];
             
             let decipher = crypto.createDecipheriv('aes-256-cbc', key, iv);
@@ -81,7 +100,8 @@ function decrypt(message, key) {
             decrypted += decipher.final('utf8');
             return decrypted;
         } else {
-            throw new Error('Invalid encrypted message format');
+            // Not in expected encrypted format - probably plain text, return as-is
+            return message;
         }
     } catch (error) {
         const preview = getMessagePreview(message);
diff --git a/lib/fileUtils.js b/lib/fileUtils.js
index 91e802ef..15df2391 100644
--- a/lib/fileUtils.js
+++ b/lib/fileUtils.js
@@ -18,6 +18,49 @@ import { encrypt, decrypt } from './crypto.js';
 const pipeline = promisify(stream.pipeline);
 const MEDIA_API_URL = config.get('whisperMediaApiUrl');
 
+/**
+ * Check if a URL is a YouTube URL
+ * Validates URL structure to ensure it's a valid YouTube video URL
+ * @param {string} url - URL to check
+ * @returns {boolean} True if URL is a valid YouTube video URL
+ */
+export function isYoutubeUrl(url) {
+    if (!url || typeof url !== 'string') return false;
+    try {
+        const urlObj = new URL(url);
+
+        // Check for standard youtube.com domains
+        if (
+            urlObj.hostname === "youtube.com" ||
+            urlObj.hostname === "www.youtube.com"
+        ) {
+            // For standard watch URLs, verify they have a video ID
+            if (urlObj.pathname === "/watch") {
+                return !!urlObj.searchParams.get("v");
+            }
+            // For embed URLs, verify they have a video ID in the path
+            if (urlObj.pathname.startsWith("/embed/")) {
+                return urlObj.pathname.length > 7; // '/embed/' is 7 chars
+            }
+            // For shorts URLs, verify they have a video ID in the path
+            if (urlObj.pathname.startsWith("/shorts/")) {
+                return urlObj.pathname.length > 8; // '/shorts/' is 8 chars
+            }
+            return false;
+        }
+
+        // Check for shortened youtu.be domain
+        if (urlObj.hostname === "youtu.be") {
+            // Verify there's a video ID in the path
+            return urlObj.pathname.length > 1; // '/' is 1 char
+        }
+
+        return false;
+    } catch (err) {
+        return false;
+    }
+}
+
 // Cache xxhash instance for reuse
 let xxhashInstance = null;
 let xxhashInitPromise = null;
@@ -916,6 +959,15 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
     // If permanent=true, set retention=permanent to keep file forever
     const desiredRetention = permanent ? 'permanent' : 'temporary';
 
+    // YouTube URLs should not be added to the file collection (they are never uploaded to CFH)
+    // They can be used directly in analyzer tools without being in the collection
+    if (fileUrl && isYoutubeUrl(fileUrl)) {
+        throw new Error("YouTube URLs cannot be added to the file collection. Use the YouTube URL directly with analyzer tools instead.");
+    }
+    if (url && isYoutubeUrl(url)) {
+        throw new Error("YouTube URLs cannot be added to the file collection. Use the YouTube URL directly with analyzer tools instead.");
+    }
+    
     // If fileUrl is provided and url is not already a cloud URL, upload the file first
     let finalUrl = url;
     let finalGcs = gcs;
@@ -1494,6 +1546,16 @@ async function generateFileMessageContent(fileParam, contextId, contextKey = nul
         return null;
     }
 
+    // If fileParam is a YouTube URL, return it directly (doesn't need to be in collection)
+    if (isYoutubeUrl(fileParam)) {
+        return {
+            type: 'image_url',
+            url: fileParam,
+            gcs: null,
+            hash: null
+        };
+    }
+
     if (!contextId) {
         // Without contextId, we can't look up in collection
         // Return a basic content object from the URL
@@ -1660,6 +1722,13 @@ async function ensureShortLivedUrl(fileObject, fileHandlerUrl, contextId = null,
         return fileObject;
     }
     
+    // Note: YouTube URLs should not be in the file collection, but if one somehow got through,
+    // we'll skip hash resolution for it (defensive check)
+    // Defensive check: if URL is YouTube, return as-is (shouldn't happen, but handle gracefully)
+    if (fileObject.url && isYoutubeUrl(fileObject.url)) {
+        return fileObject;
+    }
+    
     try {
         const resolved = await checkHashExists(fileObject.hash, fileHandlerUrl, null, contextId, shortLivedMinutes);
         if (resolved && resolved.url) {
@@ -2124,6 +2193,7 @@ export {
     isTextMimeType,
     isFileInCollection,
     writeFileDataToRedis,
+    // isYoutubeUrl is exported inline above
     // Exported for testing
     extractFilenameFromUrl,
     ensureFilenameExtension,
diff --git a/pathways/system/entity/tools/sys_tool_analyzefile.js b/pathways/system/entity/tools/sys_tool_analyzefile.js
index 5445ed07..2c9f9157 100644
--- a/pathways/system/entity/tools/sys_tool_analyzefile.js
+++ b/pathways/system/entity/tools/sys_tool_analyzefile.js
@@ -3,6 +3,7 @@
 
 import { Prompt } from '../../../../server/prompt.js';
 import { generateFileMessageContent, injectFileIntoChatHistory } from '../../../../lib/fileUtils.js';
+import logger from '../../../../lib/logger.js';
 
 export default {
     prompt:
@@ -20,7 +21,7 @@ export default {
         language: "English",
     },
     max_tokens: 8192,
-    model: 'gemini-flash-20-vision',
+    model: 'gemini-flash-3-vision',
     useInputChunking: false,
     enableDuplicateRequests: false,
     timeout: 600,
@@ -132,7 +133,7 @@ export default {
         icon: "🎥",
         function: {
             name: "AnalyzeVideo",
-            description: "Use specifically for reading, analyzing, and answering questions about video or audio file content. You MUST use this tool to look at video or audio files.",
+            description: "Use specifically for reading, analyzing, and answering questions about video or audio file content. You MUST use this tool to look at video or audio files. This tool supports YouTube URLs (youtube.com, youtu.be), direct video/audio file URLs, and files from the file collection.",
             parameters: {
                 type: "object",
                 properties: {
@@ -142,7 +143,7 @@ export default {
                     },
                     file: {
                         type: "string",
-                        description: "Optional: The file to analyze (from ListFileCollection or SearchFileCollection): can be the hash, the filename, the URL, or the GCS URL. You can find available files in the availableFiles section."
+                        description: "Optional: The file to analyze. Can be: (1) A YouTube URL (youtube.com/watch?v=..., youtu.be/..., youtube.com/shorts/..., youtube.com/embed/...), (2) A direct video/audio file URL, (3) A file from the collection (hash, filename, URL, or GCS URL from ListFileCollection or SearchFileCollection). You can find available files in the availableFiles section."
                     },
                     userMessage: {
                         type: "string",
@@ -155,20 +156,38 @@ export default {
     }],
     
     executePathway: async ({args, runAllPrompts, resolver}) => {
-        // Generate file message content and inject file if provided
-        if (args.file) {
-            const fileContent = await generateFileMessageContent(args.file, args.contextId, args.contextKey);
-            if (!fileContent) {
-                throw new Error(`File not found: "${args.file}". Use ListFileCollection or SearchFileCollection to find available files.`);
+        try {
+            // Generate file message content and inject file if provided
+            if (args.file) {
+                const fileContent = await generateFileMessageContent(args.file, args.contextId, args.contextKey);
+                if (!fileContent) {
+                    const errorMessage = `File not found: "${args.file}". Use ListFileCollection or SearchFileCollection to find available files.`;
+                    resolver.tool = JSON.stringify({ toolUsed: "vision" });
+                    return JSON.stringify({ 
+                        error: errorMessage,
+                        recoveryMessage: "The file was not found. Please verify the file exists in the collection or provide a valid file reference."
+                    });
+                }
+                args.chatHistory = injectFileIntoChatHistory(args.chatHistory, fileContent);
             }
-            args.chatHistory = injectFileIntoChatHistory(args.chatHistory, fileContent);
-        }
 
-        if (args.detailedInstructions) {
-            args.chatHistory.push({role: "user", content: args.detailedInstructions});
+            if (args.detailedInstructions) {
+                args.chatHistory.push({role: "user", content: args.detailedInstructions});
+            }
+            
+            const result = await runAllPrompts({ ...args });
+            resolver.tool = JSON.stringify({ toolUsed: "vision" });
+            return result;
+        } catch (e) {
+            // Catch any errors from runAllPrompts or other operations
+            const errorMessage = e?.message || e?.toString() || String(e);
+            logger.error(`Error in analyzer tool: ${errorMessage}`);
+            
+            resolver.tool = JSON.stringify({ toolUsed: "vision" });
+            return JSON.stringify({ 
+                error: errorMessage,
+                recoveryMessage: "The file analysis failed. Please verify the file is accessible and in a supported format, or try a different file."
+            });
         }
-        const result = await runAllPrompts({ ...args });
-        resolver.tool = JSON.stringify({ toolUsed: "vision" });
-        return result;
     }
 }
diff --git a/pathways/transcribe_gemini.js b/pathways/transcribe_gemini.js
index e6767f11..ac734930 100644
--- a/pathways/transcribe_gemini.js
+++ b/pathways/transcribe_gemini.js
@@ -1,47 +1,11 @@
 import logger from "../lib/logger.js";
 import { publishRequestProgress } from "../lib/redisSubscription.js";
 import { alignSubtitles } from "../lib/util.js";
-import { getMediaChunks } from "../lib/fileUtils.js";
+import { getMediaChunks, isYoutubeUrl } from "../lib/fileUtils.js";
 import { Prompt } from "../server/prompt.js";
 
 const OFFSET_CHUNK = 500; //seconds of each chunk offset, only used if helper does not provide
 
-function isYoutubeUrl(url) {
-    try {
-        const urlObj = new URL(url);
-
-        // Check for standard youtube.com domains
-        if (
-            urlObj.hostname === "youtube.com" ||
-            urlObj.hostname === "www.youtube.com"
-        ) {
-            // For standard watch URLs, verify they have a video ID
-            if (urlObj.pathname === "/watch") {
-                return !!urlObj.searchParams.get("v");
-            }
-            // For embed URLs, verify they have a video ID in the path
-            if (urlObj.pathname.startsWith("/embed/")) {
-                return urlObj.pathname.length > 7; // '/embed/' is 7 chars
-            }
-            // For shorts URLs, verify they have a video ID in the path
-            if (urlObj.pathname.startsWith("/shorts/")) {
-                return urlObj.pathname.length > 8; // '/shorts/' is 8 chars
-            }
-            return false;
-        }
-
-        // Check for shortened youtu.be domain
-        if (urlObj.hostname === "youtu.be") {
-            // Verify there's a video ID in the path
-            return urlObj.pathname.length > 1; // '/' is 1 char
-        }
-
-        return false;
-    } catch (err) {
-        return false;
-    }
-}
-
 export default {
     prompt:
     [
diff --git a/tests/integration/features/tools/fileCollection.test.js b/tests/integration/features/tools/fileCollection.test.js
index 681eac0f..d20008aa 100644
--- a/tests/integration/features/tools/fileCollection.test.js
+++ b/tests/integration/features/tools/fileCollection.test.js
@@ -1351,3 +1351,184 @@ test('File collection encryption: Works without contextKey (no encryption)', asy
         await cleanup(contextId);
     }
 });
+
+test('File collection: YouTube URLs are rejected (cannot be added to collection)', async t => {
+    const contextId = createTestContext();
+    const youtubeUrl = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
+    
+    try {
+        // Attempt to add YouTube URL - should be rejected
+        const result = await callPathway('sys_tool_file_collection', {
+            contextId,
+            fileUrl: youtubeUrl,
+            filename: 'Test YouTube Video',
+            tags: ['video', 'youtube'],
+            notes: 'Test YouTube video',
+            userMessage: 'Add YouTube video'
+        });
+        
+        // callPathway may catch and return error as JSON string, or throw
+        // Check if it's an error response
+        try {
+            const parsed = JSON.parse(result);
+            t.falsy(parsed.success, 'Should not succeed');
+            t.truthy(parsed.error || parsed.message, 'Should have error message');
+            t.true(
+                (parsed.error || parsed.message || '').includes('YouTube URLs cannot be added'),
+                'Error should mention YouTube URLs cannot be added'
+            );
+        } catch (parseError) {
+            // If not JSON, it should be an error string
+            t.true(
+                result.includes('YouTube URLs cannot be added'),
+                'Error message should mention YouTube URLs cannot be added'
+            );
+        }
+        
+        // Verify it was NOT added to collection
+        const collection = await loadFileCollection(contextId, null, false);
+        t.is(collection.length, 0);
+    } catch (error) {
+        // If callPathway throws, verify the error message
+        t.true(
+            error.message.includes('YouTube URLs cannot be added'),
+            'Error should mention YouTube URLs cannot be added'
+        );
+        
+        // Verify it was NOT added to collection
+        const collection = await loadFileCollection(contextId, null, false);
+        t.is(collection.length, 0);
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: YouTube Shorts URLs are rejected', async t => {
+    const contextId = createTestContext();
+    const shortsUrl = 'https://www.youtube.com/shorts/abc123';
+    
+    try {
+        const result = await callPathway('sys_tool_file_collection', {
+            contextId,
+            fileUrl: shortsUrl,
+            filename: 'YouTube Short',
+            userMessage: 'Add YouTube short'
+        });
+        
+        try {
+            const parsed = JSON.parse(result);
+            t.falsy(parsed.success);
+            t.true((parsed.error || parsed.message || '').includes('YouTube URLs cannot be added'));
+        } catch (parseError) {
+            t.true(result.includes('YouTube URLs cannot be added'));
+        }
+        
+        const collection = await loadFileCollection(contextId, null, false);
+        t.is(collection.length, 0);
+    } catch (error) {
+        t.true(error.message.includes('YouTube URLs cannot be added'));
+        const collection = await loadFileCollection(contextId, null, false);
+        t.is(collection.length, 0);
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: youtu.be URLs are rejected', async t => {
+    const contextId = createTestContext();
+    const youtuBeUrl = 'https://youtu.be/dQw4w9WgXcQ';
+    
+    try {
+        const result = await callPathway('sys_tool_file_collection', {
+            contextId,
+            fileUrl: youtuBeUrl,
+            filename: 'YouTube Video',
+            userMessage: 'Add YouTube video'
+        });
+        
+        try {
+            const parsed = JSON.parse(result);
+            t.falsy(parsed.success);
+            t.true((parsed.error || parsed.message || '').includes('YouTube URLs cannot be added'));
+        } catch (parseError) {
+            t.true(result.includes('YouTube URLs cannot be added'));
+        }
+        
+        const collection = await loadFileCollection(contextId, null, false);
+        t.is(collection.length, 0);
+    } catch (error) {
+        t.true(error.message.includes('YouTube URLs cannot be added'));
+        const collection = await loadFileCollection(contextId, null, false);
+        t.is(collection.length, 0);
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('generateFileMessageContent: Accepts direct YouTube URL without collection', async t => {
+    const contextId = createTestContext();
+    const youtubeUrl = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
+    
+    try {
+        // Test that generateFileMessageContent accepts YouTube URL directly
+        // even if it's not in the collection
+        const fileContent = await generateFileMessageContent(youtubeUrl, contextId);
+        t.truthy(fileContent);
+        t.is(fileContent.url, youtubeUrl);
+        t.is(fileContent.type, 'image_url');
+        t.falsy(fileContent.gcs);
+        t.falsy(fileContent.hash);
+        
+        // Verify it's not in the collection
+        const collection = await loadFileCollection(contextId, null, false);
+        t.is(collection.length, 0);
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('generateFileMessageContent: Accepts direct youtu.be URL without collection', async t => {
+    const contextId = createTestContext();
+    const youtuBeUrl = 'https://youtu.be/dQw4w9WgXcQ';
+    
+    try {
+        const fileContent = await generateFileMessageContent(youtuBeUrl, contextId);
+        t.truthy(fileContent);
+        t.is(fileContent.url, youtuBeUrl);
+        t.is(fileContent.type, 'image_url');
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('Analyzer tool: Returns error JSON format when file not found', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        const result = await callPathway('sys_tool_analyzefile', {
+            contextId,
+            file: 'non-existent-file.jpg',
+            detailedInstructions: 'Analyze this file',
+            userMessage: 'Testing error handling'
+        });
+        
+        t.truthy(result, 'Should have a result');
+        
+        // Parse the result to check for error format
+        let parsedResult;
+        try {
+            parsedResult = JSON.parse(result);
+        } catch (error) {
+            t.fail(`Failed to parse result: ${error.message}`);
+        }
+        
+        // Should return error JSON format (same as search tools)
+        t.truthy(parsedResult.error, 'Should have error field');
+        t.truthy(parsedResult.recoveryMessage, 'Should have recoveryMessage field');
+        t.true(typeof parsedResult.error === 'string', 'Error should be a string');
+        t.true(typeof parsedResult.recoveryMessage === 'string', 'RecoveryMessage should be a string');
+        t.true(parsedResult.error.includes('File not found'), 'Error should mention file not found');
+    } finally {
+        await cleanup(contextId);
+    }
+});
diff --git a/tests/integration/graphql/features/tools/call_tools.test.js b/tests/integration/graphql/features/tools/call_tools.test.js
index a1fd3a82..8b29223f 100644
--- a/tests/integration/graphql/features/tools/call_tools.test.js
+++ b/tests/integration/graphql/features/tools/call_tools.test.js
@@ -74,6 +74,7 @@ test.after.always('cleanup', async () => {
   }
 });
 
+
 // Add after.always hook to print rankings
 test.after.always('print rankings', async () => {
   printModelRankings();
diff --git a/tests/unit/core/crypto.test.js b/tests/unit/core/crypto.test.js
index 937eced6..9f078d32 100644
--- a/tests/unit/core/crypto.test.js
+++ b/tests/unit/core/crypto.test.js
@@ -63,3 +63,44 @@ test('encrypt should handle JSON data', t => {
     const decrypted = decrypt(encrypted, systemKey);
     t.is(decrypted, jsonData);
 });
+
+// Tests for plain text detection (preventing "Invalid initialization vector" errors)
+test('decrypt should return plain text with colons as-is (not encrypted)', t => {
+    const plainText = 'Modified image from prompt: Edit the image by addi';
+    const result = decrypt(plainText, systemKey);
+    t.is(result, plainText);
+});
+
+test('decrypt should return plain text with multiple colons as-is', t => {
+    const plainText = 'test:data:with:multiple:colons';
+    const result = decrypt(plainText, systemKey);
+    t.is(result, plainText);
+});
+
+test('decrypt should return plain text without colons as-is', t => {
+    const plainText = 'simple text without colons';
+    const result = decrypt(plainText, systemKey);
+    t.is(result, plainText);
+});
+
+test('decrypt should still decrypt valid encrypted data', t => {
+    const encrypted = encrypt(testData, systemKey);
+    const decrypted = decrypt(encrypted, systemKey);
+    t.is(decrypted, testData);
+});
+
+test('decrypt should handle plain text that looks like encrypted format (2 parts)', t => {
+    // Plain text with exactly 2 colons that might be misdetected as CBC format
+    const plainText = 'part1:part2:part3';
+    const result = decrypt(plainText, systemKey);
+    // Should return as-is because IV validation will fail
+    t.is(result, plainText);
+});
+
+test('decrypt should handle plain text that looks like encrypted format (3 parts)', t => {
+    // Plain text with exactly 3 colons that might be misdetected as GCM format
+    const plainText = 'part1:part2:part3:part4';
+    const result = decrypt(plainText, systemKey);
+    // Should return as-is because it doesn't match expected format
+    t.is(result, plainText);
+});

From f8e7d3405edbba2a7608a81e2f4cb6ef5bf60526 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Fri, 19 Dec 2025 13:22:38 -0700
Subject: [PATCH 23/27] refactor: enhance file message generation and error
 handling

- Wrapped YouTube URL check in a try-catch block to prevent errors from disrupting file lookup.
- Updated the file message generation logic to create a clean chat history, avoiding confusion from previous messages.
- Improved error handling for file analysis, ensuring clearer logging and recovery messages when issues arise.
---
 lib/fileUtils.js                              | 20 ++++---
 .../entity/tools/sys_tool_analyzefile.js      | 55 +++++++++++++++++--
 server/plugins/gemini15VisionPlugin.js        | 24 ++++----
 3 files changed, 76 insertions(+), 23 deletions(-)

diff --git a/lib/fileUtils.js b/lib/fileUtils.js
index 15df2391..c4a9c79d 100644
--- a/lib/fileUtils.js
+++ b/lib/fileUtils.js
@@ -1547,13 +1547,19 @@ async function generateFileMessageContent(fileParam, contextId, contextKey = nul
     }
 
     // If fileParam is a YouTube URL, return it directly (doesn't need to be in collection)
-    if (isYoutubeUrl(fileParam)) {
-        return {
-            type: 'image_url',
-            url: fileParam,
-            gcs: null,
-            hash: null
-        };
+    // Wrap in try-catch to prevent errors from breaking file lookup
+    try {
+        if (isYoutubeUrl(fileParam)) {
+            return {
+                type: 'image_url',
+                url: fileParam,
+                gcs: null,
+                hash: null
+            };
+        }
+    } catch (error) {
+        // If YouTube URL check fails, continue with normal file lookup
+        logger.debug(`YouTube URL check failed for "${fileParam}": ${error.message}`);
     }
 
     if (!contextId) {
diff --git a/pathways/system/entity/tools/sys_tool_analyzefile.js b/pathways/system/entity/tools/sys_tool_analyzefile.js
index 2c9f9157..4d877308 100644
--- a/pathways/system/entity/tools/sys_tool_analyzefile.js
+++ b/pathways/system/entity/tools/sys_tool_analyzefile.js
@@ -157,7 +157,11 @@ export default {
     
     executePathway: async ({args, runAllPrompts, resolver}) => {
         try {
-            // Generate file message content and inject file if provided
+            // Create a clean chat history with just the file and task - don't include previous chat history
+            // This prevents confusion from function results and other context
+            const cleanChatHistory = [];
+            
+            // Generate file message content if provided
             if (args.file) {
                 const fileContent = await generateFileMessageContent(args.file, args.contextId, args.contextKey);
                 if (!fileContent) {
@@ -168,14 +172,53 @@ export default {
                         recoveryMessage: "The file was not found. Please verify the file exists in the collection or provide a valid file reference."
                     });
                 }
-                args.chatHistory = injectFileIntoChatHistory(args.chatHistory, fileContent);
+                
+                // Combine file and instructions in the same message so Gemini sees both together
+                const messageContent = [fileContent];
+                if (args.detailedInstructions) {
+                    messageContent.push({type: 'text', text: args.detailedInstructions});
+                }
+                
+                cleanChatHistory.push({role: "user", content: messageContent});
+            } else if (args.detailedInstructions) {
+                // No file, just add instructions
+                cleanChatHistory.push({role: "user", content: args.detailedInstructions});
             }
-
-            if (args.detailedInstructions) {
-                args.chatHistory.push({role: "user", content: args.detailedInstructions});
+            
+            // Use clean chat history instead of the full chat history
+            args.chatHistory = cleanChatHistory;
+            
+            // Explicitly disable function calling - this tool is just for vision analysis, not tool calls
+            // This prevents MALFORMED_FUNCTION_CALL errors
+            const result = await runAllPrompts({ ...args, tool_choice: 'none' });
+            
+            // Check for errors in resolver (ModelExecutor logs errors here when it catches exceptions)
+            if (resolver.errors && resolver.errors.length > 0) {
+                const errorMessages = Array.isArray(resolver.errors) 
+                    ? resolver.errors.map(err => err.message || err)
+                    : [resolver.errors.message || resolver.errors];
+                
+                const errorMessageStr = errorMessages.join('; ');
+                logger.error(`Analyzer tool error: ${errorMessageStr}`);
+                
+                resolver.tool = JSON.stringify({ toolUsed: "vision" });
+                return JSON.stringify({ 
+                    error: errorMessageStr,
+                    recoveryMessage: "The file analysis failed. Please verify the file is accessible and in a supported format, or try a different file."
+                });
+            }
+            
+            // Handle null response (can happen when ModelExecutor catches an error but doesn't log it)
+            if (!result) {
+                const errorMessage = 'Model execution returned null - the model request likely failed';
+                logger.error(`Error in analyzer tool: ${errorMessage}`);
+                resolver.tool = JSON.stringify({ toolUsed: "vision" });
+                return JSON.stringify({ 
+                    error: errorMessage,
+                    recoveryMessage: "The file analysis failed. Please verify the file is accessible and in a supported format, or try a different file."
+                });
             }
             
-            const result = await runAllPrompts({ ...args });
             resolver.tool = JSON.stringify({ toolUsed: "vision" });
             return result;
         } catch (e) {
diff --git a/server/plugins/gemini15VisionPlugin.js b/server/plugins/gemini15VisionPlugin.js
index 81cbc816..16d10c04 100644
--- a/server/plugins/gemini15VisionPlugin.js
+++ b/server/plugins/gemini15VisionPlugin.js
@@ -267,19 +267,19 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
 
         const baseParameters = super.getRequestParameters(text, parameters, prompt, cortexRequest);
 
+        // Handle tool_choice parameter - convert OpenAI format to Gemini toolConfig
+        let toolChoice = parameters.tool_choice;
+        if (typeof toolChoice === 'string' && toolChoice !== 'auto' && toolChoice !== 'none' && toolChoice !== 'required' && toolChoice !== 'any') {
+            try {
+                toolChoice = JSON.parse(toolChoice);
+            } catch (e) {
+                toolChoice = 'auto';
+            }
+        }
+
         if (convertedTools[0]?.functionDeclarations?.length > 0) {
             baseParameters.tools = convertedTools;
             
-            // Handle tool_choice parameter - convert OpenAI format to Gemini toolConfig
-            let toolChoice = parameters.tool_choice;
-            if (typeof toolChoice === 'string' && toolChoice !== 'auto' && toolChoice !== 'none' && toolChoice !== 'required' && toolChoice !== 'any') {
-                try {
-                    toolChoice = JSON.parse(toolChoice);
-                } catch (e) {
-                    toolChoice = 'auto';
-                }
-            }
-            
             if (toolChoice) {
                 if (typeof toolChoice === 'string') {
                     if (toolChoice === 'auto') {
@@ -299,6 +299,10 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                     };
                 }
             }
+        } else if (toolChoice === 'none') {
+            // Even when no tools are provided, if tool_choice is 'none', explicitly disable function calling
+            // This prevents MALFORMED_FUNCTION_CALL errors when chat history contains function messages
+            baseParameters.toolConfig = { functionCallingConfig: { mode: 'NONE' } };
         }
 
         return baseParameters;

From 0f0c8d99c30c74b27629661d9dcb71755d1b603c Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Fri, 19 Dec 2025 14:20:29 -0700
Subject: [PATCH 24/27] feat: add file collection cache invalidation and
 serialization for edits

- Introduced a new function to invalidate the file collection cache, ensuring that updates reflect immediately in subsequent operations.
- Implemented serialization for file edit operations to prevent concurrent modifications, enhancing data integrity during file updates.
- Updated file collection management to ensure cache is invalidated after file removals and edits, improving consistency in file operations.
- Enhanced tests to validate the new cache invalidation and serialization features, ensuring robust functionality across file operations.
---
 lib/fileUtils.js                              |  15 +-
 .../system/entity/tools/sys_tool_editfile.js  | 555 ++++++++++--------
 .../entity/tools/sys_tool_file_collection.js  |  20 +-
 .../features/tools/fileCollection.test.js     |  99 +++-
 .../features/tools/fileOperations.test.js     | 326 ++++++++++
 5 files changed, 768 insertions(+), 247 deletions(-)

diff --git a/lib/fileUtils.js b/lib/fileUtils.js
index c4a9c79d..d8a2bd1d 100644
--- a/lib/fileUtils.js
+++ b/lib/fileUtils.js
@@ -440,6 +440,16 @@ function getCollectionCacheKey(contextId, contextKey) {
     return `${contextId}-fileCollection-${contextKey || 'default'}`;
 }
 
+/**
+ * Invalidate file collection cache for a given context
+ * @param {string} contextId - Context ID for the file collection
+ * @param {string} contextKey - Optional context key for encryption
+ */
+export function invalidateFileCollectionCache(contextId, contextKey = null) {
+    const cacheKey = getCollectionCacheKey(contextId, contextKey);
+    fileCollectionCache.delete(cacheKey);
+}
+
 /**
  * Extract files from chat history
  * @param {Array} chatHistory - Chat history to scan
@@ -832,9 +842,8 @@ async function updateFileMetadata(contextId, hash, metadata, contextKey = null)
         // Write back to hash map (atomic operation) - encryption happens in helper
         await writeFileDataToRedis(redisClient, contextMapKey, hash, fileData, contextKey);
         
-        // Invalidate cache
-        const cacheKey = getCollectionCacheKey(contextId, null);
-        fileCollectionCache.delete(cacheKey);
+        // Invalidate cache (use contextKey to match the correct cache key)
+        invalidateFileCollectionCache(contextId, contextKey);
         
         return true;
     } catch (e) {
diff --git a/pathways/system/entity/tools/sys_tool_editfile.js b/pathways/system/entity/tools/sys_tool_editfile.js
index 11d35fc2..db62091d 100644
--- a/pathways/system/entity/tools/sys_tool_editfile.js
+++ b/pathways/system/entity/tools/sys_tool_editfile.js
@@ -2,7 +2,39 @@
 // Entity tool that modifies existing files by replacing line ranges or exact string matches
 import logger from '../../../../lib/logger.js';
 import { axios } from '../../../../lib/requestExecutor.js';
-import { uploadFileToCloud, findFileInCollection, loadFileCollection, getMimeTypeFromFilename, resolveFileParameter, deleteFileByHash, isTextMimeType, updateFileMetadata, writeFileDataToRedis } from '../../../../lib/fileUtils.js';
+import { uploadFileToCloud, findFileInCollection, loadFileCollection, getMimeTypeFromFilename, resolveFileParameter, deleteFileByHash, isTextMimeType, updateFileMetadata, writeFileDataToRedis, invalidateFileCollectionCache } from '../../../../lib/fileUtils.js';
+
+// In-process serialization: prevents concurrent edits to the same file on this instance
+// Uses promise chaining to execute edits sequentially per file
+const editQueues = new Map();
+
+/**
+ * Serialize edit operations per file to prevent concurrent edits on the same instance
+ * Uses promise chaining to execute edits sequentially. No deadlock risk (single resource lock).
+ * @param {string} contextId - Context ID
+ * @param {string} fileId - File ID
+ * @param {Function} editFn - Async function that performs the edit
+ * @returns {Promise} Promise that resolves when this edit completes
+ */
+async function serializeEdit(contextId, fileId, editFn) {
+    const lockKey = `${contextId}:${fileId}`;
+    
+    // Get existing queue or start with resolved promise
+    let queue = editQueues.get(lockKey) || Promise.resolve();
+    
+    // Chain this operation after the previous one
+    // Timeout protection: pathway timeout (120s) will handle stuck operations
+    const operation = queue.then(editFn).finally(() => {
+        // Cleanup: remove queue if we're still the current one (no new operations queued)
+        // This prevents memory leaks if operations complete
+        if (editQueues.get(lockKey) === operation) {
+            editQueues.delete(lockKey);
+        }
+    });
+    
+    editQueues.set(lockKey, operation);
+    return operation;
+}
 
 export default {
     prompt: [],
@@ -44,7 +76,7 @@ export default {
         },
         {
             type: "function",
-            icon: "🔍",
+            icon: "✏️",
             function: {
                 name: "EditFileBySearchAndReplace",
                 description: "Search and replace exact string matches in a file. Use this when you know the exact text to find and replace. The file must exist in your file collection and must be a text-type file (text, markdown, html, csv, etc.). After modification, the old file version is deleted from cloud storage and the new version is uploaded. The collection entry is updated with the new URL and hash.",
@@ -176,288 +208,341 @@ export default {
         }
 
         try {
-            // Resolve the file parameter to a URL using the common utility
-            const fileUrl = await resolveFileParameter(file, contextId, contextKey);
-            
-            if (!fileUrl) {
-                const errorResult = {
-                    success: false,
-                    error: `File not found: "${file}". Use ListFileCollection or SearchFileCollection to find available files.`
-                };
-                resolver.tool = JSON.stringify({ toolUsed: toolName });
-                return JSON.stringify(errorResult);
-            }
-
-            // Find the file in the collection to get metadata (for updating later)
-            // We'll load it again inside the lock, but need to verify it exists first
-            const collection = await loadFileCollection(contextId, contextKey, true);
+            // Resolve file ID first (needed for serialization)
+            const collection = await loadFileCollection(contextId, contextKey, false);
             const foundFile = findFileInCollection(file, collection);
-
+            
             if (!foundFile) {
                 const errorResult = {
                     success: false,
-                    error: `File not found in collection: "${file}"`
+                    error: `File not found in collection: "${file}". Use ListFileCollection or SearchFileCollection to find available files.`
                 };
                 resolver.tool = JSON.stringify({ toolUsed: toolName });
                 return JSON.stringify(errorResult);
             }
             
-            // Store the file ID for updating inside the lock
-            const fileIdToUpdate = foundFile.id;
-
-            // Download the current file content
-            logger.info(`Downloading file for modification: ${fileUrl}`);
-            const downloadResponse = await axios.get(fileUrl, {
-                responseType: 'arraybuffer',
-                timeout: 60000,
-                validateStatus: (status) => status >= 200 && status < 400
-            });
-
-            if (downloadResponse.status !== 200 || !downloadResponse.data) {
-                throw new Error(`Failed to download file: ${downloadResponse.status}`);
-            }
-
-            // Explicitly decode as UTF-8 to prevent mojibake (encoding corruption)
-            const originalContent = Buffer.from(downloadResponse.data).toString('utf8');
-            let modifiedContent;
-            let modificationInfo = {};
-
-            if (isEditByLine) {
-                // Line-based replacement mode
-                const allLines = originalContent.split(/\r?\n/);
-                const totalLines = allLines.length;
-
-                // Validate line range
-                if (startLine > totalLines) {
+            const fileId = foundFile.id;
+            
+            // Serialize edits to this file (prevents concurrent edits on same instance)
+            return await serializeEdit(contextId, fileId, async () => {
+                // CRITICAL: Reload collection FIRST to get latest file data (may have changed from previous serialized edit)
+                // This must happen inside serializeEdit to ensure we see the previous edit's changes
+                const currentCollection = await loadFileCollection(contextId, contextKey, false);
+                const currentFile = findFileInCollection(file, currentCollection);
+
+                if (!currentFile) {
                     const errorResult = {
                         success: false,
-                        error: `startLine (${startLine}) exceeds file length (${totalLines} lines)`
+                        error: `File not found in collection: "${file}"`
                     };
-                    resolver.tool = JSON.stringify({ toolUsed: "EditFileByLine" });
+                    resolver.tool = JSON.stringify({ toolUsed: toolName });
                     return JSON.stringify(errorResult);
                 }
-
-                // Perform the line replacement
-                const startIndex = startLine - 1;
-                const endIndex = Math.min(endLine, totalLines);
-                
-                // Split the replacement content into lines
-                const replacementLines = content.split(/\r?\n/);
                 
-                // Build the modified content
-                const beforeLines = allLines.slice(0, startIndex);
-                const afterLines = allLines.slice(endIndex);
-                const modifiedLines = [...beforeLines, ...replacementLines, ...afterLines];
-                modifiedContent = modifiedLines.join('\n');
-
-                modificationInfo = {
-                    mode: 'line-based',
-                    originalLines: totalLines,
-                    modifiedLines: modifiedLines.length,
-                    replacedLines: endLine - startLine + 1,
-                    insertedLines: replacementLines.length,
-                    startLine: startLine,
-                    endLine: endLine
-                };
-            } else if (isSearchReplace) {
-                // Search and replace mode
-                if (!originalContent.includes(oldString)) {
+                // Store the file ID for updating
+                let fileIdToUpdate = currentFile.id;
+
+                // Resolve file URL AFTER reloading collection to ensure we get the latest URL
+                // Use the file from the reloaded collection, not the initial resolution
+                const fileUrl = currentFile.url;
+            
+                if (!fileUrl) {
                     const errorResult = {
                         success: false,
-                        error: `oldString not found in file. The exact string must match (including whitespace and newlines).`
+                        error: `File URL not found for: "${file}". The file may have been modified or removed.`
                     };
-                    resolver.tool = JSON.stringify({ toolUsed: "EditFileBySearchAndReplace" });
+                    resolver.tool = JSON.stringify({ toolUsed: toolName });
                     return JSON.stringify(errorResult);
                 }
 
-                // Count occurrences
-                const occurrences = (originalContent.match(new RegExp(oldString.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g')) || []).length;
-                
-                if (replaceAll) {
-                    modifiedContent = originalContent.split(oldString).join(newString);
-                    modificationInfo = {
-                        mode: 'string-based',
-                        replaceAll: true,
-                        occurrencesReplaced: occurrences
-                    };
-                } else {
-                    // Replace only first occurrence
-                    modifiedContent = originalContent.replace(oldString, newString);
+                // Download the current file content
+                logger.info(`Downloading file for modification: ${fileUrl}`);
+                const downloadResponse = await axios.get(fileUrl, {
+                    responseType: 'arraybuffer',
+                    timeout: 60000,
+                    validateStatus: (status) => status >= 200 && status < 400
+                });
+
+                if (downloadResponse.status !== 200 || !downloadResponse.data) {
+                    throw new Error(`Failed to download file: ${downloadResponse.status}`);
+                }
+
+                // Explicitly decode as UTF-8 to prevent mojibake (encoding corruption)
+                const originalContent = Buffer.from(downloadResponse.data).toString('utf8');
+                let modifiedContent;
+                let modificationInfo = {};
+
+                if (isEditByLine) {
+                    // Line-based replacement mode
+                    const allLines = originalContent.split(/\r?\n/);
+                    const totalLines = allLines.length;
+
+                    // Validate line range
+                    if (startLine > totalLines) {
+                        const errorResult = {
+                            success: false,
+                            error: `startLine (${startLine}) exceeds file length (${totalLines} lines)`
+                        };
+                        resolver.tool = JSON.stringify({ toolUsed: "EditFileByLine" });
+                        return JSON.stringify(errorResult);
+                    }
+
+                    // Perform the line replacement
+                    const startIndex = startLine - 1;
+                    const endIndex = Math.min(endLine, totalLines);
+                    
+                    // Split the replacement content into lines
+                    const replacementLines = content.split(/\r?\n/);
+                    
+                    // Build the modified content
+                    const beforeLines = allLines.slice(0, startIndex);
+                    const afterLines = allLines.slice(endIndex);
+                    const modifiedLines = [...beforeLines, ...replacementLines, ...afterLines];
+                    modifiedContent = modifiedLines.join('\n');
+
                     modificationInfo = {
-                        mode: 'string-based',
-                        replaceAll: false,
-                        occurrencesReplaced: 1,
-                        totalOccurrences: occurrences
+                        mode: 'line-based',
+                        originalLines: totalLines,
+                        modifiedLines: modifiedLines.length,
+                        replacedLines: endLine - startLine + 1,
+                        insertedLines: replacementLines.length,
+                        startLine: startLine,
+                        endLine: endLine
                     };
+                } else if (isSearchReplace) {
+                    // Search and replace mode
+                    if (!originalContent.includes(oldString)) {
+                        const errorResult = {
+                            success: false,
+                            error: `oldString not found in file. The exact string must match (including whitespace and newlines).`
+                        };
+                        resolver.tool = JSON.stringify({ toolUsed: "EditFileBySearchAndReplace" });
+                        return JSON.stringify(errorResult);
+                    }
+
+                    // Count occurrences
+                    const occurrences = (originalContent.match(new RegExp(oldString.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g')) || []).length;
+                    
+                    if (replaceAll) {
+                        modifiedContent = originalContent.split(oldString).join(newString);
+                        modificationInfo = {
+                            mode: 'string-based',
+                            replaceAll: true,
+                            occurrencesReplaced: occurrences
+                        };
+                    } else {
+                        // Replace only first occurrence
+                        modifiedContent = originalContent.replace(oldString, newString);
+                        modificationInfo = {
+                            mode: 'string-based',
+                            replaceAll: false,
+                            occurrencesReplaced: 1,
+                            totalOccurrences: occurrences
+                        };
+                    }
                 }
-            }
 
-            // Determine MIME type from filename using utility function
-            // Use displayFilename (user-friendly) if available, otherwise fall back to filename (CFH-managed)
-            const filename = foundFile.displayFilename || foundFile.filename || 'modified.txt';
-            let mimeType = getMimeTypeFromFilename(filename, 'text/plain');
-            
-            // Add charset=utf-8 for text-based MIME types
-            if (isTextMimeType(mimeType)) {
-                mimeType = `${mimeType}; charset=utf-8`;
-            }
+                // Determine MIME type from filename using utility function
+                // Use displayFilename (user-friendly) if available, otherwise fall back to filename (CFH-managed)
+                const filename = currentFile.displayFilename || currentFile.filename || 'modified.txt';
+                let mimeType = getMimeTypeFromFilename(filename, 'text/plain');
+                
+                // Add charset=utf-8 for text-based MIME types
+                if (isTextMimeType(mimeType)) {
+                    mimeType = `${mimeType}; charset=utf-8`;
+                }
 
-            // Upload the modified file FIRST (safer: prevent data loss if upload fails)
-            const fileBuffer = Buffer.from(modifiedContent, 'utf8');
-            const uploadResult = await uploadFileToCloud(
-                fileBuffer,
-                mimeType,
-                filename,
-                resolver,
-                contextId
-            );
-
-            if (!uploadResult || !uploadResult.url) {
-                throw new Error('Failed to upload modified file to cloud storage');
-            }
+                // Upload the modified file FIRST (safer: prevent data loss if upload fails)
+                const fileBuffer = Buffer.from(modifiedContent, 'utf8');
+                const uploadResult = await uploadFileToCloud(
+                    fileBuffer,
+                    mimeType,
+                    filename,
+                    resolver,
+                    contextId
+                );
+
+                if (!uploadResult || !uploadResult.url) {
+                    throw new Error('Failed to upload modified file to cloud storage');
+                }
 
-            // Update the file collection entry directly (atomic operation)
-            // First find the file to get its current hash
-            const currentCollection = await loadFileCollection(contextId, contextKey, false);
-            const fileToUpdate = currentCollection.find(f => f.id === fileIdToUpdate);
-            if (!fileToUpdate) {
-                throw new Error(`File with ID "${fileIdToUpdate}" not found in collection`);
-            }
-            
-            const oldHashToDelete = fileToUpdate.hash || null;
-            
-            // Write new entry with CFH data (url, gcs, hash) + Cortex metadata
-            // If hash changed, this creates a new entry; if same hash, it updates the existing one
-            if (uploadResult.hash) {
-                const { getRedisClient } = await import('../../../../lib/fileUtils.js');
-                const redisClient = await getRedisClient();
-                if (redisClient) {
-                    const contextMapKey = `FileStoreMap:ctx:${contextId}`;
-                    
-                    // Get existing CFH data for the new hash (if any)
-                    const existingDataStr = await redisClient.hget(contextMapKey, uploadResult.hash);
-                    let existingData = {};
-                    if (existingDataStr) {
-                        try {
-                            existingData = JSON.parse(existingDataStr);
-                        } catch (e) {
-                            existingData = {};
+                // Update the file collection entry directly (atomic operation)
+                // Reload collection to get the latest file data (important after prior edits)
+                const latestCollection = await loadFileCollection(contextId, contextKey, false);
+                let fileToUpdate = latestCollection.find(f => f.id === fileIdToUpdate);
+                
+                // If not found by ID, try to find by the original file parameter (in case lookup by ID failed)
+                if (!fileToUpdate) {
+                    fileToUpdate = findFileInCollection(file, latestCollection);
+                    if (fileToUpdate) {
+                        // Update fileIdToUpdate to use the found file's ID
+                        fileIdToUpdate = fileToUpdate.id;
+                    }
+                }
+                
+                if (!fileToUpdate) {
+                    throw new Error(`File with ID "${fileIdToUpdate}" not found in collection. The file may have been modified or removed.`);
+                }
+                
+                const oldHashToDelete = fileToUpdate.hash || null;
+                
+                // Write new entry with CFH data (url, gcs, hash) + Cortex metadata
+                // If hash changed, this creates a new entry; if same hash, it updates the existing one
+                if (uploadResult.hash) {
+                    const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+                    const redisClient = await getRedisClient();
+                    if (redisClient) {
+                        const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+                        
+                        // Get existing CFH data for the new hash (if any)
+                        const existingDataStr = await redisClient.hget(contextMapKey, uploadResult.hash);
+                        let existingData = {};
+                        if (existingDataStr) {
+                            try {
+                                existingData = JSON.parse(existingDataStr);
+                            } catch (e) {
+                                existingData = {};
+                            }
+                        }
+                        
+                        // Merge CFH data (url, gcs, hash) with Cortex metadata
+                        const fileData = {
+                            ...existingData, // Preserve any existing CFH data
+                            // CFH-managed fields (from upload result)
+                            url: uploadResult.url,
+                            gcs: uploadResult.gcs || null,
+                            hash: uploadResult.hash,
+                            filename: uploadResult.filename || fileToUpdate.filename || filename, // Use CFH filename if available, otherwise preserve
+                            // Cortex-managed metadata
+                            id: fileToUpdate.id, // Keep same ID
+                            displayFilename: fileToUpdate.displayFilename || filename, // Preserve user-friendly filename
+                            tags: fileToUpdate.tags || [],
+                            notes: fileToUpdate.notes || '',
+                            mimeType: fileToUpdate.mimeType || mimeType || null,
+                            inCollection: ['*'], // Mark as global chat file (available to all chats)
+                            addedDate: fileToUpdate.addedDate, // Keep original added date
+                            lastAccessed: new Date().toISOString(),
+                            permanent: fileToUpdate.permanent || false
+                        };
+                        
+                        // Write new entry (atomic operation) - encryption happens in helper
+                        await writeFileDataToRedis(redisClient, contextMapKey, uploadResult.hash, fileData, contextKey);
+                        
+                        // If hash changed, remove old entry
+                        if (oldHashToDelete && oldHashToDelete !== uploadResult.hash) {
+                            await redisClient.hdel(contextMapKey, oldHashToDelete);
                         }
+                        
+                        // Invalidate cache immediately so subsequent operations get fresh data
+                        invalidateFileCollectionCache(contextId, contextKey);
                     }
+                } else if (fileToUpdate.hash) {
+                    // Same hash, just update Cortex metadata (filename, lastAccessed)
+                    await updateFileMetadata(contextId, fileToUpdate.hash, {
+                        filename: filename,
+                        lastAccessed: new Date().toISOString()
+                    }, contextKey);
                     
-                    // Merge CFH data (url, gcs, hash) with Cortex metadata
-                    const fileData = {
-                        ...existingData, // Preserve any existing CFH data
-                        // CFH-managed fields (from upload result)
+                    // Invalidate cache after metadata update
+                    invalidateFileCollectionCache(contextId, contextKey);
+                }
+
+                // Now it is safe to delete the old file version (after lock succeeds)
+                // This ensures we're deleting the correct hash even if concurrent edits occurred
+                if (oldHashToDelete) {
+                    // Fire-and-forget async deletion for better performance, but log errors
+                    // We don't want to fail the whole operation if cleanup fails, since we have the new file
+                    (async () => {
+                        try {
+                            logger.info(`Deleting old file version with hash ${oldHashToDelete} (background task)`);
+                            await deleteFileByHash(oldHashToDelete, resolver, contextId);
+                        } catch (cleanupError) {
+                            logger.warn(`Failed to cleanup old file version (hash: ${oldHashToDelete}): ${cleanupError.message}`);
+                        }
+                    })().catch(err => logger.error(`Async cleanup error: ${err}`));
+                } else {
+                    logger.info(`No hash found for old file, skipping deletion`);
+                }
+                
+                // Get the updated file info for the result
+                // Use useCache: false to ensure we get fresh data after Redis write
+                const updatedCollection = await loadFileCollection(contextId, contextKey, false);
+                const updatedFile = updatedCollection.find(f => f.id === fileIdToUpdate);
+                
+                if (!updatedFile) {
+                    logger.warn(`File with ID "${fileIdToUpdate}" not found in updated collection. This may indicate a timing issue.`);
+                    // Fall back to using uploadResult data directly
+                    const fallbackFile = {
+                        id: fileIdToUpdate,
                         url: uploadResult.url,
-                        gcs: uploadResult.gcs || null,
-                        hash: uploadResult.hash,
-                        filename: uploadResult.filename || fileToUpdate.filename || filename, // Use CFH filename if available, otherwise preserve
-                        // Cortex-managed metadata
-                        id: fileToUpdate.id, // Keep same ID
-                        displayFilename: fileToUpdate.displayFilename || filename, // Preserve user-friendly filename
-                        tags: fileToUpdate.tags || [],
-                        notes: fileToUpdate.notes || '',
-                        mimeType: fileToUpdate.mimeType || mimeType || null,
-                        inCollection: ['*'], // Mark as global chat file (available to all chats)
-                        addedDate: fileToUpdate.addedDate, // Keep original added date
-                        lastAccessed: new Date().toISOString(),
-                        permanent: fileToUpdate.permanent || false
+                        hash: uploadResult.hash
                     };
-                    
-                    // Write new entry (atomic operation) - encryption happens in helper
-                    await writeFileDataToRedis(redisClient, contextMapKey, uploadResult.hash, fileData, contextKey);
-                    
-                    // If hash changed, remove old entry
-                    if (oldHashToDelete && oldHashToDelete !== uploadResult.hash) {
-                        await redisClient.hdel(contextMapKey, oldHashToDelete);
-                    }
+                    logger.info(`Using fallback file data: ${JSON.stringify(fallbackFile)}`);
                 }
-            } else if (fileToUpdate.hash) {
-                // Same hash, just update Cortex metadata (filename, lastAccessed)
-                await updateFileMetadata(contextId, fileToUpdate.hash, {
-                    filename: filename,
-                    lastAccessed: new Date().toISOString()
-                }, contextKey);
-            }
 
-            // Now it is safe to delete the old file version (after lock succeeds)
-            // This ensures we're deleting the correct hash even if concurrent edits occurred
-            if (oldHashToDelete) {
-                // Fire-and-forget async deletion for better performance, but log errors
-                // We don't want to fail the whole operation if cleanup fails, since we have the new file
-                (async () => {
-                    try {
-                        logger.info(`Deleting old file version with hash ${oldHashToDelete} (background task)`);
-                        await deleteFileByHash(oldHashToDelete, resolver, contextId);
-                    } catch (cleanupError) {
-                        logger.warn(`Failed to cleanup old file version (hash: ${oldHashToDelete}): ${cleanupError.message}`);
+                // Build result message
+                let message;
+                if (isEditByLine) {
+                    message = `File "${filename}" modified successfully. Replaced lines ${startLine}-${endLine} (${endLine - startLine + 1} lines) with ${modificationInfo.insertedLines} line(s).`;
+                } else if (isSearchReplace) {
+                    if (replaceAll) {
+                        message = `File "${filename}" modified successfully. Replaced all ${modificationInfo.occurrencesReplaced} occurrence(s) of the specified string.`;
+                    } else {
+                        message = `File "${filename}" modified successfully. Replaced first occurrence of the specified string${modificationInfo.totalOccurrences > 1 ? ` (${modificationInfo.totalOccurrences} total occurrences found)` : ''}.`;
                     }
-                })().catch(err => logger.error(`Async cleanup error: ${err}`));
-            } else {
-                logger.info(`No hash found for old file, skipping deletion`);
-            }
-            
-            // Get the updated file info for the result
-            // Use useCache: false to ensure we get fresh data after Redis write
-            const updatedCollection = await loadFileCollection(contextId, contextKey, false);
-            const updatedFile = updatedCollection.find(f => f.id === fileIdToUpdate);
-            
-            if (!updatedFile) {
-                logger.warn(`File with ID "${fileIdToUpdate}" not found in updated collection. This may indicate a timing issue.`);
-                // Fall back to using uploadResult data directly
-                const fallbackFile = {
-                    id: fileIdToUpdate,
-                    url: uploadResult.url,
-                    hash: uploadResult.hash
-                };
-                logger.info(`Using fallback file data: ${JSON.stringify(fallbackFile)}`);
-            }
+                }
 
-            // Build result message
-            let message;
-            if (isEditByLine) {
-                message = `File "${filename}" modified successfully. Replaced lines ${startLine}-${endLine} (${endLine - startLine + 1} lines) with ${modificationInfo.insertedLines} line(s).`;
-            } else if (isSearchReplace) {
-                if (replaceAll) {
-                    message = `File "${filename}" modified successfully. Replaced all ${modificationInfo.occurrencesReplaced} occurrence(s) of the specified string.`;
+                const result = {
+                    success: true,
+                    filename: filename,
+                    fileId: updatedFile?.id || fileIdToUpdate,
+                    url: uploadResult.url, // Always use the new URL from upload
+                    gcs: uploadResult.gcs || null,
+                    hash: uploadResult.hash || null,
+                    ...modificationInfo,
+                    message: message
+                };
+                
+                // Log for debugging
+                if (!updatedFile) {
+                    logger.warn(`EditFile: Could not find updated file in collection, but upload succeeded. Using uploadResult URL: ${uploadResult.url}`);
                 } else {
-                    message = `File "${filename}" modified successfully. Replaced first occurrence of the specified string${modificationInfo.totalOccurrences > 1 ? ` (${modificationInfo.totalOccurrences} total occurrences found)` : ''}.`;
+                    logger.info(`EditFile: Successfully updated file. New URL: ${uploadResult.url}, New hash: ${uploadResult.hash}`);
                 }
-            }
 
-            const result = {
-                success: true,
-                filename: filename,
-                fileId: updatedFile?.id || fileIdToUpdate,
-                url: uploadResult.url, // Always use the new URL from upload
-                gcs: uploadResult.gcs || null,
-                hash: uploadResult.hash || null,
-                ...modificationInfo,
-                message: message
-            };
-            
-            // Log for debugging
-            if (!updatedFile) {
-                logger.warn(`EditFile: Could not find updated file in collection, but upload succeeded. Using uploadResult URL: ${uploadResult.url}`);
-            } else {
-                logger.info(`EditFile: Successfully updated file. New URL: ${uploadResult.url}, New hash: ${uploadResult.hash}`);
-            }
-
-            resolver.tool = JSON.stringify({ toolUsed: toolName });
-            return JSON.stringify(result);
+                resolver.tool = JSON.stringify({ toolUsed: toolName });
+                return JSON.stringify(result);
+            }).catch(error => {
+                let errorMsg;
+                if (error?.message) {
+                    errorMsg = error.message;
+                } else if (error?.errors && Array.isArray(error.errors)) {
+                    // Handle AggregateError
+                    errorMsg = error.errors.map(e => e?.message || String(e)).join('; ');
+                } else {
+                    errorMsg = String(error);
+                }
+                logger.error(`Error modifying file: ${errorMsg}`);
+                
+                const errorResult = {
+                    success: false,
+                    error: errorMsg
+                };
 
+                resolver.tool = JSON.stringify({ toolUsed: toolName });
+                return JSON.stringify(errorResult);
+            });
         } catch (error) {
+            // Handle errors before serialization (file not found, validation errors, etc.)
             let errorMsg;
             if (error?.message) {
                 errorMsg = error.message;
             } else if (error?.errors && Array.isArray(error.errors)) {
-                // Handle AggregateError
                 errorMsg = error.errors.map(e => e?.message || String(e)).join('; ');
             } else {
                 errorMsg = String(error);
             }
-            logger.error(`Error modifying file: ${errorMsg}`);
+            logger.error(`Error in file edit operation: ${errorMsg}`);
             
             const errorResult = {
                 success: false,
diff --git a/pathways/system/entity/tools/sys_tool_file_collection.js b/pathways/system/entity/tools/sys_tool_file_collection.js
index 72f3aa06..a33b5787 100644
--- a/pathways/system/entity/tools/sys_tool_file_collection.js
+++ b/pathways/system/entity/tools/sys_tool_file_collection.js
@@ -2,7 +2,7 @@
 // Tool pathway that manages user file collections (add, search, list files)
 // Uses Redis hash maps (FileStoreMap:ctx:<contextId>) for storage
 import logger from '../../../../lib/logger.js';
-import { addFileToCollection, loadFileCollection, findFileInCollection, deleteFileByHash, updateFileMetadata } from '../../../../lib/fileUtils.js';
+import { addFileToCollection, loadFileCollection, findFileInCollection, deleteFileByHash, updateFileMetadata, invalidateFileCollectionCache } from '../../../../lib/fileUtils.js';
 
 export default {
     prompt: [],
@@ -239,9 +239,14 @@ export default {
                 
                 // Filter and sort results (for display only, not modifying)
                 let results = updatedFiles.filter(file => {
-                    const displayFilename = file.displayFilename || '';
+                    // Fallback to filename if displayFilename is not set
+                    const displayFilename = file.displayFilename || file.filename || '';
+                    const filename = file.filename || '';
                     
-                    const filenameMatch = displayFilename.toLowerCase().includes(queryLower);
+                    // Check both displayFilename and filename for matches
+                    // (displayFilename may be different from filename, so check both)
+                    const filenameMatch = displayFilename.toLowerCase().includes(queryLower) || 
+                                         (filename && filename !== displayFilename && filename.toLowerCase().includes(queryLower));
                     const notesMatch = file.notes && file.notes.toLowerCase().includes(queryLower);
                     const tagMatch = Array.isArray(file.tags) && file.tags.some(tag => tag.toLowerCase().includes(queryLower));
                     
@@ -358,12 +363,11 @@ export default {
                     for (const fileInfo of hashesToDelete) {
                         await redisClient.hdel(contextMapKey, fileInfo.hash);
                     }
-                    
-                    // Invalidate cache
-                    const { getCollectionCacheKey } = await import('../../../../lib/fileUtils.js');
-                    const cacheKey = getCollectionCacheKey(contextId, contextKey);
-                    // Cache is in fileUtils, we'll let it expire naturally
                 }
+                
+                // Always invalidate cache immediately so list operations reflect removals
+                // (even if Redis operations failed, cache might be stale)
+                invalidateFileCollectionCache(contextId, contextKey);
 
                 // Delete files from cloud storage ASYNC (fire and forget, but log errors)
                 // We do this after updating collection so user gets fast response and files are "gone" from UI immediately
diff --git a/tests/integration/features/tools/fileCollection.test.js b/tests/integration/features/tools/fileCollection.test.js
index d20008aa..cacd81c0 100644
--- a/tests/integration/features/tools/fileCollection.test.js
+++ b/tests/integration/features/tools/fileCollection.test.js
@@ -175,6 +175,49 @@ test('File collection: Search files', async t => {
     }
 });
 
+test('File collection: Search by filename when displayFilename not set', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add file with only filename (no displayFilename)
+        // This tests the bug fix where search only checked displayFilename
+        await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/smoketest-tools.txt',
+            filename: 'smoketest-tools.txt',
+            tags: ['smoketest', 'text'],
+            notes: 'Created to test SearchFileCollection',
+            userMessage: 'Add smoketest file'
+        });
+        
+        // Search by filename - should find it even if displayFilename not set
+        const result1 = await callPathway('sys_tool_file_collection', {
+            contextId,
+            query: 'smoketest',
+            userMessage: 'Search for smoketest'
+        });
+        
+        const parsed1 = JSON.parse(result1);
+        t.is(parsed1.success, true);
+        t.is(parsed1.count, 1);
+        t.true(parsed1.files[0].displayFilename === 'smoketest-tools.txt' || 
+               parsed1.files[0].filename === 'smoketest-tools.txt');
+        
+        // Search by full filename
+        const result2 = await callPathway('sys_tool_file_collection', {
+            contextId,
+            query: 'smoketest-tools',
+            userMessage: 'Search for smoketest-tools'
+        });
+        
+        const parsed2 = JSON.parse(result2);
+        t.is(parsed2.success, true);
+        t.is(parsed2.count, 1);
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
 test('File collection: Remove single file', async t => {
     const contextId = createTestContext();
     
@@ -210,7 +253,7 @@ test('File collection: Remove single file', async t => {
         t.is(parsed.removedFiles[0].displayFilename, 'file1.jpg');
         t.true(parsed.message.includes('Cloud storage cleanup started in background'));
         
-        // Verify it was removed
+        // Verify it was removed (cache should be invalidated immediately)
         const listResult = await callPathway('sys_tool_file_collection', {
             contextId,
             userMessage: 'List files'
@@ -224,6 +267,60 @@ test('File collection: Remove single file', async t => {
     }
 });
 
+test('File collection: Remove file - cache invalidation', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add files
+        const addResult1 = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/file1.jpg',
+            filename: 'file1.jpg',
+            userMessage: 'Add file 1'
+        });
+        const file1Id = JSON.parse(addResult1).fileId;
+        
+        const addResult2 = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/file2.pdf',
+            filename: 'file2.pdf',
+            userMessage: 'Add file 2'
+        });
+        const file2Id = JSON.parse(addResult2).fileId;
+        
+        // Verify both files are in collection
+        const listBefore = await callPathway('sys_tool_file_collection', {
+            contextId,
+            userMessage: 'List files before removal'
+        });
+        const listBeforeParsed = JSON.parse(listBefore);
+        t.is(listBeforeParsed.totalFiles, 2);
+        
+        // Remove file1
+        const removeResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            fileIds: [file1Id],
+            userMessage: 'Remove file 1'
+        });
+        
+        const removeParsed = JSON.parse(removeResult);
+        t.is(removeParsed.success, true);
+        t.is(removeParsed.removedCount, 1);
+        
+        // Immediately list files - should reflect removal (cache invalidation test)
+        const listAfter = await callPathway('sys_tool_file_collection', {
+            contextId,
+            userMessage: 'List files after removal'
+        });
+        const listAfterParsed = JSON.parse(listAfter);
+        t.is(listAfterParsed.totalFiles, 1, 'List should immediately reflect removal (cache invalidated)');
+        t.false(listAfterParsed.files.some(f => (f.displayFilename || f.filename) === 'file1.jpg'));
+        t.true(listAfterParsed.files.some(f => (f.displayFilename || f.filename) === 'file2.pdf'));
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
 test('File collection: Remove multiple files', async t => {
     const contextId = createTestContext();
     
diff --git a/tests/integration/features/tools/fileOperations.test.js b/tests/integration/features/tools/fileOperations.test.js
index b6ae39b9..5f00f168 100644
--- a/tests/integration/features/tools/fileOperations.test.js
+++ b/tests/integration/features/tools/fileOperations.test.js
@@ -539,6 +539,140 @@ test('EditFileByLine: Error handling - invalid line range', async t => {
     }
 });
 
+test('EditFileByLine: Works after prior SearchAndReplace edit', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Write initial file
+        const initialContent = 'Version: v1\nLine2: alpha\nLine3: bravo\nLine4: charlie';
+        const writeResult = await callPathway('sys_tool_writefile', {
+            contextId,
+            content: initialContent,
+            filename: 'smoketest-tools.txt',
+            userMessage: 'Writing file for sequential edit test'
+        });
+        
+        const writeParsed = JSON.parse(writeResult);
+        
+        if (!writeParsed.success && writeParsed.error?.includes('WHISPER_MEDIA_API_URL')) {
+            t.log('Test skipped - file handler URL not configured');
+            t.pass();
+            return;
+        }
+        
+        t.is(writeParsed.success, true);
+        const fileId = writeParsed.fileId || 'smoketest-tools.txt';
+        
+        await new Promise(resolve => setTimeout(resolve, 500));
+        
+        // First edit: SearchAndReplace (changes hash)
+        const searchReplaceResult = await callPathway('sys_tool_editfile', {
+            contextId,
+            file: fileId,
+            oldString: 'Version: v1',
+            newString: 'Version: v2',
+            replaceAll: false,
+            userMessage: 'First edit: SearchAndReplace'
+        });
+        
+        const searchReplaceParsed = JSON.parse(searchReplaceResult);
+        t.is(searchReplaceParsed.success, true);
+        t.truthy(searchReplaceParsed.url);
+        t.truthy(searchReplaceParsed.hash);
+        
+        // Wait a moment for collection to update
+        await new Promise(resolve => setTimeout(resolve, 500));
+        
+        // Second edit: EditFileByLine (should work after hash change)
+        const editByLineResult = await callPathway('sys_tool_editfile', {
+            contextId,
+            file: fileId, // Use same fileId - should resolve correctly after hash change
+            startLine: 3,
+            endLine: 3,
+            content: 'Line3: BRAVO_EDITED',
+            userMessage: 'Second edit: EditFileByLine after SearchAndReplace'
+        });
+        
+        const editByLineParsed = JSON.parse(editByLineResult);
+        t.is(editByLineParsed.success, true, 'EditFileByLine should work after prior SearchAndReplace edit');
+        t.is(editByLineParsed.replacedLines, 1);
+        t.is(editByLineParsed.insertedLines, 1);
+        
+        // Verify final content
+        await new Promise(resolve => setTimeout(resolve, 500));
+        const readResult = await callPathway('sys_tool_readfile', {
+            contextId,
+            file: fileId,
+            userMessage: 'Reading final file content'
+        });
+        
+        const readParsed = JSON.parse(readResult);
+        t.is(readParsed.success, true);
+        t.true(readParsed.content.includes('Version: v2'), 'Should have v2 from SearchAndReplace');
+        t.true(readParsed.content.includes('BRAVO_EDITED'), 'Should have edited line from EditFileByLine');
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('ReadTextFile: Gets fresh content after EditFileByLine', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Write initial file
+        const initialContent = 'Line1: alpha\nLine2: bravo\nLine3: charlie';
+        const writeResult = await callPathway('sys_tool_writefile', {
+            contextId,
+            content: initialContent,
+            filename: 'read-after-edit.txt',
+            userMessage: 'Writing file for read-after-edit test'
+        });
+        
+        const writeParsed = JSON.parse(writeResult);
+        
+        if (!writeParsed.success && writeParsed.error?.includes('WHISPER_MEDIA_API_URL')) {
+            t.log('Test skipped - file handler URL not configured');
+            t.pass();
+            return;
+        }
+        
+        t.is(writeParsed.success, true);
+        const fileId = writeParsed.fileId || 'read-after-edit.txt';
+        
+        await new Promise(resolve => setTimeout(resolve, 500));
+        
+        // Edit the file
+        const editResult = await callPathway('sys_tool_editfile', {
+            contextId,
+            file: fileId,
+            startLine: 2,
+            endLine: 2,
+            content: 'Line2: BRAVO_EDITED',
+            userMessage: 'Editing file'
+        });
+        
+        const editParsed = JSON.parse(editResult);
+        t.is(editParsed.success, true);
+        
+        // Wait a moment for collection to update
+        await new Promise(resolve => setTimeout(resolve, 500));
+        
+        // Read file - should get fresh content (not cached)
+        const readResult = await callPathway('sys_tool_readfile', {
+            contextId,
+            file: fileId,
+            userMessage: 'Reading file after edit'
+        });
+        
+        const readParsed = JSON.parse(readResult);
+        t.is(readParsed.success, true);
+        t.true(readParsed.content.includes('BRAVO_EDITED'), 'ReadTextFile should return fresh content after edit');
+        t.false(readParsed.content.includes('Line2: bravo'), 'Should not have old content');
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
 test('EditFileByLine: Error handling - line out of range', async t => {
     const contextId = createTestContext();
     
@@ -879,6 +1013,198 @@ test('EditFile: Old file preserved if upload fails (data integrity)', async t =>
     }
 });
 
+// ========== Serialization Tests ==========
+
+test('EditFile: Concurrent edits are serialized (no race conditions)', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Write initial file with numbered lines
+        const initialContent = 'Line 1\nLine 2\nLine 3\nLine 4\nLine 5';
+        const writeResult = await callPathway('sys_tool_writefile', {
+            contextId,
+            content: initialContent,
+            filename: 'serialization-test.txt',
+            userMessage: 'Writing file for serialization test'
+        });
+        
+        const writeParsed = JSON.parse(writeResult);
+        
+        if (!writeParsed.success && writeParsed.error?.includes('WHISPER_MEDIA_API_URL')) {
+            t.log('Test skipped - file handler URL not configured');
+            t.pass();
+            return;
+        }
+        
+        t.is(writeParsed.success, true);
+        const fileId = writeParsed.fileId || 'serialization-test.txt';
+        
+        await new Promise(resolve => setTimeout(resolve, 500));
+        
+        // Trigger multiple concurrent edits on the same file
+        // Each edit modifies a different line to verify they all apply
+        const editPromises = [
+            callPathway('sys_tool_editfile', {
+                contextId,
+                file: fileId,
+                startLine: 1,
+                endLine: 1,
+                content: 'Line 1: EDIT_A',
+                userMessage: 'Concurrent edit A'
+            }),
+            callPathway('sys_tool_editfile', {
+                contextId,
+                file: fileId,
+                startLine: 2,
+                endLine: 2,
+                content: 'Line 2: EDIT_B',
+                userMessage: 'Concurrent edit B'
+            }),
+            callPathway('sys_tool_editfile', {
+                contextId,
+                file: fileId,
+                startLine: 3,
+                endLine: 3,
+                content: 'Line 3: EDIT_C',
+                userMessage: 'Concurrent edit C'
+            }),
+            callPathway('sys_tool_editfile', {
+                contextId,
+                file: fileId,
+                startLine: 4,
+                endLine: 4,
+                content: 'Line 4: EDIT_D',
+                userMessage: 'Concurrent edit D'
+            })
+        ];
+        
+        // Execute all edits concurrently
+        const editResults = await Promise.all(editPromises);
+        
+        // Verify all edits succeeded
+        const editParsed = editResults.map(r => JSON.parse(r));
+        editParsed.forEach((result, index) => {
+            t.is(result.success, true, `Edit ${String.fromCharCode(65 + index)} should succeed`);
+        });
+        
+        // Wait a moment for collection to update
+        await new Promise(resolve => setTimeout(resolve, 500));
+        
+        // Read the final file content
+        const readResult = await callPathway('sys_tool_readfile', {
+            contextId,
+            file: fileId,
+            userMessage: 'Reading final file after concurrent edits'
+        });
+        
+        const readParsed = JSON.parse(readResult);
+        t.is(readParsed.success, true);
+        
+        // Verify all edits were applied (serialization ensures no lost updates)
+        const lines = readParsed.content.split('\n');
+        t.is(lines[0], 'Line 1: EDIT_A', 'Line 1 should have edit A');
+        t.is(lines[1], 'Line 2: EDIT_B', 'Line 2 should have edit B');
+        t.is(lines[2], 'Line 3: EDIT_C', 'Line 3 should have edit C');
+        t.is(lines[3], 'Line 4: EDIT_D', 'Line 4 should have edit D');
+        t.is(lines[4], 'Line 5', 'Line 5 should be unchanged');
+        
+        // Verify file has exactly 5 lines (no corruption from concurrent edits)
+        t.is(readParsed.totalLines, 5, 'File should have exactly 5 lines');
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('EditFile: Sequential edits maintain order (serialization verification)', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Write initial file
+        const initialContent = 'Version: 0';
+        const writeResult = await callPathway('sys_tool_writefile', {
+            contextId,
+            content: initialContent,
+            filename: 'order-test.txt',
+            userMessage: 'Writing file for order test'
+        });
+        
+        const writeParsed = JSON.parse(writeResult);
+        
+        if (!writeParsed.success && writeParsed.error?.includes('WHISPER_MEDIA_API_URL')) {
+            t.log('Test skipped - file handler URL not configured');
+            t.pass();
+            return;
+        }
+        
+        t.is(writeParsed.success, true);
+        const fileId = writeParsed.fileId || 'order-test.txt';
+        
+        await new Promise(resolve => setTimeout(resolve, 500));
+        
+        // Trigger multiple concurrent edits that each append to the file
+        // If serialization works, each edit should see the previous one's changes
+        const editPromises = [
+            callPathway('sys_tool_editfile', {
+                contextId,
+                file: fileId,
+                startLine: 1,
+                endLine: 1,
+                content: 'Version: 0\nEdit: 1',
+                userMessage: 'Edit 1'
+            }),
+            callPathway('sys_tool_editfile', {
+                contextId,
+                file: fileId,
+                startLine: 1,
+                endLine: 2,
+                content: 'Version: 0\nEdit: 1\nEdit: 2',
+                userMessage: 'Edit 2'
+            }),
+            callPathway('sys_tool_editfile', {
+                contextId,
+                file: fileId,
+                startLine: 1,
+                endLine: 3,
+                content: 'Version: 0\nEdit: 1\nEdit: 2\nEdit: 3',
+                userMessage: 'Edit 3'
+            })
+        ];
+        
+        // Execute all edits concurrently
+        const editResults = await Promise.all(editPromises);
+        
+        // All should succeed (serialization prevents conflicts)
+        const editParsed = editResults.map(r => JSON.parse(r));
+        editParsed.forEach((result, index) => {
+            t.is(result.success, true, `Edit ${index + 1} should succeed`);
+        });
+        
+        // Wait for collection to update
+        await new Promise(resolve => setTimeout(resolve, 500));
+        
+        // Read final content
+        const readResult = await callPathway('sys_tool_readfile', {
+            contextId,
+            file: fileId,
+            userMessage: 'Reading final file'
+        });
+        
+        const readParsed = JSON.parse(readResult);
+        t.is(readParsed.success, true);
+        
+        // Verify final content - should have all edits applied in order
+        // Since edits are serialized, the last one to complete should have the final state
+        const lines = readParsed.content.split('\n');
+        t.true(lines.length >= 3, 'File should have at least 3 lines');
+        t.true(readParsed.content.includes('Version: 0'), 'Should contain original content');
+        t.true(readParsed.content.includes('Edit: 1'), 'Should contain edit 1');
+        t.true(readParsed.content.includes('Edit: 2'), 'Should contain edit 2');
+        t.true(readParsed.content.includes('Edit: 3'), 'Should contain edit 3');
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
 // ========== Integration Tests ==========
 
 test('File Operations: Write, Read, Modify workflow', async t => {

From a1c7edb2e6a5d9e73f72c36a50be284bc3f4b288 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Fri, 19 Dec 2025 15:06:19 -0700
Subject: [PATCH 25/27] feat: implement file size limits and caching for edit
 and write operations

- Introduced maximum file size limits (50MB) for editing and writing files to prevent memory issues during operations.
- Added local caching for file content during edits to optimize performance and reduce redundant downloads/uploads.
- Enhanced error handling to provide clear feedback when file sizes exceed the defined limits, improving user experience.
- Updated the serialization logic for file edits to utilize cached content, ensuring efficient processing of sequential edits.
---
 .../system/entity/tools/sys_tool_editfile.js  | 335 +++++++++++-------
 .../system/entity/tools/sys_tool_writefile.js |  15 +
 2 files changed, 213 insertions(+), 137 deletions(-)

diff --git a/pathways/system/entity/tools/sys_tool_editfile.js b/pathways/system/entity/tools/sys_tool_editfile.js
index db62091d..ba171c18 100644
--- a/pathways/system/entity/tools/sys_tool_editfile.js
+++ b/pathways/system/entity/tools/sys_tool_editfile.js
@@ -4,16 +4,32 @@ import logger from '../../../../lib/logger.js';
 import { axios } from '../../../../lib/requestExecutor.js';
 import { uploadFileToCloud, findFileInCollection, loadFileCollection, getMimeTypeFromFilename, resolveFileParameter, deleteFileByHash, isTextMimeType, updateFileMetadata, writeFileDataToRedis, invalidateFileCollectionCache } from '../../../../lib/fileUtils.js';
 
+// Maximum file size for editing (50MB) - prevents memory blowup on huge files
+const MAX_EDITABLE_FILE_SIZE = 50 * 1024 * 1024;
+
+function formatBytes(bytes) {
+    if (bytes === 0) return '0 B';
+    const k = 1024;
+    const sizes = ['B', 'KB', 'MB', 'GB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return Math.round(bytes / Math.pow(k, i) * 100) / 100 + ' ' + sizes[i];
+}
+
 // In-process serialization: prevents concurrent edits to the same file on this instance
 // Uses promise chaining to execute edits sequentially per file
 const editQueues = new Map();
 
+// Local file cache: avoids repeated downloads/uploads for sequential edits
+// Key: lockKey (contextId:fileId), Value: { content, file, dirty }
+const fileContentCache = new Map();
+
 /**
  * Serialize edit operations per file to prevent concurrent edits on the same instance
  * Uses promise chaining to execute edits sequentially. No deadlock risk (single resource lock).
+ * Also manages local file caching: downloads once, uploads once when session ends.
  * @param {string} contextId - Context ID
  * @param {string} fileId - File ID
- * @param {Function} editFn - Async function that performs the edit
+ * @param {Function} editFn - Async function that performs the edit, receives { cachedContent, cachedFile } or null
  * @returns {Promise} Promise that resolves when this edit completes
  */
 async function serializeEdit(contextId, fileId, editFn) {
@@ -24,11 +40,22 @@ async function serializeEdit(contextId, fileId, editFn) {
     
     // Chain this operation after the previous one
     // Timeout protection: pathway timeout (120s) will handle stuck operations
-    const operation = queue.then(editFn).finally(() => {
+    const operation = queue.then(async () => {
+        // Pass cached content to edit function (if available)
+        const cached = fileContentCache.get(lockKey);
+        const result = await editFn(cached);
+        
+        // Check if we're the last operation (no more edits queued)
+        // If yes, we need to flush (upload); if no, skip upload
+        const isLastOperation = (editQueues.get(lockKey) === operation);
+        
+        return { ...result, _isLastOperation: isLastOperation, _lockKey: lockKey };
+    }).finally(() => {
         // Cleanup: remove queue if we're still the current one (no new operations queued)
         // This prevents memory leaks if operations complete
         if (editQueues.get(lockKey) === operation) {
             editQueues.delete(lockKey);
+            fileContentCache.delete(lockKey); // Clear cache when session ends
         }
     });
     
@@ -36,6 +63,13 @@ async function serializeEdit(contextId, fileId, editFn) {
     return operation;
 }
 
+/**
+ * Update the local file cache with modified content
+ */
+function updateFileCache(lockKey, content, file) {
+    fileContentCache.set(lockKey, { content, file, dirty: true });
+}
+
 export default {
     prompt: [],
     timeout: 120,
@@ -224,51 +258,71 @@ export default {
             const fileId = foundFile.id;
             
             // Serialize edits to this file (prevents concurrent edits on same instance)
-            return await serializeEdit(contextId, fileId, async () => {
-                // CRITICAL: Reload collection FIRST to get latest file data (may have changed from previous serialized edit)
-                // This must happen inside serializeEdit to ensure we see the previous edit's changes
-                const currentCollection = await loadFileCollection(contextId, contextKey, false);
-                const currentFile = findFileInCollection(file, currentCollection);
-
-                if (!currentFile) {
-                    const errorResult = {
-                        success: false,
-                        error: `File not found in collection: "${file}"`
-                    };
-                    resolver.tool = JSON.stringify({ toolUsed: toolName });
-                    return JSON.stringify(errorResult);
-                }
+            // The callback receives cached content if available (from previous edits in this session)
+            const editResult = await serializeEdit(contextId, fileId, async (cached) => {
+                const lockKey = `${contextId}:${fileId}`;
+                let currentFile;
+                let originalContent;
                 
-                // Store the file ID for updating
-                let fileIdToUpdate = currentFile.id;
+                if (cached && cached.content !== undefined) {
+                    // Use cached content from previous edit in this session (skip download)
+                    originalContent = cached.content;
+                    currentFile = cached.file;
+                    logger.info(`Using cached content for: ${currentFile.displayFilename || file}`);
+                } else {
+                    // First edit in session: load collection and download file
+                    const currentCollection = await loadFileCollection(contextId, contextKey, false);
+                    currentFile = findFileInCollection(file, currentCollection);
 
-                // Resolve file URL AFTER reloading collection to ensure we get the latest URL
-                // Use the file from the reloaded collection, not the initial resolution
-                const fileUrl = currentFile.url;
-            
-                if (!fileUrl) {
-                    const errorResult = {
-                        success: false,
-                        error: `File URL not found for: "${file}". The file may have been modified or removed.`
-                    };
-                    resolver.tool = JSON.stringify({ toolUsed: toolName });
-                    return JSON.stringify(errorResult);
-                }
+                    if (!currentFile) {
+                        const errorResult = {
+                            success: false,
+                            error: `File not found in collection: "${file}"`
+                        };
+                        resolver.tool = JSON.stringify({ toolUsed: toolName });
+                        return { jsonResult: JSON.stringify(errorResult) };
+                    }
+                    
+                    const fileUrl = currentFile.url;
+                
+                    if (!fileUrl) {
+                        const errorResult = {
+                            success: false,
+                            error: `File URL not found for: "${file}". The file may have been modified or removed.`
+                        };
+                        resolver.tool = JSON.stringify({ toolUsed: toolName });
+                        return { jsonResult: JSON.stringify(errorResult) };
+                    }
 
-                // Download the current file content
-                logger.info(`Downloading file for modification: ${fileUrl}`);
-                const downloadResponse = await axios.get(fileUrl, {
-                    responseType: 'arraybuffer',
-                    timeout: 60000,
-                    validateStatus: (status) => status >= 200 && status < 400
-                });
+                    // Download the file content
+                    logger.info(`Downloading file for modification: ${fileUrl}`);
+                    const downloadResponse = await axios.get(fileUrl, {
+                        responseType: 'arraybuffer',
+                        timeout: 60000,
+                        validateStatus: (status) => status >= 200 && status < 400
+                    });
 
-                if (downloadResponse.status !== 200 || !downloadResponse.data) {
-                    throw new Error(`Failed to download file: ${downloadResponse.status}`);
-                }
+                    if (downloadResponse.status !== 200 || !downloadResponse.data) {
+                        throw new Error(`Failed to download file: ${downloadResponse.status}`);
+                    }
+
+                    // Check file size to prevent memory blowup
+                    const fileSize = downloadResponse.data.length;
+                    if (fileSize > MAX_EDITABLE_FILE_SIZE) {
+                        const errorResult = {
+                            success: false,
+                            error: `File too large for editing (${formatBytes(fileSize)}). Maximum editable file size is ${formatBytes(MAX_EDITABLE_FILE_SIZE)}. Consider splitting the file or using a different approach.`
+                        };
+                        resolver.tool = JSON.stringify({ toolUsed: toolName });
+                        return { jsonResult: JSON.stringify(errorResult) };
+                    }
 
-                // Explicitly decode as UTF-8 to prevent mojibake (encoding corruption)
-                const originalContent = Buffer.from(downloadResponse.data).toString('utf8');
+                    // Explicitly decode as UTF-8 to prevent mojibake (encoding corruption)
+                    originalContent = Buffer.from(downloadResponse.data).toString('utf8');
+                }
+                
+                // Store the file ID for updating
+                let fileIdToUpdate = currentFile.id;
                 let modifiedContent;
                 let modificationInfo = {};
 
@@ -284,7 +338,7 @@ export default {
                             error: `startLine (${startLine}) exceeds file length (${totalLines} lines)`
                         };
                         resolver.tool = JSON.stringify({ toolUsed: "EditFileByLine" });
-                        return JSON.stringify(errorResult);
+                        return { jsonResult: JSON.stringify(errorResult) };
                     }
 
                     // Perform the line replacement
@@ -292,7 +346,9 @@ export default {
                     const endIndex = Math.min(endLine, totalLines);
                     
                     // Split the replacement content into lines
-                    const replacementLines = content.split(/\r?\n/);
+                    // Strip trailing newlines to prevent extra blank lines being inserted
+                    const trimmedContent = content.replace(/[\r\n]+$/, '');
+                    const replacementLines = trimmedContent.split(/\r?\n/);
                     
                     // Build the modified content
                     const beforeLines = allLines.slice(0, startIndex);
@@ -317,7 +373,7 @@ export default {
                             error: `oldString not found in file. The exact string must match (including whitespace and newlines).`
                         };
                         resolver.tool = JSON.stringify({ toolUsed: "EditFileBySearchAndReplace" });
-                        return JSON.stringify(errorResult);
+                        return { jsonResult: JSON.stringify(errorResult) };
                     }
 
                     // Count occurrences
@@ -352,14 +408,69 @@ export default {
                     mimeType = `${mimeType}; charset=utf-8`;
                 }
 
-                // Upload the modified file FIRST (safer: prevent data loss if upload fails)
+                // Update local cache with modified content
+                // The wrapper will decide whether to upload (only on last operation)
+                updateFileCache(lockKey, modifiedContent, currentFile);
+
+                // Build result message
+                let message;
+                if (isEditByLine) {
+                    message = `File "${filename}" modified successfully. Replaced lines ${startLine}-${endLine} (${endLine - startLine + 1} lines) with ${modificationInfo.insertedLines} line(s).`;
+                } else if (isSearchReplace) {
+                    if (replaceAll) {
+                        message = `File "${filename}" modified successfully. Replaced all ${modificationInfo.occurrencesReplaced} occurrence(s) of the specified string.`;
+                    } else {
+                        message = `File "${filename}" modified successfully. Replaced first occurrence of the specified string${modificationInfo.totalOccurrences > 1 ? ` (${modificationInfo.totalOccurrences} total occurrences found)` : ''}.`;
+                    }
+                }
+
+                // Return edit result with data needed for upload (wrapper handles upload decision)
+                return {
+                    modifiedContent,
+                    currentFile,
+                    fileIdToUpdate,
+                    filename,
+                    mimeType,
+                    modificationInfo,
+                    message,
+                    // Pass these for upload phase
+                    contextId,
+                    contextKey,
+                    resolver,
+                    file, // original file parameter for fallback lookup
+                    isEditByLine,
+                    isSearchReplace,
+                    replaceAll,
+                    startLine,
+                    endLine
+                };
+            });
+
+            // Handle early return (error cases)
+            if (editResult.jsonResult) {
+                return editResult.jsonResult;
+            }
+
+            // Check if we need to upload (only on last operation in queue)
+            if (editResult._isLastOperation) {
+                // Flush: upload the final content and update metadata
+                const { modifiedContent, currentFile, fileIdToUpdate: initialFileId, filename, mimeType, 
+                        modificationInfo, message, contextId: ctxId, contextKey: ctxKey, resolver: res,
+                        file: fileParam, isEditByLine: isByLine, isSearchReplace: isSR, replaceAll: repAll,
+                        startLine: sLine, endLine: eLine } = editResult;
+                
+                let fileIdToUpdate = initialFileId;
+                
+                logger.info(`Flushing cached edits for: ${filename}`);
+                
+                // Upload the modified file
                 const fileBuffer = Buffer.from(modifiedContent, 'utf8');
                 const uploadResult = await uploadFileToCloud(
                     fileBuffer,
                     mimeType,
                     filename,
-                    resolver,
-                    contextId
+                    res,
+                    ctxId
                 );
 
                 if (!uploadResult || !uploadResult.url) {
@@ -367,15 +478,13 @@ export default {
                 }
 
                 // Update the file collection entry directly (atomic operation)
-                // Reload collection to get the latest file data (important after prior edits)
-                const latestCollection = await loadFileCollection(contextId, contextKey, false);
+                const latestCollection = await loadFileCollection(ctxId, ctxKey, false);
                 let fileToUpdate = latestCollection.find(f => f.id === fileIdToUpdate);
                 
-                // If not found by ID, try to find by the original file parameter (in case lookup by ID failed)
+                // If not found by ID, try to find by the original file parameter
                 if (!fileToUpdate) {
-                    fileToUpdate = findFileInCollection(file, latestCollection);
+                    fileToUpdate = findFileInCollection(fileParam, latestCollection);
                     if (fileToUpdate) {
-                        // Update fileIdToUpdate to use the found file's ID
                         fileIdToUpdate = fileToUpdate.id;
                     }
                 }
@@ -387,14 +496,12 @@ export default {
                 const oldHashToDelete = fileToUpdate.hash || null;
                 
                 // Write new entry with CFH data (url, gcs, hash) + Cortex metadata
-                // If hash changed, this creates a new entry; if same hash, it updates the existing one
                 if (uploadResult.hash) {
                     const { getRedisClient } = await import('../../../../lib/fileUtils.js');
                     const redisClient = await getRedisClient();
                     if (redisClient) {
-                        const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+                        const contextMapKey = `FileStoreMap:ctx:${ctxId}`;
                         
-                        // Get existing CFH data for the new hash (if any)
                         const existingDataStr = await redisClient.hget(contextMapKey, uploadResult.hash);
                         let existingData = {};
                         if (existingDataStr) {
@@ -405,135 +512,89 @@ export default {
                             }
                         }
                         
-                        // Merge CFH data (url, gcs, hash) with Cortex metadata
                         const fileData = {
-                            ...existingData, // Preserve any existing CFH data
-                            // CFH-managed fields (from upload result)
+                            ...existingData,
                             url: uploadResult.url,
                             gcs: uploadResult.gcs || null,
                             hash: uploadResult.hash,
-                            filename: uploadResult.filename || fileToUpdate.filename || filename, // Use CFH filename if available, otherwise preserve
-                            // Cortex-managed metadata
-                            id: fileToUpdate.id, // Keep same ID
-                            displayFilename: fileToUpdate.displayFilename || filename, // Preserve user-friendly filename
+                            filename: uploadResult.filename || fileToUpdate.filename || filename,
+                            id: fileToUpdate.id,
+                            displayFilename: fileToUpdate.displayFilename || filename,
                             tags: fileToUpdate.tags || [],
                             notes: fileToUpdate.notes || '',
                             mimeType: fileToUpdate.mimeType || mimeType || null,
-                            inCollection: ['*'], // Mark as global chat file (available to all chats)
-                            addedDate: fileToUpdate.addedDate, // Keep original added date
+                            inCollection: ['*'],
+                            addedDate: fileToUpdate.addedDate,
                             lastAccessed: new Date().toISOString(),
                             permanent: fileToUpdate.permanent || false
                         };
                         
-                        // Write new entry (atomic operation) - encryption happens in helper
-                        await writeFileDataToRedis(redisClient, contextMapKey, uploadResult.hash, fileData, contextKey);
+                        await writeFileDataToRedis(redisClient, contextMapKey, uploadResult.hash, fileData, ctxKey);
                         
-                        // If hash changed, remove old entry
                         if (oldHashToDelete && oldHashToDelete !== uploadResult.hash) {
                             await redisClient.hdel(contextMapKey, oldHashToDelete);
                         }
                         
-                        // Invalidate cache immediately so subsequent operations get fresh data
-                        invalidateFileCollectionCache(contextId, contextKey);
+                        invalidateFileCollectionCache(ctxId, ctxKey);
                     }
                 } else if (fileToUpdate.hash) {
-                    // Same hash, just update Cortex metadata (filename, lastAccessed)
-                    await updateFileMetadata(contextId, fileToUpdate.hash, {
+                    await updateFileMetadata(ctxId, fileToUpdate.hash, {
                         filename: filename,
                         lastAccessed: new Date().toISOString()
-                    }, contextKey);
+                    }, ctxKey);
                     
-                    // Invalidate cache after metadata update
-                    invalidateFileCollectionCache(contextId, contextKey);
+                    invalidateFileCollectionCache(ctxId, ctxKey);
                 }
 
-                // Now it is safe to delete the old file version (after lock succeeds)
-                // This ensures we're deleting the correct hash even if concurrent edits occurred
-                if (oldHashToDelete) {
-                    // Fire-and-forget async deletion for better performance, but log errors
-                    // We don't want to fail the whole operation if cleanup fails, since we have the new file
+                // Delete old file version (fire-and-forget)
+                if (oldHashToDelete && oldHashToDelete !== uploadResult.hash) {
                     (async () => {
                         try {
                             logger.info(`Deleting old file version with hash ${oldHashToDelete} (background task)`);
-                            await deleteFileByHash(oldHashToDelete, resolver, contextId);
+                            await deleteFileByHash(oldHashToDelete, res, ctxId);
                         } catch (cleanupError) {
-                            logger.warn(`Failed to cleanup old file version (hash: ${oldHashToDelete}): ${cleanupError.message}`);
+                            logger.warn(`Failed to cleanup old file version: ${cleanupError.message}`);
                         }
                     })().catch(err => logger.error(`Async cleanup error: ${err}`));
-                } else {
-                    logger.info(`No hash found for old file, skipping deletion`);
-                }
-                
-                // Get the updated file info for the result
-                // Use useCache: false to ensure we get fresh data after Redis write
-                const updatedCollection = await loadFileCollection(contextId, contextKey, false);
-                const updatedFile = updatedCollection.find(f => f.id === fileIdToUpdate);
-                
-                if (!updatedFile) {
-                    logger.warn(`File with ID "${fileIdToUpdate}" not found in updated collection. This may indicate a timing issue.`);
-                    // Fall back to using uploadResult data directly
-                    const fallbackFile = {
-                        id: fileIdToUpdate,
-                        url: uploadResult.url,
-                        hash: uploadResult.hash
-                    };
-                    logger.info(`Using fallback file data: ${JSON.stringify(fallbackFile)}`);
-                }
-
-                // Build result message
-                let message;
-                if (isEditByLine) {
-                    message = `File "${filename}" modified successfully. Replaced lines ${startLine}-${endLine} (${endLine - startLine + 1} lines) with ${modificationInfo.insertedLines} line(s).`;
-                } else if (isSearchReplace) {
-                    if (replaceAll) {
-                        message = `File "${filename}" modified successfully. Replaced all ${modificationInfo.occurrencesReplaced} occurrence(s) of the specified string.`;
-                    } else {
-                        message = `File "${filename}" modified successfully. Replaced first occurrence of the specified string${modificationInfo.totalOccurrences > 1 ? ` (${modificationInfo.totalOccurrences} total occurrences found)` : ''}.`;
-                    }
                 }
 
                 const result = {
                     success: true,
                     filename: filename,
-                    fileId: updatedFile?.id || fileIdToUpdate,
-                    url: uploadResult.url, // Always use the new URL from upload
+                    fileId: fileIdToUpdate,
+                    url: uploadResult.url,
                     gcs: uploadResult.gcs || null,
                     hash: uploadResult.hash || null,
                     ...modificationInfo,
                     message: message
                 };
                 
-                // Log for debugging
-                if (!updatedFile) {
-                    logger.warn(`EditFile: Could not find updated file in collection, but upload succeeded. Using uploadResult URL: ${uploadResult.url}`);
-                } else {
-                    logger.info(`EditFile: Successfully updated file. New URL: ${uploadResult.url}, New hash: ${uploadResult.hash}`);
-                }
+                logger.info(`EditFile: Flushed and uploaded. New URL: ${uploadResult.url}, New hash: ${uploadResult.hash}`);
 
                 resolver.tool = JSON.stringify({ toolUsed: toolName });
                 return JSON.stringify(result);
-            }).catch(error => {
-                let errorMsg;
-                if (error?.message) {
-                    errorMsg = error.message;
-                } else if (error?.errors && Array.isArray(error.errors)) {
-                    // Handle AggregateError
-                    errorMsg = error.errors.map(e => e?.message || String(e)).join('; ');
-                } else {
-                    errorMsg = String(error);
-                }
-                logger.error(`Error modifying file: ${errorMsg}`);
+            } else {
+                // Intermediate edit: content cached, upload deferred to last operation
+                const { filename, modificationInfo, message, isEditByLine, isSearchReplace, replaceAll,
+                        startLine, endLine, currentFile } = editResult;
                 
-                const errorResult = {
-                    success: false,
-                    error: errorMsg
+                logger.info(`EditFile: Cached edit for: ${filename} (upload deferred)`);
+
+                const result = {
+                    success: true,
+                    filename: filename,
+                    fileId: currentFile.id,
+                    // No URL/hash yet - upload pending
+                    pending: true,
+                    ...modificationInfo,
+                    message: `${message} (upload pending - will be saved with next operation)`
                 };
 
                 resolver.tool = JSON.stringify({ toolUsed: toolName });
-                return JSON.stringify(errorResult);
-            });
+                return JSON.stringify(result);
+            }
         } catch (error) {
-            // Handle errors before serialization (file not found, validation errors, etc.)
+            // Handle errors in file edit operation
             let errorMsg;
             if (error?.message) {
                 errorMsg = error.message;
diff --git a/pathways/system/entity/tools/sys_tool_writefile.js b/pathways/system/entity/tools/sys_tool_writefile.js
index d6bb3daa..cb9468c1 100644
--- a/pathways/system/entity/tools/sys_tool_writefile.js
+++ b/pathways/system/entity/tools/sys_tool_writefile.js
@@ -3,6 +3,9 @@
 import logger from '../../../../lib/logger.js';
 import { uploadFileToCloud, addFileToCollection, getMimeTypeFromFilename, isTextMimeType } from '../../../../lib/fileUtils.js';
 
+// Maximum file size for writing (50MB) - prevents memory issues
+const MAX_WRITABLE_FILE_SIZE = 50 * 1024 * 1024;
+
 // Helper function to format file size
 function formatFileSize(bytes) {
     if (bytes === 0) return '0 B';
@@ -123,6 +126,18 @@ export default {
         try {
             // Convert content to buffer
             const fileBuffer = Buffer.from(content, 'utf8');
+            
+            // Check file size to prevent memory issues
+            if (fileBuffer.length > MAX_WRITABLE_FILE_SIZE) {
+                const errorResult = {
+                    success: false,
+                    filename: filename,
+                    error: `Content too large to write (${formatFileSize(fileBuffer.length)}). Maximum file size is ${formatFileSize(MAX_WRITABLE_FILE_SIZE)}.`
+                };
+                resolver.tool = JSON.stringify({ toolUsed: "WriteFile" });
+                return JSON.stringify(errorResult);
+            }
+            
             logger.info(`Prepared content buffer for file: ${filename} (${fileBuffer.length} bytes)`);
 
             // Determine MIME type from filename using utility function

From b3d5a72539aabf55b4588c140ed8841fde5a539d Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Fri, 19 Dec 2025 23:14:03 -0700
Subject: [PATCH 26/27] feat: implement sync and strip functionality for chat
 history file management

- Added `syncAndStripFilesFromChatHistory` function to sync files from chat history to the collection while replacing file content with placeholders.
- Introduced `stripAllFilesFromChatHistory` to remove file and image content from messages, enhancing chat history readability.
- Updated `addFileToCollection` to ensure correct ID usage for existing files, preventing ID mismatches in Redis.
- Enhanced file collection tool to support metadata updates, including renaming, tagging, and notes management, with atomic operations for improved data integrity.
- Added comprehensive tests for the new file metadata update functionality and sync operations, ensuring robust handling of file collections.
---
 lib/fileUtils.js                              | 142 +++++-
 pathways/system/entity/sys_entity_agent.js    |  15 +-
 .../entity/tools/sys_tool_file_collection.js  | 196 ++++++--
 .../features/tools/fileCollection.test.js     | 426 ++++++++++++++++++
 4 files changed, 741 insertions(+), 38 deletions(-)

diff --git a/lib/fileUtils.js b/lib/fileUtils.js
index d8a2bd1d..b3657263 100644
--- a/lib/fileUtils.js
+++ b/lib/fileUtils.js
@@ -1046,10 +1046,15 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
                 // Merge CFH data with Cortex metadata
                 // If file already exists with same hash, update metadata but keep the existing entry
                 // Mark as inCollection: true (chat files that should appear in file collection)
+                
+                // IMPORTANT: Use existing ID if file already exists, to prevent ID mismatch
+                // between what we return and what's actually stored in Redis
+                const actualId = existingData.id || fileEntry.id;
+                
                 const fileData = {
                     ...existingData, // Preserve CFH data (url, gcs, filename, etc.)
-                    // Update Cortex metadata (use new ID if this is a new entry, otherwise keep existing)
-                    id: existingData.id || fileEntry.id,
+                    // Update Cortex metadata (use existing ID if entry exists, otherwise new ID)
+                    id: actualId,
                     url: finalUrl, // Use new URL (guaranteed to be truthy at this point)
                     gcs: finalGcs || existingData.gcs || null, // Use new GCS if provided, otherwise keep existing
                     // Preserve CFH's filename (managed by CFH), store user-provided filename as displayFilename
@@ -1066,6 +1071,13 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
                 
                 // Write back to hash map (atomic operation) - encryption happens in helper
                 await writeFileDataToRedis(redisClient, contextMapKey, storageHash, fileData, contextKey);
+                
+                // Update fileEntry.id to match what's actually stored in Redis
+                // This ensures the caller gets the correct ID for subsequent operations
+                fileEntry.id = actualId;
+                
+                // Invalidate cache to ensure subsequent operations see the updated data
+                invalidateFileCollectionCache(contextId, contextKey);
             }
         } catch (e) {
             // Log but don't fail - metadata update is best effort
@@ -1393,6 +1405,131 @@ async function getAvailableFiles(chatHistory, contextId, contextKey = null) {
     return await getAvailableFilesFromCollection(contextId, contextKey);
 }
 
+/**
+ * Sync files from chat history to collection and strip file content from messages.
+ * Files are synced to the collection where they can be accessed via tools (AnalyzeFile, ReadTextFile, etc.)
+ * File content is replaced with placeholders to avoid sending large files to the model.
+ * @param {Array} chatHistory - Chat history array
+ * @param {string} contextId - Context ID for file collection
+ * @param {string|null} contextKey - Optional encryption key
+ * @returns {Promise<{chatHistory: Array, availableFiles: string}>} Modified chat history and available files string
+ */
+async function syncAndStripFilesFromChatHistory(chatHistory, contextId, contextKey = null) {
+    if (!chatHistory || !Array.isArray(chatHistory)) {
+        return { chatHistory: chatHistory || [], availableFiles: 'No files available.' };
+    }
+
+    if (!contextId) {
+        // No contextId - can't sync to collection, just strip files
+        const strippedHistory = stripAllFilesFromChatHistory(chatHistory);
+        return { chatHistory: strippedHistory, availableFiles: 'No files available.' };
+    }
+
+    // Sync files to collection first
+    await syncFilesToCollection(chatHistory, contextId, contextKey);
+    
+    // Get available files string
+    const availableFiles = await getAvailableFilesFromCollection(contextId, contextKey);
+    
+    // Strip all file content from chat history
+    const strippedHistory = stripAllFilesFromChatHistory(chatHistory);
+    
+    return { chatHistory: strippedHistory, availableFiles };
+}
+
+/**
+ * Strip all file and image content from chat history, replacing with placeholders.
+ * @param {Array} chatHistory - Chat history array
+ * @returns {Array} Chat history with file content replaced by placeholders
+ */
+function stripAllFilesFromChatHistory(chatHistory) {
+    if (!chatHistory || !Array.isArray(chatHistory)) {
+        return chatHistory || [];
+    }
+
+    return chatHistory.map(message => {
+        if (!message || message.role !== 'user' || !message.content) {
+            return message;
+        }
+
+        // Handle array content
+        if (Array.isArray(message.content)) {
+            const newContent = message.content.map(item => {
+                const contentObj = typeof item === 'string' ? tryParseJson(item) : item;
+                if (contentObj && (contentObj.type === 'image_url' || contentObj.type === 'file')) {
+                    // Extract filename for placeholder
+                    const filename = extractFilenameFromFileContent(contentObj);
+                    return { type: 'text', text: `[File: ${filename} - available via file tools]` };
+                }
+                return item;
+            });
+            return { ...message, content: newContent };
+        }
+
+        // Handle object content
+        if (typeof message.content === 'object' && message.content !== null) {
+            if (message.content.type === 'image_url' || message.content.type === 'file') {
+                const filename = extractFilenameFromFileContent(message.content);
+                return { ...message, content: `[File: ${filename} - available via file tools]` };
+            }
+        }
+
+        // Handle string content (might be JSON)
+        if (typeof message.content === 'string') {
+            const contentObj = tryParseJson(message.content);
+            if (contentObj && (contentObj.type === 'image_url' || contentObj.type === 'file')) {
+                const filename = extractFilenameFromFileContent(contentObj);
+                return { ...message, content: `[File: ${filename} - available via file tools]` };
+            }
+        }
+
+        return message;
+    });
+}
+
+/**
+ * Try to parse JSON, return null if it fails
+ */
+function tryParseJson(str) {
+    try {
+        return JSON.parse(str);
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * Extract filename from file content object for placeholder
+ */
+function extractFilenameFromFileContent(content) {
+    if (!content) return 'unknown file';
+    
+    // Try various filename sources
+    if (content.originalFilename) return content.originalFilename;
+    if (content.filename) return content.filename;
+    if (content.name) return content.name;
+    
+    // Try to extract from URL
+    const url = content.url || content.image_url?.url || content.gcs;
+    if (url) {
+        try {
+            const urlPath = new URL(url).pathname;
+            const basename = urlPath.split('/').pop();
+            if (basename && basename.length > 0 && basename !== '/') {
+                // Decode and clean up the filename
+                return decodeURIComponent(basename).replace(/\?.*$/, '');
+            }
+        } catch {
+            // URL parsing failed
+        }
+    }
+    
+    // Fallback based on type
+    if (content.type === 'image_url') return 'image';
+    if (content.type === 'file') return 'file';
+    return 'unknown file';
+}
+
 /**
  * Find a file in the collection by ID, URL, hash, or filename
  * First tries exact matches, then falls back to simple "contains" matches on displayFilename, filename, URL, and GCS
@@ -2188,6 +2325,7 @@ export {
     getAvailableFilesFromCollection,
     formatFilesForTemplate,
     getAvailableFiles,
+    syncAndStripFilesFromChatHistory,
     findFileInCollection,
     // resolveFileParameter is exported inline above
     generateFileMessageContent,
diff --git a/pathways/system/entity/sys_entity_agent.js b/pathways/system/entity/sys_entity_agent.js
index e51649a1..43d78046 100644
--- a/pathways/system/entity/sys_entity_agent.js
+++ b/pathways/system/entity/sys_entity_agent.js
@@ -5,8 +5,7 @@ const MAX_TOOL_CALLS = 50;
 import { callPathway, callTool, say, sendToolStart, sendToolFinish } from '../../../lib/pathwayTools.js';
 import logger from '../../../lib/logger.js';
 import { config } from '../../../config.js';
-import { chatArgsHasImageUrl, removeOldImageAndFileContent } from '../../../lib/util.js';
-import { getAvailableFiles } from '../../../lib/fileUtils.js';
+import { syncAndStripFilesFromChatHistory } from '../../../lib/fileUtils.js';
 import { Prompt } from '../../../server/prompt.js';
 import { getToolsForEntity, loadEntityConfig } from './tools/shared/sys_entity_tools.js';
 import CortexResponse from '../../../lib/cortexResponse.js';
@@ -513,12 +512,12 @@ export default {
             args.chatHistory = args.chatHistory.slice(-20);
         }
 
-        // Get available files from collection (async, syncs files from chat history)
-        const availableFiles = await getAvailableFiles(args.chatHistory, args.contextId, args.contextKey);
-
-        // remove old image and file content
-        const visionContentPresent = chatArgsHasImageUrl(args);
-        visionContentPresent && (args.chatHistory = removeOldImageAndFileContent(args.chatHistory));
+        // Sync files from chat history to collection and strip file content
+        // Files are accessible via tools (AnalyzeFile, ReadTextFile, etc.)
+        const { chatHistory: strippedHistory, availableFiles } = await syncAndStripFilesFromChatHistory(
+            args.chatHistory, args.contextId, args.contextKey
+        );
+        args.chatHistory = strippedHistory;
 
         // truncate the chat history in case there is really long content
         const truncatedChatHistory = resolver.modelExecutor.plugin.truncateMessagesToTargetLength(args.chatHistory, null, 1000);
diff --git a/pathways/system/entity/tools/sys_tool_file_collection.js b/pathways/system/entity/tools/sys_tool_file_collection.js
index a33b5787..42928d2e 100644
--- a/pathways/system/entity/tools/sys_tool_file_collection.js
+++ b/pathways/system/entity/tools/sys_tool_file_collection.js
@@ -1,6 +1,7 @@
 // sys_tool_file_collection.js
-// Tool pathway that manages user file collections (add, search, list files)
+// Tool pathway that manages user file collections (add, search, list, update, remove files)
 // Uses Redis hash maps (FileStoreMap:ctx:<contextId>) for storage
+// Supports atomic rename/tag/notes updates via UpdateFileMetadata
 import logger from '../../../../lib/logger.js';
 import { addFileToCollection, loadFileCollection, findFileInCollection, deleteFileByHash, updateFileMetadata, invalidateFileCollectionCache } from '../../../../lib/fileUtils.js';
 
@@ -144,6 +145,55 @@ export default {
                     required: ["fileIds", "userMessage"]
                 }
             }
+        },
+        {
+            type: "function",
+            icon: "✏️",
+            function: {
+                name: "UpdateFileMetadata",
+                description: "Update metadata for a file in your collection. Use this to rename files, update tags, or add/modify notes. This is an atomic operation - safer than add+delete for renaming.",
+                parameters: {
+                    type: "object",
+                    properties: {
+                        file: {
+                            type: "string",
+                            description: "The file to update - can be the current filename, hash, URL, or ID from ListFileCollection"
+                        },
+                        newFilename: {
+                            type: "string",
+                            description: "Optional: New filename/title for the file (renames the file)"
+                        },
+                        tags: {
+                            type: "array",
+                            items: { type: "string" },
+                            description: "Optional: New tags to set for this file (replaces existing tags)"
+                        },
+                        addTags: {
+                            type: "array",
+                            items: { type: "string" },
+                            description: "Optional: Tags to add to the file's existing tags"
+                        },
+                        removeTags: {
+                            type: "array",
+                            items: { type: "string" },
+                            description: "Optional: Tags to remove from the file's existing tags"
+                        },
+                        notes: {
+                            type: "string",
+                            description: "Optional: New notes/description for the file (replaces existing notes)"
+                        },
+                        permanent: {
+                            type: "boolean",
+                            description: "Optional: If true, marks the file as permanent (won't be auto-cleaned). If false, marks as temporary."
+                        },
+                        userMessage: {
+                            type: "string",
+                            description: "A user-friendly message that describes what you're doing with this tool"
+                        }
+                    },
+                    required: ["file", "userMessage"]
+                }
+            }
         }
     ],
 
@@ -151,7 +201,16 @@ export default {
         const { contextId, contextKey } = args;
 
         // Determine which function was called based on which parameters are present
-        const isAdd = args.fileUrl !== undefined || args.url !== undefined;
+        // Order matters: check most specific operations first
+        const isUpdate = args.file !== undefined && (
+            args.newFilename !== undefined || 
+            args.tags !== undefined || 
+            args.addTags !== undefined || 
+            args.removeTags !== undefined || 
+            args.notes !== undefined || 
+            args.permanent !== undefined
+        );
+        const isAdd = !isUpdate && (args.fileUrl !== undefined || args.url !== undefined);
         const isSearch = args.query !== undefined;
         const isRemove = args.fileIds !== undefined || args.fileId !== undefined;
 
@@ -160,7 +219,101 @@ export default {
                 throw new Error("contextId is required for file collection operations");
             }
 
-            if (isAdd) {
+            if (isUpdate) {
+                // Update file metadata (rename, tags, notes, permanent)
+                const { file, newFilename, tags, addTags, removeTags, notes, permanent } = args;
+                
+                if (!file) {
+                    throw new Error("file parameter is required - specify the file by filename, hash, URL, or ID");
+                }
+
+                // Load collection and find the file
+                const collection = await loadFileCollection(contextId, contextKey, false);
+                const foundFile = findFileInCollection(file, collection);
+                
+                if (!foundFile) {
+                    throw new Error(`File not found: "${file}". Use ListFileCollection to see available files.`);
+                }
+
+                if (!foundFile.hash) {
+                    throw new Error(`File "${file}" has no hash - cannot update metadata`);
+                }
+
+                // Build the metadata update object
+                const metadataUpdate = {};
+                
+                // Handle filename rename
+                if (newFilename !== undefined) {
+                    metadataUpdate.displayFilename = newFilename;
+                }
+                
+                // Handle tags - three modes: replace all, add, or remove
+                if (tags !== undefined) {
+                    // Replace all tags
+                    metadataUpdate.tags = Array.isArray(tags) ? tags : [];
+                } else if (addTags !== undefined || removeTags !== undefined) {
+                    // Merge with existing tags
+                    let currentTags = Array.isArray(foundFile.tags) ? [...foundFile.tags] : [];
+                    
+                    // Add new tags (avoid duplicates)
+                    if (addTags && Array.isArray(addTags)) {
+                        for (const tag of addTags) {
+                            const normalizedTag = tag.toLowerCase();
+                            if (!currentTags.some(t => t.toLowerCase() === normalizedTag)) {
+                                currentTags.push(tag);
+                            }
+                        }
+                    }
+                    
+                    // Remove tags
+                    if (removeTags && Array.isArray(removeTags)) {
+                        const removeSet = new Set(removeTags.map(t => t.toLowerCase()));
+                        currentTags = currentTags.filter(t => !removeSet.has(t.toLowerCase()));
+                    }
+                    
+                    metadataUpdate.tags = currentTags;
+                }
+                
+                // Handle notes
+                if (notes !== undefined) {
+                    metadataUpdate.notes = notes;
+                }
+                
+                // Handle permanent flag
+                if (permanent !== undefined) {
+                    metadataUpdate.permanent = permanent;
+                }
+                
+                // Always update lastAccessed
+                metadataUpdate.lastAccessed = new Date().toISOString();
+
+                // Perform the atomic update
+                const success = await updateFileMetadata(contextId, foundFile.hash, metadataUpdate, contextKey);
+                
+                if (!success) {
+                    throw new Error(`Failed to update file metadata for "${file}"`);
+                }
+
+                // Build result with what was updated
+                const updates = [];
+                if (newFilename !== undefined) updates.push(`renamed to "${newFilename}"`);
+                if (tags !== undefined) updates.push(`tags set to [${tags.join(', ')}]`);
+                if (addTags !== undefined) updates.push(`added tags [${addTags.join(', ')}]`);
+                if (removeTags !== undefined) updates.push(`removed tags [${removeTags.join(', ')}]`);
+                if (notes !== undefined) updates.push(`notes updated`);
+                if (permanent !== undefined) updates.push(`marked as ${permanent ? 'permanent' : 'temporary'}`);
+
+                resolver.tool = JSON.stringify({ toolUsed: "UpdateFileMetadata" });
+                return JSON.stringify({
+                    success: true,
+                    file: foundFile.displayFilename || foundFile.filename || file,
+                    fileId: foundFile.id,
+                    hash: foundFile.hash,
+                    updates: updates,
+                    message: `File "${foundFile.displayFilename || foundFile.filename || file}" updated: ${updates.join(', ')}`
+                });
+
+            } else if (isAdd) {
                 // Add file to collection
                 const { fileUrl, url, gcs, filename, tags = [], notes = '', hash = null, permanent = false } = args;
                 
@@ -306,27 +459,27 @@ export default {
                     throw new Error("fileIds array is required and must not be empty");
                 }
 
-                let removedCount = 0;
-                let removedFiles = [];
                 let notFoundFiles = [];
                 let filesToRemove = [];
 
-                // Load collection once to find all files
+                // Load collection ONCE to find all files and their hashes
+                // Use useCache: false to get fresh data
                 const collection = await loadFileCollection(contextId, contextKey, false);
                 
-                // Resolve all files
+                // Resolve all files and collect their info in a single pass
                 for (const target of targetFiles) {
                     if (target === '*') continue; // Skip wildcard if passed
                     
                     const foundFile = findFileInCollection(target, collection);
                     
                     if (foundFile) {
-                        // Avoid duplicates
-                        if (!filesToRemove.some(f => f.id === foundFile.id)) {
+                        // Avoid duplicates (by hash since that's the unique key in Redis)
+                        if (!filesToRemove.some(f => f.hash === foundFile.hash)) {
                             filesToRemove.push({
                                 id: foundFile.id,
                                 displayFilename: foundFile.displayFilename || foundFile.filename || null,
-                                hash: foundFile.hash || null
+                                hash: foundFile.hash || null,
+                                permanent: foundFile.permanent ?? false
                             });
                         }
                     } else {
@@ -338,22 +491,9 @@ export default {
                     throw new Error(`No files found matching: ${notFoundFiles.join(', ')}`);
                 }
 
-                // Remove files directly from hash map (atomic operations)
-                // Load collection to get hashes, then delete entries directly
-                const allFiles = await loadFileCollection(contextId, contextKey, false);
-                const fileIdsToRemove = new Set(filesToRemove.map(f => f.id));
-                const hashesToDelete = [];
-                
-                // Collect hashes to delete (hash is always present - either actual hash or generated from URL)
-                allFiles.forEach(file => {
-                    if (fileIdsToRemove.has(file.id) && file.hash) {
-                        hashesToDelete.push({
-                            hash: file.hash,
-                            displayFilename: file.displayFilename || file.filename || 'unknown',
-                            permanent: file.permanent ?? false
-                        });
-                    }
-                });
+                // Use the hashes collected from the single collection load
+                // No need to reload - we already have all the info we need
+                const hashesToDelete = filesToRemove.filter(f => f.hash);
                 
                 // Delete entries directly from hash map (atomic operations)
                 const { getRedisClient } = await import('../../../../lib/fileUtils.js');
@@ -390,8 +530,8 @@ export default {
                     }
                 })().catch(err => logger.error(`Async cloud deletion error: ${err}`));
 
-                removedCount = filesToRemove.length;
-                removedFiles = filesToRemove;
+                const removedCount = filesToRemove.length;
+                const removedFiles = filesToRemove;
 
                 // Get remaining files count after deletion
                 const remainingCollection = await loadFileCollection(contextId, contextKey, false);
diff --git a/tests/integration/features/tools/fileCollection.test.js b/tests/integration/features/tools/fileCollection.test.js
index cacd81c0..fe2bef03 100644
--- a/tests/integration/features/tools/fileCollection.test.js
+++ b/tests/integration/features/tools/fileCollection.test.js
@@ -1158,6 +1158,432 @@ test('File collection: Sync files from chat history', async t => {
     }
 });
 
+// ============================================
+// UpdateFileMetadata Tool Tests
+// ============================================
+
+test('File collection: UpdateFileMetadata tool - Rename file', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add a file first
+        const addResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/old-name.pdf',
+            filename: 'old-name.pdf',
+            tags: ['test'],
+            userMessage: 'Add file'
+        });
+        
+        const addParsed = JSON.parse(addResult);
+        t.is(addParsed.success, true);
+        const originalFileId = addParsed.fileId;
+        
+        // Rename using UpdateFileMetadata tool
+        const updateResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            file: 'old-name.pdf',
+            newFilename: 'new-name.pdf',
+            userMessage: 'Rename file'
+        });
+        
+        const updateParsed = JSON.parse(updateResult);
+        t.is(updateParsed.success, true);
+        t.is(updateParsed.file, 'old-name.pdf');
+        t.true(updateParsed.message.includes('renamed to "new-name.pdf"'));
+        
+        // Verify rename persisted
+        const collection = await loadFileCollection(contextId, null, false);
+        const updatedFile = collection.find(f => f.id === originalFileId);
+        t.truthy(updatedFile);
+        t.is(updatedFile.displayFilename, 'new-name.pdf');
+        t.is(updatedFile.id, originalFileId); // ID should be preserved
+        t.is(updatedFile.url, 'https://example.com/old-name.pdf'); // URL should be preserved
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: UpdateFileMetadata tool - Replace all tags', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add file with initial tags
+        const addResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/test.pdf',
+            filename: 'test.pdf',
+            tags: ['old', 'tags'],
+            userMessage: 'Add file'
+        });
+        
+        const addParsed = JSON.parse(addResult);
+        t.is(addParsed.success, true);
+        
+        // Replace all tags
+        const updateResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            file: 'test.pdf',
+            tags: ['new', 'replaced', 'tags'],
+            userMessage: 'Replace tags'
+        });
+        
+        const updateParsed = JSON.parse(updateResult);
+        t.is(updateParsed.success, true);
+        
+        // Verify tags were replaced
+        const collection = await loadFileCollection(contextId, null, false);
+        const file = collection.find(f => f.id === addParsed.fileId);
+        t.deepEqual(file.tags, ['new', 'replaced', 'tags']);
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: UpdateFileMetadata tool - Add tags', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add file with initial tags
+        const addResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/test.pdf',
+            filename: 'test.pdf',
+            tags: ['existing', 'tag'],
+            userMessage: 'Add file'
+        });
+        
+        const addParsed = JSON.parse(addResult);
+        t.is(addParsed.success, true);
+        
+        // Add more tags
+        const updateResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            file: 'test.pdf',
+            addTags: ['new', 'added'],
+            userMessage: 'Add tags'
+        });
+        
+        const updateParsed = JSON.parse(updateResult);
+        t.is(updateParsed.success, true);
+        
+        // Verify tags were added (should contain both old and new)
+        const collection = await loadFileCollection(contextId, null, false);
+        const file = collection.find(f => f.id === addParsed.fileId);
+        t.is(file.tags.length, 4);
+        t.true(file.tags.includes('existing'));
+        t.true(file.tags.includes('tag'));
+        t.true(file.tags.includes('new'));
+        t.true(file.tags.includes('added'));
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: UpdateFileMetadata tool - Remove tags', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add file with tags
+        const addResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/test.pdf',
+            filename: 'test.pdf',
+            tags: ['keep', 'remove1', 'remove2', 'also-keep'],
+            userMessage: 'Add file'
+        });
+        
+        const addParsed = JSON.parse(addResult);
+        t.is(addParsed.success, true);
+        
+        // Remove specific tags
+        const updateResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            file: 'test.pdf',
+            removeTags: ['remove1', 'remove2'],
+            userMessage: 'Remove tags'
+        });
+        
+        const updateParsed = JSON.parse(updateResult);
+        t.is(updateParsed.success, true);
+        
+        // Verify tags were removed
+        const collection = await loadFileCollection(contextId, null, false);
+        const file = collection.find(f => f.id === addParsed.fileId);
+        t.is(file.tags.length, 2);
+        t.true(file.tags.includes('keep'));
+        t.true(file.tags.includes('also-keep'));
+        t.false(file.tags.includes('remove1'));
+        t.false(file.tags.includes('remove2'));
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: UpdateFileMetadata tool - Add and remove tags together', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add file with tags
+        const addResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/test.pdf',
+            filename: 'test.pdf',
+            tags: ['old1', 'old2', 'remove-me'],
+            userMessage: 'Add file'
+        });
+        
+        const addParsed = JSON.parse(addResult);
+        t.is(addParsed.success, true);
+        
+        // Add and remove tags in one operation
+        const updateResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            file: 'test.pdf',
+            addTags: ['new1', 'new2'],
+            removeTags: ['remove-me'],
+            userMessage: 'Update tags'
+        });
+        
+        const updateParsed = JSON.parse(updateResult);
+        t.is(updateParsed.success, true);
+        
+        // Verify tags were updated correctly
+        const collection = await loadFileCollection(contextId, null, false);
+        const file = collection.find(f => f.id === addParsed.fileId);
+        t.is(file.tags.length, 4);
+        t.true(file.tags.includes('old1'));
+        t.true(file.tags.includes('old2'));
+        t.true(file.tags.includes('new1'));
+        t.true(file.tags.includes('new2'));
+        t.false(file.tags.includes('remove-me'));
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: UpdateFileMetadata tool - Update notes', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add file with initial notes
+        const addResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/test.pdf',
+            filename: 'test.pdf',
+            notes: 'Initial notes',
+            userMessage: 'Add file'
+        });
+        
+        const addParsed = JSON.parse(addResult);
+        t.is(addParsed.success, true);
+        
+        // Update notes
+        const updateResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            file: 'test.pdf',
+            notes: 'Updated notes with more detail',
+            userMessage: 'Update notes'
+        });
+        
+        const updateParsed = JSON.parse(updateResult);
+        t.is(updateParsed.success, true);
+        
+        // Verify notes were updated
+        const collection = await loadFileCollection(contextId, null, false);
+        const file = collection.find(f => f.id === addParsed.fileId);
+        t.is(file.notes, 'Updated notes with more detail');
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: UpdateFileMetadata tool - Update permanent flag', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add file (defaults to temporary)
+        const addResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/test.pdf',
+            filename: 'test.pdf',
+            userMessage: 'Add file'
+        });
+        
+        const addParsed = JSON.parse(addResult);
+        t.is(addParsed.success, true);
+        
+        // Mark as permanent
+        const updateResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            file: 'test.pdf',
+            permanent: true,
+            userMessage: 'Mark as permanent'
+        });
+        
+        const updateParsed = JSON.parse(updateResult);
+        t.is(updateParsed.success, true);
+        
+        // Verify permanent flag was set
+        const collection = await loadFileCollection(contextId, null, false);
+        const file = collection.find(f => f.id === addParsed.fileId);
+        t.is(file.permanent, true);
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: UpdateFileMetadata tool - Combined updates', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add file
+        const addResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/original.pdf',
+            filename: 'original.pdf',
+            tags: ['old'],
+            notes: 'Old notes',
+            userMessage: 'Add file'
+        });
+        
+        const addParsed = JSON.parse(addResult);
+        t.is(addParsed.success, true);
+        const originalFileId = addParsed.fileId;
+        
+        // Update everything at once
+        const updateResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            file: 'original.pdf',
+            newFilename: 'renamed-and-tagged.pdf',
+            tags: ['new', 'tags'],
+            notes: 'New notes',
+            permanent: true,
+            userMessage: 'Full update'
+        });
+        
+        const updateParsed = JSON.parse(updateResult);
+        t.is(updateParsed.success, true);
+        t.true(updateParsed.message.includes('renamed'));
+        t.true(updateParsed.message.includes('tags set'));
+        t.true(updateParsed.message.includes('notes updated'));
+        t.true(updateParsed.message.includes('permanent'));
+        
+        // Verify all updates persisted
+        const collection = await loadFileCollection(contextId, null, false);
+        const file = collection.find(f => f.id === originalFileId);
+        t.is(file.displayFilename, 'renamed-and-tagged.pdf');
+        t.deepEqual(file.tags, ['new', 'tags']);
+        t.is(file.notes, 'New notes');
+        t.is(file.permanent, true);
+        t.is(file.id, originalFileId); // ID preserved
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: UpdateFileMetadata tool - File not found error', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Try to update a non-existent file
+        const updateResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            file: 'nonexistent.pdf',
+            newFilename: 'new-name.pdf',
+            userMessage: 'Update missing file'
+        });
+        
+        const updateParsed = JSON.parse(updateResult);
+        t.is(updateParsed.success, false);
+        t.true(updateParsed.error.includes('not found'));
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: UpdateFileMetadata tool - Find file by ID', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add file
+        const addResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/test.pdf',
+            filename: 'test.pdf',
+            userMessage: 'Add file'
+        });
+        
+        const addParsed = JSON.parse(addResult);
+        t.is(addParsed.success, true);
+        const fileId = addParsed.fileId;
+        
+        // Update using file ID instead of filename
+        const updateResult = await callPathway('sys_tool_file_collection', {
+            contextId,
+            file: fileId,
+            newFilename: 'renamed-by-id.pdf',
+            userMessage: 'Update by ID'
+        });
+        
+        const updateParsed = JSON.parse(updateResult);
+        t.is(updateParsed.success, true);
+        
+        // Verify update worked
+        const collection = await loadFileCollection(contextId, null, false);
+        const file = collection.find(f => f.id === fileId);
+        t.is(file.displayFilename, 'renamed-by-id.pdf');
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('File collection: addFileToCollection returns correct ID for existing files', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add file first time
+        const addResult1 = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/duplicate.pdf',
+            filename: 'first.pdf',
+            tags: ['first'],
+            userMessage: 'Add file first time'
+        });
+        
+        const addParsed1 = JSON.parse(addResult1);
+        t.is(addParsed1.success, true);
+        const firstFileId = addParsed1.fileId;
+        
+        // Add same file again (same URL = same hash)
+        const addResult2 = await callPathway('sys_tool_file_collection', {
+            contextId,
+            url: 'https://example.com/duplicate.pdf',
+            filename: 'second.pdf',
+            tags: ['second'],
+            userMessage: 'Add same file again'
+        });
+        
+        const addParsed2 = JSON.parse(addResult2);
+        t.is(addParsed2.success, true);
+        
+        // The returned ID should match the first one (same hash = same entry)
+        t.is(addParsed2.fileId, firstFileId, 'Second add should return same ID as first');
+        
+        // Verify only one entry exists (not duplicated)
+        const collection = await loadFileCollection(contextId, null, false);
+        t.is(collection.length, 1);
+        
+        // Verify metadata was merged (tags from second add, but same ID)
+        const file = collection[0];
+        t.is(file.id, firstFileId);
+        t.deepEqual(file.tags, ['second']); // New tags replaced old ones
+        t.is(file.displayFilename, 'second.pdf'); // New filename
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
 // ============================================
 // File Collection Encryption Tests
 // ============================================

From 11262611857068453e6944ebaf10929c11715d18 Mon Sep 17 00:00:00 2001
From: Jason McCartney <jmac@theroot.org>
Date: Fri, 19 Dec 2025 23:28:01 -0700
Subject: [PATCH 27/27] feat: add sys_tool_slides_gemini for generating
 presentation visuals

- Introduced a new tool for creating slides, infographics, and presentations using Gemini 3 Pro image generation.
- Implemented input parameters for detailed instructions, filename prefix, and tagging for generated content.
- Added functionality to resolve input images, upload generated visuals to cloud storage, and manage file collections.
- Enhanced error handling and logging for improved user feedback during image generation and upload processes.
---
 .../entity/tools/sys_tool_slides_gemini.js    | 184 ++++++++++++++++++
 1 file changed, 184 insertions(+)
 create mode 100644 pathways/system/entity/tools/sys_tool_slides_gemini.js

diff --git a/pathways/system/entity/tools/sys_tool_slides_gemini.js b/pathways/system/entity/tools/sys_tool_slides_gemini.js
new file mode 100644
index 00000000..dabd03e8
--- /dev/null
+++ b/pathways/system/entity/tools/sys_tool_slides_gemini.js
@@ -0,0 +1,184 @@
+// sys_tool_slides_gemini.js
+// Entity tool that creates slides, infographics, and presentations using Gemini 3 Pro image generation
+import { callPathway } from '../../../../lib/pathwayTools.js';
+import { uploadImageToCloud, addFileToCollection, resolveFileParameter } from '../../../../lib/fileUtils.js';
+
+export default {
+    prompt: [],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    inputParameters: {
+        model: 'oai-gpt4o',
+        contextId: '',
+        contextKey: '',
+    },
+    timeout: 300,
+    toolDefinition: [{
+        type: "function",
+        enabled: true,
+        icon: "📊",
+        function: {
+            name: "GenerateSlides",
+            description: "Use when asked to create, generate, or design slides, infographics, presentations, or visual content optimized for presentations. This tool is specifically designed for creating presentation-ready visuals including slide layouts, infographic designs, charts, diagrams, and other visual content that would be used in presentations. It uses Gemini 3 Pro image generation which excels at creating structured, professional presentation content. After you have generated the content, you must include it in your response to show it to the user.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "A very detailed prompt describing the slide, infographic, or presentation content you want to create. Be specific about the layout, design style, content structure, color scheme, typography preferences, and any specific elements you want included (e.g., 'Create a professional slide with a title at the top, three bullet points in the middle, and a chart on the right side. Use a blue and white color scheme with modern sans-serif fonts.'). For infographics, specify the data visualization needs, layout structure, and visual hierarchy. The more detailed and descriptive the prompt, the better the result."
+                    },
+                    filenamePrefix: {
+                        type: "string",
+                        description: "Optional: A descriptive prefix to use for the generated image filename (e.g., 'slide', 'infographic', 'presentation', 'chart'). If not provided, defaults to 'presentation-slide'."
+                    },
+                    tags: {
+                        type: "array",
+                        items: {
+                            type: "string"
+                        },
+                        description: "Optional: Array of tags to categorize the content (e.g., ['slide', 'infographic', 'presentation', 'chart']). Will be merged with default tags ['presentation', 'generated']."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "userMessage"]
+            }
+        }
+    }],
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const pathwayResolver = resolver;
+
+        try {   
+            let model = "gemini-pro-3-image";
+            let prompt = args.detailedInstructions || "";
+            
+            // Resolve input images to URLs using the common utility
+            // For Gemini, prefer GCS URLs over Azure URLs
+            // Fail early if any provided image cannot be resolved
+            const resolvedInputImages = [];
+            if (args.inputImages && Array.isArray(args.inputImages)) {
+                if (!args.contextId) {
+                    throw new Error("contextId is required when using the 'inputImages' parameter. Use ListFileCollection or SearchFileCollection to find available files.");
+                }
+                
+                // Limit to 3 images maximum
+                const imagesToProcess = args.inputImages.slice(0, 3);
+                
+                for (let i = 0; i < imagesToProcess.length; i++) {
+                    const imageRef = imagesToProcess[i];
+                    const resolved = await resolveFileParameter(imageRef, args.contextId, args.contextKey, { preferGcs: true });
+                    if (!resolved) {
+                        throw new Error(`File not found: "${imageRef}". Use ListFileCollection or SearchFileCollection to find available files.`);
+                    }
+                    resolvedInputImages.push(resolved);
+                }
+            }
+            
+            // Call the image generation pathway using Gemini 3
+            let result = await callPathway('image_gemini_3', {
+                ...args, 
+                text: prompt,
+                model, 
+                stream: false,
+                input_image: resolvedInputImages.length > 0 ? resolvedInputImages[0] : undefined,
+                input_image_2: resolvedInputImages.length > 1 ? resolvedInputImages[1] : undefined,
+                input_image_3: resolvedInputImages.length > 2 ? resolvedInputImages[2] : undefined,
+                optimizePrompt: true,
+            }, pathwayResolver);
+
+            pathwayResolver.tool = JSON.stringify({ toolUsed: "slides" });
+
+            if (pathwayResolver.pathwayResultData) {
+                if (pathwayResolver.pathwayResultData.artifacts && Array.isArray(pathwayResolver.pathwayResultData.artifacts)) {
+                    const uploadedImages = [];
+                    
+                    // Process each image artifact
+                    for (const artifact of pathwayResolver.pathwayResultData.artifacts) {
+                        if (artifact.type === 'image' && artifact.data && artifact.mimeType) {
+                            try {
+                                // Upload image to cloud storage (returns {url, gcs, hash})
+                                const uploadResult = await uploadImageToCloud(artifact.data, artifact.mimeType, pathwayResolver, args.contextId);
+                                
+                                const imageUrl = uploadResult.url || uploadResult;
+                                const imageGcs = uploadResult.gcs || null;
+                                const imageHash = uploadResult.hash || null;
+                                
+                                uploadedImages.push({
+                                    type: 'image',
+                                    url: imageUrl,
+                                    gcs: imageGcs,
+                                    hash: imageHash,
+                                    mimeType: artifact.mimeType
+                                });
+                                
+                                // Add uploaded image to file collection if contextId is available
+                                if (args.contextId && imageUrl) {
+                                    try {
+                                        // Generate filename from mimeType (e.g., "image/png" -> "png")
+                                        const extension = artifact.mimeType.split('/')[1] || 'png';
+                                        // Use hash for uniqueness if available, otherwise use timestamp and index
+                                        const uniqueId = imageHash ? imageHash.substring(0, 8) : `${Date.now()}-${uploadedImages.length}`;
+                                        
+                                        // Determine filename prefix
+                                        const defaultPrefix = 'presentation-slide';
+                                        const filenamePrefix = args.filenamePrefix || defaultPrefix;
+                                        
+                                        // Sanitize the prefix to ensure it's a valid filename component
+                                        const sanitizedPrefix = filenamePrefix.replace(/[^a-zA-Z0-9_-]/g, '-').toLowerCase();
+                                        const filename = `${sanitizedPrefix}-${uniqueId}.${extension}`;
+                                        
+                                        // Merge provided tags with default tags
+                                        const defaultTags = ['presentation', 'generated'];
+                                        const providedTags = Array.isArray(args.tags) ? args.tags : [];
+                                        const allTags = [...defaultTags, ...providedTags.filter(tag => !defaultTags.includes(tag))];
+                                        
+                                        // Use the centralized utility function to add to collection
+                                        await addFileToCollection(
+                                            args.contextId,
+                                            args.contextKey || '',
+                                            imageUrl,
+                                            imageGcs,
+                                            filename,
+                                            allTags,
+                                            `Generated presentation content from prompt: ${args.detailedInstructions || 'presentation generation'}`,
+                                            imageHash,
+                                            null,
+                                            pathwayResolver,
+                                            true // permanent => retention=permanent
+                                        );
+                                    } catch (collectionError) {
+                                        // Log but don't fail - file collection is optional
+                                        pathwayResolver.logWarning(`Failed to add image to file collection: ${collectionError.message}`);
+                                    }
+                                }
+                            } catch (uploadError) {
+                                pathwayResolver.logError(`Failed to upload artifact: ${uploadError.message}`);
+                                // Keep original artifact as fallback
+                                uploadedImages.push(artifact);
+                            }
+                        } else {
+                            // Keep non-image artifacts as-is
+                            uploadedImages.push(artifact);
+                        }
+                    }
+                    
+                    // Return the urls of the uploaded images as text in the result
+                    result = result ? result + '\n' + uploadedImages.map(image => image.url || image).join('\n') : uploadedImages.map(image => image.url || image).join('\n');
+                }
+            } else {
+                // If result is not a CortexResponse, log a warning but return as-is
+                pathwayResolver.logWarning('No artifacts to upload');
+                result = result + '\n' + 'No presentation content generated';
+            }
+
+            return result;
+
+        } catch (e) {
+            pathwayResolver.logError(e.message ?? e);
+            return await callPathway('sys_generator_error', { ...args, text: e.message }, pathwayResolver);
+        }
+    }
+};
+