From bd775cd817f9307146be3b6d037fac74b84ecdf3 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Fri, 3 Oct 2025 13:29:25 -0700 Subject: [PATCH] feat: add optional vacuum on upload --- bridge/cgo_bridge.go | 36 +++++++++++++++++ client/main.go | 4 +- client/sync/coordinator.go | 79 +++++++++++++++++++++++++++++++++++++- 3 files changed, 116 insertions(+), 3 deletions(-) diff --git a/bridge/cgo_bridge.go b/bridge/cgo_bridge.go index c8d9782..b1af425 100644 --- a/bridge/cgo_bridge.go +++ b/bridge/cgo_bridge.go @@ -19,6 +19,26 @@ package bridge // Forward declarations for Go callbacks extern int go_local_read_callback(void* user_data, uint8_t* buffer, int size); extern int go_local_write_callback(void* user_data, uint8_t* buffer, int size); + +// Helper function to execute VACUUM on a database +static int execute_vacuum(const char* db_path) { + sqlite3 *db; + char *err_msg = NULL; + int rc; + + rc = sqlite3_open(db_path, &db); + if (rc != SQLITE_OK) { + return rc; + } + + rc = sqlite3_exec(db, "VACUUM", NULL, NULL, &err_msg); + if (err_msg) { + sqlite3_free(err_msg); + } + + sqlite3_close(db); + return rc; +} */ import "C" import ( @@ -330,3 +350,19 @@ func cgoRunDirectSync(originDbPath, replicaDbPath string, dryRun bool, verbose i return nil } + +// ExecuteVacuum runs VACUUM on the specified database +func ExecuteVacuum(dbPath string) error { + cDbPath := C.CString(dbPath) + defer C.free(unsafe.Pointer(cDbPath)) + + result := C.execute_vacuum(cDbPath) + if result != 0 { + return &SQLiteRsyncError{ + Code: int(result), + Message: "VACUUM failed", + } + } + + return nil +} diff --git a/client/main.go b/client/main.go index 2183898..24a8b9a 100644 --- a/client/main.go +++ b/client/main.go @@ -29,6 +29,7 @@ var ( replicaID string logger *zap.Logger showVersion bool + vacuum bool ) var MAX_MESSAGE_SIZE = 4096 @@ -162,6 +163,7 @@ func runSync(cmd *cobra.Command, args []string) error { DryRun: dryRun, Logger: logger, Verbose: verbose, + Vacuum: vacuum, }) // Execute the operation @@ -276,7 +278,7 @@ func init() { rootCmd.Flags().BoolVar(&SetPublic, "public", false, "Enable public access to the replica (initial PUSH only)") rootCmd.Flags().BoolVar(&dryRun, "dry", false, "Perform a dry run without making changes") rootCmd.Flags().BoolVarP(&showVersion, "version", "v", false, "Show version information") - + rootCmd.Flags().BoolVar(&vacuum, "vacuum", false, "VACUUM database into temp file before PUSH (optimizes and compacts)") } func main() { diff --git a/client/sync/coordinator.go b/client/sync/coordinator.go index 18aa1df..c2a524f 100644 --- a/client/sync/coordinator.go +++ b/client/sync/coordinator.go @@ -46,6 +46,7 @@ type CoordinatorConfig struct { DryRun bool Logger *zap.Logger Verbose bool + Vacuum bool } // Coordinator manages sync operations and subscriptions @@ -439,6 +440,22 @@ func (c *Coordinator) executePush() error { return fmt.Errorf("database file does not exist: %s", c.config.LocalPath) } + // Handle --vacuum flag: create vacuumed temp file and use it for push + var actualPushPath string + var cleanupVacuum func() + if c.config.Vacuum { + fmt.Println("🗜️ Creating vacuumed copy for PUSH...") + tempPath, cleanup, err := c.vacuumDatabase(c.config.LocalPath) + if err != nil { + return fmt.Errorf("vacuum failed: %w", err) + } + actualPushPath = tempPath + cleanupVacuum = cleanup + defer cleanupVacuum() + } else { + actualPushPath = c.config.LocalPath + } + // Resolve authentication authResult, err := c.resolveAuth("push") if err != nil { @@ -457,9 +474,9 @@ func (c *Coordinator) executePush() error { remotePath = c.config.RemotePath } - // Create local client for SQLite operations + // Create local client for SQLite operations (using actualPushPath which may be vacuumed) localClient, err := bridge.New(&bridge.BridgeConfig{ - DatabasePath: c.config.LocalPath, + DatabasePath: actualPushPath, DryRun: c.config.DryRun, Logger: c.logger.Named("local"), EnableSQLiteRsyncLogging: c.config.Verbose, @@ -653,6 +670,64 @@ func (c *Coordinator) executeLocalSync() error { return nil } +// vacuumDatabase creates a vacuumed copy of the database in a temp file +func (c *Coordinator) vacuumDatabase(sourcePath string) (string, func(), error) { + // Get source database size + sourceInfo, err := os.Stat(sourcePath) + if err != nil { + return "", nil, fmt.Errorf("failed to stat source database: %w", err) + } + sourceSize := sourceInfo.Size() + + // Check available disk space in /tmp + var stat syscall.Statfs_t + if err := syscall.Statfs("/tmp", &stat); err != nil { + return "", nil, fmt.Errorf("failed to check disk space: %w", err) + } + availableSpace := int64(stat.Bavail * uint64(stat.Bsize)) + + // Ensure at least 16KB will remain after copy + requiredSpace := sourceSize + 16384 + if availableSpace < requiredSpace { + return "", nil, fmt.Errorf("insufficient disk space: need %d bytes, only %d available", requiredSpace, availableSpace) + } + + // Create temp file with pattern /tmp/sqlrsync-temp-* + tempFile, err := os.CreateTemp("", "sqlrsync-temp-*.sqlite") + if err != nil { + return "", nil, fmt.Errorf("failed to create temp file: %w", err) + } + tempPath := tempFile.Name() + tempFile.Close() + + // Copy source database to temp location + sourceData, err := os.ReadFile(sourcePath) + if err != nil { + os.Remove(tempPath) + return "", nil, fmt.Errorf("failed to read source database: %w", err) + } + + if err := os.WriteFile(tempPath, sourceData, 0644); err != nil { + os.Remove(tempPath) + return "", nil, fmt.Errorf("failed to write temp database: %w", err) + } + + // Execute VACUUM on the temp file using bridge + if err := bridge.ExecuteVacuum(tempPath); err != nil { + os.Remove(tempPath) + return "", nil, fmt.Errorf("VACUUM execution failed: %w", err) + } + + // Return temp path and cleanup function + cleanup := func() { + if err := os.Remove(tempPath); err != nil { + c.logger.Warn("Failed to remove temp vacuum file", zap.String("path", tempPath), zap.Error(err)) + } + } + + return tempPath, cleanup, nil +} + // Close cleanly shuts down the coordinator func (c *Coordinator) Close() error { c.cancel()