From d7bc5984aae89cd3b19290fe01e46357ce377801 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Sat, 4 Oct 2025 14:39:25 -0700 Subject: [PATCH 01/20] fix: rename token to key fix: create local secrets with o+rw feat: after failed anon access, prompt for key fix: if we should prompt OR it's a push, prompt --- .gitignore | 1 + client/auth/config.go | 7 +---- client/auth/resolver.go | 41 +++++++++++++++-------------- client/main.go | 8 +++--- client/remote/client.go | 14 +++++----- client/subscription/manager.go | 41 +++++++++++++++++++++++++++-- client/sync/coordinator.go | 48 ++++++++++++++++++++-------------- 7 files changed, 102 insertions(+), 58 deletions(-) diff --git a/.gitignore b/.gitignore index 248f074..91a2606 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ tmp/ client/sqlrsync client/sqlrsync client/sqlrsync_simple +asciinema/ diff --git a/client/auth/config.go b/client/auth/config.go index bea5925..b526b2a 100644 --- a/client/auth/config.go +++ b/client/auth/config.go @@ -166,17 +166,12 @@ func SaveLocalSecretsConfig(config *LocalSecretsConfig) error { return fmt.Errorf("failed to create directory %s: %w", dir, err) } - file, err := os.Create(path) + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return fmt.Errorf("failed to create local-secrets config file %s: %w", path, err) } defer file.Close() - // Set file permissions to 0600 (read/write for owner only) - if err := file.Chmod(0600); err != nil { - return fmt.Errorf("failed to set permissions on local-secrets config file: %w", err) - } - encoder := toml.NewEncoder(file) if err := encoder.Encode(config); err != nil { return fmt.Errorf("failed to write local-secrets config: %w", err) diff --git a/client/auth/resolver.go b/client/auth/resolver.go index 96bf8f0..58724fc 100644 --- a/client/auth/resolver.go +++ b/client/auth/resolver.go @@ -12,7 +12,7 @@ import ( // ResolveResult contains the resolved authentication information type ResolveResult struct { - AccessToken string + AccessKey string ReplicaID string ServerURL string RemotePath string @@ -53,9 +53,9 @@ func (r *Resolver) Resolve(req *ResolveRequest) (*ResolveResult, error) { } // 1. Try environment variable first - if token := os.Getenv("SQLRSYNC_AUTH_TOKEN"); token != "" { - r.logger.Debug("Using SQLRSYNC_AUTH_TOKEN from environment") - result.AccessToken = token + if key := os.Getenv("SQLRSYNC_AUTH_KEY"); key != "" { + r.logger.Debug("Using SQLRSYNC_AUTH_KEY from environment") + result.AccessKey = key result.ReplicaID = req.ProvidedReplicaID return result, nil } @@ -63,14 +63,14 @@ func (r *Resolver) Resolve(req *ResolveRequest) (*ResolveResult, error) { // 2. Try explicitly provided keys if req.ProvidedPullKey != "" { r.logger.Debug("Using provided pull key") - result.AccessToken = req.ProvidedPullKey + result.AccessKey = req.ProvidedPullKey result.ReplicaID = req.ProvidedReplicaID return result, nil } if req.ProvidedPushKey != "" { r.logger.Debug("Using provided push key") - result.AccessToken = req.ProvidedPushKey + result.AccessKey = req.ProvidedPushKey result.ReplicaID = req.ProvidedReplicaID return result, nil } @@ -114,7 +114,7 @@ func (r *Resolver) Resolve(req *ResolveRequest) (*ResolveResult, error) { if req.Operation == "push" { if os.Getenv("SQLRSYNC_ADMIN_KEY") != "" { r.logger.Debug("Using SQLRSYNC_ADMIN_KEY from environment") - result.AccessToken = os.Getenv("SQLRSYNC_ADMIN_KEY") + result.AccessKey = os.Getenv("SQLRSYNC_ADMIN_KEY") result.ShouldPrompt = false return result, nil } @@ -126,7 +126,7 @@ func (r *Resolver) Resolve(req *ResolveRequest) (*ResolveResult, error) { // 5. If it's a pull, maybe no key needed if req.Operation == "pull" || req.Operation == "subscribe" { - result.AccessToken = "" + result.AccessKey = "" result.ShouldPrompt = false return result, nil } @@ -169,7 +169,7 @@ func (r *Resolver) resolveFromLocalSecrets(absLocalPath, serverURL string, resul } r.logger.Debug("Found authentication in local secrets config") - result.AccessToken = dbConfig.PushKey + result.AccessKey = dbConfig.PushKey result.ReplicaID = dbConfig.ReplicaID result.RemotePath = dbConfig.RemotePath result.ServerURL = dbConfig.Server @@ -193,7 +193,7 @@ func (r *Resolver) resolveFromDashFile(localPath string, result *ResolveResult) } r.logger.Debug("Found authentication in -sqlrsync file") - result.AccessToken = dashSQLRsync.PullKey + result.AccessKey = dashSQLRsync.PullKey result.ReplicaID = dashSQLRsync.ReplicaID result.RemotePath = dashSQLRsync.RemotePath result.ServerURL = dashSQLRsync.Server @@ -201,24 +201,25 @@ func (r *Resolver) resolveFromDashFile(localPath string, result *ResolveResult) return result, nil } -// PromptForAdminKey prompts the user for an admin key -func (r *Resolver) PromptForAdminKey(serverURL string) (string, error) { +// PromptForKey prompts the user for an key +func (r *Resolver) PromptForKey(serverURL string, remotePath string, keyType string) (string, error) { httpServer := strings.Replace(serverURL, "ws", "http", 1) - fmt.Println("No Key provided. Creating a new Replica? Get a key at " + httpServer + "/namespaces") - fmt.Print(" Enter an Account Admin Key to create a new Replica: ") + fmt.Println("Replica not found when using unauthenticated access. Try again using a key or check your spelling.") + fmt.Println(" Get a key at " + httpServer + "/namespaces or " + httpServer + "/" + remotePath) + fmt.Print(" Provide a key to " + keyType + ": ") reader := bufio.NewReader(os.Stdin) - token, err := reader.ReadString('\n') + key, err := reader.ReadString('\n') if err != nil { - return "", fmt.Errorf("failed to read admin key: %w", err) + return "", fmt.Errorf("failed to read key: %w", err) } - token = strings.TrimSpace(token) - if token == "" { - return "", fmt.Errorf("admin key cannot be empty") + key = strings.TrimSpace(key) + if key == "" { + return "", fmt.Errorf("key cannot be empty") } - return token, nil + return key, nil } // SavePushResult saves the result of a successful push operation diff --git a/client/main.go b/client/main.go index 45e972a..c04b663 100644 --- a/client/main.go +++ b/client/main.go @@ -155,7 +155,7 @@ func runSync(cmd *cobra.Command, args []string) error { // Create sync coordinator coordinator := sync.NewCoordinator(&sync.CoordinatorConfig{ ServerURL: serverURL, - ProvidedAuthToken: getAuthToken(), + ProvidedAuthKey: getAuthKey(), ProvidedPullKey: pullKey, ProvidedPushKey: pushKey, ProvidedReplicaID: replicaID, @@ -224,10 +224,10 @@ func determineOperation(args []string) (sync.Operation, string, string, error) { return sync.Operation(0), "", "", fmt.Errorf("invalid arguments") } -func getAuthToken() string { +func getAuthKey() string { // Try environment variable first - if token := os.Getenv("SQLRSYNC_AUTH_TOKEN"); token != "" { - return token + if key := os.Getenv("SQLRSYNC_AUTH_KEY"); key != "" { + return key } // Try pull/push keys diff --git a/client/remote/client.go b/client/remote/client.go index 0b00cd2..4eaa070 100644 --- a/client/remote/client.go +++ b/client/remote/client.go @@ -401,9 +401,9 @@ type Config struct { EnableTrafficInspection bool // Enable detailed traffic logging InspectionDepth int // How many bytes to inspect (default: 32) PingPong bool - AuthToken string + AuthKey string ClientVersion string // version of the client software - SendKeyRequest bool // the -sqlrsync file doesn't exist, so make a token + SendKeyRequest bool // the -sqlrsync file doesn't exist, so make a key SendConfigCmd bool // we don't have the version number or remote path LocalHostname string @@ -685,9 +685,9 @@ func (c *Client) Connect() error { headers := http.Header{} - headers.Set("Authorization", c.config.AuthToken) + headers.Set("Authorization", c.config.AuthKey) - headers.Set("X-ClientVersion", c.config.ClientVersion); + headers.Set("X-ClientVersion", c.config.ClientVersion) if c.config.WsID != "" { headers.Set("X-ClientID", c.config.WsID) @@ -882,9 +882,9 @@ func (c *Client) Read(buffer []byte) (int, error) { if c.config.Subscribe { return 1 * time.Hour } - // Use a longer timeout if sync is completed to allow final transaction processing + // Use a shorter timeout if sync is completed to allow final transaction processing if c.isSyncCompleted() { - return 2 * time.Second + return 1 * time.Second } return 30 * time.Second }()): @@ -1012,7 +1012,7 @@ func (c *Client) Close() { if c.conn != nil { // Send close message closeMessage := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") - err := c.conn.WriteControl(websocket.CloseMessage, closeMessage, time.Now().Add(5*time.Second)) + err := c.conn.WriteControl(websocket.CloseMessage, closeMessage, time.Now().Add(3*time.Second)) if err != nil { c.logger.Debug("Error sending close message", zap.Error(err)) } else { diff --git a/client/subscription/manager.go b/client/subscription/manager.go index 74e45c2..5c3139b 100644 --- a/client/subscription/manager.go +++ b/client/subscription/manager.go @@ -1,12 +1,14 @@ package subscription import ( + "bufio" "context" "encoding/json" "fmt" "io" "net/http" "net/url" + "os" "strings" "sync" "time" @@ -38,7 +40,7 @@ type Message struct { type ManagerConfig struct { ServerURL string ReplicaPath string - AuthToken string + AccessKey string ReplicaID string WsID string // websocket ID for client identification ClientVersion string // version of the client software @@ -199,7 +201,7 @@ func (m *Manager) doConnect() error { u.Path = strings.TrimSuffix(u.Path, "/") + "/sapi/subscribe/" + m.config.ReplicaPath headers := http.Header{} - headers.Set("Authorization", m.config.AuthToken) + headers.Set("Authorization", m.config.AccessKey) if m.config.ReplicaID != "" { headers.Set("X-ReplicaID", m.config.ReplicaID) } @@ -229,6 +231,20 @@ func (m *Manager) doConnect() error { } } + // Connect to remote server + if strings.Contains(err.Error(), "key is not authorized") || strings.Contains(err.Error(), "404 Path not found") { + if m.config.AccessKey == "" { + key, err := PromptForKey(m.config.ServerURL, m.config.ReplicaPath, "PULL") + if err != nil { + return fmt.Errorf("manager failed to get key interactively: %w", err) + } + m.config.AccessKey = key + return m.doConnect() + } else { + return fmt.Errorf("manager failed to connect to server: %w", err) + } + } + // Create a clean error message var errorMsg strings.Builder errorMsg.WriteString(fmt.Sprintf("HTTP %d (%s)", statusCode, statusText)) @@ -574,3 +590,24 @@ func (m *Manager) pingLoop() { } } } + +// PromptForKey prompts the user for an admin key +func PromptForKey(serverURL string, remotePath string, keyType string) (string, error) { + httpServer := strings.Replace(serverURL, "ws", "http", 1) + fmt.Println("Replica not found when using unauthenticated access. Try again using a key or check your spelling.") + fmt.Println(" Get a key at " + httpServer + "/namespaces or " + httpServer + "/" + remotePath) + fmt.Print(" Provide a key to " + keyType + ": ") + + reader := bufio.NewReader(os.Stdin) + key, err := reader.ReadString('\n') + if err != nil { + return "", fmt.Errorf("failed to read admin key: %w", err) + } + + key = strings.TrimSpace(key) + if key == "" { + return "", fmt.Errorf("admin key cannot be empty") + } + + return key, nil +} diff --git a/client/sync/coordinator.go b/client/sync/coordinator.go index 4da1af6..e57ea56 100644 --- a/client/sync/coordinator.go +++ b/client/sync/coordinator.go @@ -32,7 +32,7 @@ const ( // CoordinatorConfig holds sync coordinator configuration type CoordinatorConfig struct { ServerURL string - ProvidedAuthToken string // Explicitly provided auth token + ProvidedAuthKey string // Explicitly provided auth key ProvidedPullKey string // Explicitly provided pull key ProvidedPushKey string // Explicitly provided push key ProvidedReplicaID string // Explicitly provided replica ID @@ -133,10 +133,10 @@ func (c *Coordinator) displayDryRunInfo(operation string, authResult *auth.Resol if operation != "local" { fmt.Printf(" - Server: %s\n", color.GreenString(serverURL)) - if authResult.AccessToken != "" { - fmt.Printf(" - Access Token: %s\n", color.GreenString(authResult.AccessToken)) + if authResult.AccessKey != "" { + fmt.Printf(" - Access Key: %s\n", color.GreenString(authResult.AccessKey)) } else { - fmt.Printf(" - Access Token: %s\n", color.YellowString("(none)")) + fmt.Printf(" - Access Key: %s\n", color.YellowString("(none)")) } if operation == "push" { @@ -179,14 +179,14 @@ func (c *Coordinator) resolveAuth(operation string) (*auth.ResolveResult, error) Logger: c.logger, } - // Try explicit auth token first - if c.config.ProvidedAuthToken != "" { + // Try explicit auth key first + if c.config.ProvidedAuthKey != "" { return &auth.ResolveResult{ - AccessToken: c.config.ProvidedAuthToken, - ReplicaID: c.config.ProvidedReplicaID, - ServerURL: c.config.ServerURL, - RemotePath: c.config.RemotePath, - LocalPath: c.config.LocalPath, + AccessKey: c.config.ProvidedAuthKey, + ReplicaID: c.config.ProvidedReplicaID, + ServerURL: c.config.ServerURL, + RemotePath: c.config.RemotePath, + LocalPath: c.config.LocalPath, }, nil } @@ -196,12 +196,12 @@ func (c *Coordinator) resolveAuth(operation string) (*auth.ResolveResult, error) } // If prompting is needed for push operations - if result.ShouldPrompt && operation == "push" { - token, err := c.authResolver.PromptForAdminKey(c.config.ServerURL) + if result.ShouldPrompt || operation == "push" { + key, err := c.authResolver.PromptForKey(c.config.ServerURL, c.config.RemotePath, "PUSH") if err != nil { return nil, err } - result.AccessToken = token + result.AccessKey = key result.ShouldPrompt = false } @@ -245,7 +245,7 @@ func (c *Coordinator) executeSubscribe() error { c.subManager = subscription.NewManager(&subscription.ManagerConfig{ ServerURL: authResult.ServerURL, ReplicaPath: authResult.RemotePath, - AuthToken: authResult.AccessToken, + AccessKey: authResult.AccessKey, ReplicaID: authResult.ReplicaID, WsID: c.config.WsID, ClientVersion: c.config.ClientVersion, @@ -366,7 +366,7 @@ func (c *Coordinator) executePull(isSubscription bool) error { // Create remote client for WebSocket transport remoteClient, err := remote.New(&remote.Config{ ServerURL: serverURL + "/sapi/pull/" + remotePath, - AuthToken: authResult.AccessToken, + AuthKey: authResult.AccessKey, ReplicaID: authResult.ReplicaID, Timeout: 8000, PingPong: false, // No ping/pong needed for single sync @@ -398,7 +398,17 @@ func (c *Coordinator) executePull(isSubscription bool) error { // Connect to remote server if err := remoteClient.Connect(); err != nil { - return fmt.Errorf("failed to connect to server: %w", err) + if (strings.Contains(err.Error(), "key is not authorized") || strings.Contains(err.Error(), "404 Path not found")) && authResult.AccessKey == "" { + key, err := c.authResolver.PromptForKey(c.config.ServerURL, c.config.RemotePath, "PULL") + if err != nil { + return fmt.Errorf("coordinator failed to get key interactively: %w", err) + } + c.config.ProvidedAuthKey = key + return c.executePull(isSubscription) + } else { + + return fmt.Errorf("coordinator failed to connect to server: %w", err) + } } // Create local client for SQLite operations @@ -495,7 +505,7 @@ func (c *Coordinator) executePush() error { ServerURL: serverURL + "/sapi/push/" + remotePath, PingPong: true, Timeout: 15000, - AuthToken: authResult.AccessToken, + AuthKey: authResult.AccessKey, Logger: c.logger.Named("remote"), EnableTrafficInspection: c.config.Verbose, LocalHostname: localHostname, @@ -507,7 +517,7 @@ func (c *Coordinator) executePush() error { CommitMessage: c.config.CommitMessage, WsID: c.config.WsID, // Add websocket ID ClientVersion: c.config.ClientVersion, - ProgressCallback: nil, //remote.DefaultProgressCallback(remote.FormatSimple), + ProgressCallback: nil, //remote.DefaultProgressCallback(remote.FormatSimple), ProgressConfig: &remote.ProgressConfig{ Enabled: true, Format: remote.FormatSimple, From b4e1c949ff38dff7e78482357c1f66cbcebf1d47 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Sat, 4 Oct 2025 14:41:54 -0700 Subject: [PATCH 02/20] fix: add make install to Makefile --- client/Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/client/Makefile b/client/Makefile index ec5744f..26981ad 100644 --- a/client/Makefile +++ b/client/Makefile @@ -1,5 +1,5 @@ # SQLite Rsync Go Client Makefile -.PHONY: all build clean test deps check-deps install-deps run help +.PHONY: all build clean test deps check-deps install-deps install run help # Build configuration BINARY_NAME := sqlrsync @@ -55,6 +55,12 @@ build: $(SQLITE_RSYNC_LIB) CGO_ENABLED=$(CGO_ENABLED) CGO_LDFLAGS="-L$(BRIDGE_LIB_DIR) -lsqlite_rsync" go build $(GOFLAGS) -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/$(BINARY_NAME) $(MAIN_FILE) @echo "✓ Build complete: $(BUILD_DIR)/$(BINARY_NAME)" +# Install the binary to system path +install: build + @echo "Installing $(BINARY_NAME) to /usr/local/bin/..." + cp $(BUILD_DIR)/$(BINARY_NAME) /usr/local/bin/$(BINARY_NAME) + @echo "✓ Install complete: /usr/local/bin/$(BINARY_NAME)" + # Build with debug symbols build-debug: check-deps @echo "Building $(BINARY_NAME) with debug symbols..." @@ -104,6 +110,7 @@ help: @echo " all - Check dependencies and build (default)" @echo " build - Build the binary" @echo " build-debug - Build with debug symbols" + @echo " install - Build and install binary to /usr/local/bin" @echo " clean - Remove build artifacts" @echo " deps - Download Go dependencies" @echo " check-deps - Check system dependencies" @@ -118,5 +125,6 @@ help: @echo "Usage examples:" @echo " make build" @echo " make run" + @echo " make install" @echo " make run-dry" @echo " make test" \ No newline at end of file From c36db4f09fe40b71fbb14f1a47ba608f717726bb Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Sat, 4 Oct 2025 14:51:52 -0700 Subject: [PATCH 03/20] feat: add earthquakes example --- examples/earthquakes/gov.usgs.earthquakes.sh | 142 +++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100755 examples/earthquakes/gov.usgs.earthquakes.sh diff --git a/examples/earthquakes/gov.usgs.earthquakes.sh b/examples/earthquakes/gov.usgs.earthquakes.sh new file mode 100755 index 0000000..914ddfe --- /dev/null +++ b/examples/earthquakes/gov.usgs.earthquakes.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +# USGS Earthquake Data Synchronization Script +# Downloads earthquake data every 50 minutes and syncs to SQLRsync + +# Configuration +FILE=earthquakes.db +TABLE=earthquakes +UPDATES=50m +URL=https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_month.csv +SQLRSYNC_PATH=usgs.gov/earthquakes.db +PRIMARY_KEY=id +MODE="INSERT OR REPLACE INTO" +SCHEMA="time TEXT, latitude REAL, longitude REAL, depth REAL, mag REAL, magType TEXT, nst INTEGER, gap REAL, dmin REAL, rms REAL, net TEXT, id TEXT PRIMARY KEY, updated TEXT, place TEXT, type TEXT, horizontalError REAL, depthError REAL, magError REAL, magNst INTEGER, status TEXT, locationSource TEXT, magSource TEXT" + +# Convert time interval to seconds for sleep +convert_to_seconds() { + local time_str="$1" + local num="${time_str%[a-zA-Z]*}" + local unit="${time_str#$num}" + + case "$unit" in + s|sec) echo "$num" ;; + m|min) echo $((num * 60)) ;; + h|hour) echo $((num * 3600)) ;; + d|day) echo $((num * 86400)) ;; + *) echo 3000 ;; # default to 50 minutes + esac +} + +# Initialize database and table +init_database() { + echo "Initializing database: $FILE" + sqlite3 "$FILE" "CREATE TABLE IF NOT EXISTS $TABLE ($SCHEMA);" + if [ $? -eq 0 ]; then + echo "Database initialized successfully" + else + echo "Error: Failed to initialize database" + exit 1 + fi +} + +# Download and import data with in-memory staging +sync_data() { + echo "$(date): Downloading earthquake data from USGS..." + + # Download CSV data + local temp_file=$(mktemp) + if curl -s -f "$URL" -o "$temp_file"; then + echo "Data downloaded successfully" + + # Get record count before import + local count_before=$(sqlite3 "$FILE" "SELECT COUNT(*) FROM $TABLE;" 2>/dev/null || echo "0") + + # Use in-memory database for staging to avoid bloating main database + sqlite3 "$FILE" </dev/null | head -3 + + # Sync to SQLRsync server if path is configured + if [ -n "$SQLRSYNC_PATH" ] && command -v sqlrsync >/dev/null 2>&1; then + echo "Syncing to SQLRsync server: $SQLRSYNC_PATH" + sqlrsync "$FILE" "$SQLRSYNC_PATH" + if [ $? -eq 0 ]; then + echo "Successfully synced to server" + else + echo "Warning: Failed to sync to server" + fi + fi + else + echo "Error: Failed to import data" + fi + else + echo "Error: Failed to download data from $URL" + fi + + rm -f "$temp_file" +} + +# Main execution +main() { + echo "Fetch CSV to SQLite Data Sync Starting..." + echo "Configuration:" + echo " Database: $FILE" + echo " Table: $TABLE" + echo " Update interval: $UPDATES" + echo " Data source: $URL" + echo " SQLRsync path: $SQLRSYNC_PATH" + echo "" + + # Initialize database + init_database + + # Convert update interval to seconds + local sleep_seconds=$(convert_to_seconds "$UPDATES") + echo "Update interval: $sleep_seconds seconds" + echo "" + + # Initial sync + sync_data + + # Continuous sync loop + echo "Starting continuous sync (Ctrl+C to stop)..." + while true; do + echo "Sleeping for $UPDATES ($sleep_seconds seconds)..." + sleep "$sleep_seconds" + sync_data + done +} + +# Run main function if script is executed directly +if [ "${BASH_SOURCE[0]}" == "${0}" ]; then + main "$@" +fi + From 572701d928cda73bac9eb3391f75c8ebde17b992 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Sat, 4 Oct 2025 14:54:07 -0700 Subject: [PATCH 04/20] prettier --- client/auth/resolver.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/client/auth/resolver.go b/client/auth/resolver.go index 58724fc..66a24ad 100644 --- a/client/auth/resolver.go +++ b/client/auth/resolver.go @@ -22,14 +22,14 @@ type ResolveResult struct { // ResolveRequest contains the parameters for authentication resolution type ResolveRequest struct { - LocalPath string - RemotePath string - ServerURL string - ProvidedPullKey string - ProvidedPushKey string + LocalPath string + RemotePath string + ServerURL string + ProvidedPullKey string + ProvidedPushKey string ProvidedReplicaID string - Operation string // "pull", "push", "subscribe" - Logger *zap.Logger + Operation string // "pull", "push", "subscribe" + Logger *zap.Logger } // Resolver handles authentication and configuration resolution @@ -87,7 +87,7 @@ func (r *Resolver) Resolve(req *ResolveRequest) (*ResolveResult, error) { if req.ServerURL == "wss://sqlrsync.com" { if localSecretsConfig, err := LoadLocalSecretsConfig(); err == nil { if dbConfig := localSecretsConfig.FindDatabaseByPath(absLocalPath); dbConfig != nil { - r.logger.Debug("Using server URL from local secrets config", + r.logger.Debug("Using server URL from local secrets config", zap.String("configuredServer", dbConfig.Server), zap.String("defaultServer", req.ServerURL)) result.ServerURL = dbConfig.Server @@ -138,7 +138,7 @@ func (r *Resolver) Resolve(req *ResolveRequest) (*ResolveResult, error) { // resolveFromLocalSecrets attempts to resolve auth from local-secrets.toml func (r *Resolver) resolveFromLocalSecrets(absLocalPath, serverURL string, result *ResolveResult) (*ResolveResult, error) { r.logger.Debug("Attempting to resolve from local secrets", zap.String("absLocalPath", absLocalPath), zap.String("serverURL", serverURL)) - + localSecretsConfig, err := LoadLocalSecretsConfig() if err != nil { r.logger.Debug("Failed to load local secrets config", zap.Error(err)) @@ -162,8 +162,8 @@ func (r *Resolver) resolveFromLocalSecrets(absLocalPath, serverURL string, resul } if dbConfig.Server != serverURL { - r.logger.Debug("Server URL mismatch", - zap.String("configured", dbConfig.Server), + r.logger.Debug("Server URL mismatch", + zap.String("configured", dbConfig.Server), zap.String("requested", serverURL)) return nil, fmt.Errorf("server URL mismatch: configured=%s, requested=%s", dbConfig.Server, serverURL) } @@ -280,4 +280,4 @@ func (r *Resolver) CheckNeedsDashFile(localPath, remotePath string) bool { } return dashSQLRsync.RemotePath != remotePath -} \ No newline at end of file +} From 3858b7d1fadcc44fb3afd20c86ca1d16ee55119b Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Sat, 4 Oct 2025 14:57:07 -0700 Subject: [PATCH 05/20] fix: ah, that and should have been an or --- client/sync/coordinator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/sync/coordinator.go b/client/sync/coordinator.go index e57ea56..1b25c05 100644 --- a/client/sync/coordinator.go +++ b/client/sync/coordinator.go @@ -196,7 +196,7 @@ func (c *Coordinator) resolveAuth(operation string) (*auth.ResolveResult, error) } // If prompting is needed for push operations - if result.ShouldPrompt || operation == "push" { + if result.ShouldPrompt || (operation == "push" && result.AccessKey == "") { key, err := c.authResolver.PromptForKey(c.config.ServerURL, c.config.RemotePath, "PUSH") if err != nil { return nil, err From 025927c1319c845263735e2a8b47ca437690d094 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Sat, 4 Oct 2025 14:58:34 -0700 Subject: [PATCH 06/20] version bump to 0.0.6 --- client/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/main.go b/client/main.go index 45e972a..d604baa 100644 --- a/client/main.go +++ b/client/main.go @@ -15,7 +15,7 @@ import ( "github.com/sqlrsync/sqlrsync.com/sync" ) -var VERSION = "0.0.5" +var VERSION = "0.0.6" var ( serverURL string verbose bool From cb2c441da0921c075c311f9c906a3169def16a4b Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Sat, 4 Oct 2025 16:17:00 -0700 Subject: [PATCH 07/20] fix: improve in memory update of db --- examples/earthquakes/gov.usgs.earthquakes.sh | 24 +++++--------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/examples/earthquakes/gov.usgs.earthquakes.sh b/examples/earthquakes/gov.usgs.earthquakes.sh index 914ddfe..eb5eb61 100755 --- a/examples/earthquakes/gov.usgs.earthquakes.sh +++ b/examples/earthquakes/gov.usgs.earthquakes.sh @@ -53,26 +53,14 @@ sync_data() { local count_before=$(sqlite3 "$FILE" "SELECT COUNT(*) FROM $TABLE;" 2>/dev/null || echo "0") # Use in-memory database for staging to avoid bloating main database - sqlite3 "$FILE" < Date: Sat, 4 Oct 2025 17:20:50 -0700 Subject: [PATCH 08/20] fix: more data for usgs --- examples/earthquakes/gov.usgs.earthquakes.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/earthquakes/gov.usgs.earthquakes.sh b/examples/earthquakes/gov.usgs.earthquakes.sh index eb5eb61..3a3b7ab 100755 --- a/examples/earthquakes/gov.usgs.earthquakes.sh +++ b/examples/earthquakes/gov.usgs.earthquakes.sh @@ -8,6 +8,7 @@ FILE=earthquakes.db TABLE=earthquakes UPDATES=50m URL=https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_month.csv +URL=https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.csv SQLRSYNC_PATH=usgs.gov/earthquakes.db PRIMARY_KEY=id MODE="INSERT OR REPLACE INTO" From f1ed6fbaacb993657120fd5a504e0d9d5e75536a Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Sat, 4 Oct 2025 17:54:23 -0700 Subject: [PATCH 09/20] feat: check integrity before push and after pull --- bridge/cgo_bridge.go | 30 ++++++++++++++++++ bridge/client.go | 29 +++++++++++++++++- bridge/sqlite_rsync_wrapper.c | 57 +++++++++++++++++++++++++++++++++++ bridge/sqlite_rsync_wrapper.h | 4 +++ client/sync/coordinator.go | 44 ++++++++++++++++++--------- 5 files changed, 148 insertions(+), 16 deletions(-) diff --git a/bridge/cgo_bridge.go b/bridge/cgo_bridge.go index c8d9782..ab6970d 100644 --- a/bridge/cgo_bridge.go +++ b/bridge/cgo_bridge.go @@ -65,6 +65,36 @@ func cgoGetDatabaseInfo(dbPath string) (*DatabaseInfo, error) { return info, nil } +// SQLRSYNC +// CheckIntegrity checks the database integrity using PRAGMA integrity_check +func CheckIntegrity(dbPath string) (bool, string, error) { + return cgoCheckIntegrity(dbPath) +} + +// SQLRSYNC +// cgoCheckIntegrity checks the database integrity using PRAGMA integrity_check +func cgoCheckIntegrity(dbPath string) (bool, string, error) { + cDbPath := C.CString(dbPath) + defer C.free(unsafe.Pointer(cDbPath)) + + const errorMsgSize = 1024 + errorMsg := make([]byte, errorMsgSize) + cErrorMsg := (*C.char)(unsafe.Pointer(&errorMsg[0])) + + result := C.sqlite_rsync_check_integrity(cDbPath, cErrorMsg, C.int(errorMsgSize)) + + switch result { + case 0: + return true, "", nil // Database is OK + case 1: + return false, C.GoString(cErrorMsg), nil // Database is corrupted + default: + return false, C.GoString(cErrorMsg), &SQLiteRsyncError{ + Code: int(result), + Message: "failed to check database integrity", + } + } +} // RunOriginSync wraps the C function to run origin synchronization func RunOriginSync(dbPath string, dryRun bool, client *BridgeClient) error { diff --git a/bridge/client.go b/bridge/client.go index 43ccb5e..0d5cd27 100644 --- a/bridge/client.go +++ b/bridge/client.go @@ -2,6 +2,7 @@ package bridge import ( "fmt" + "os" "go.uber.org/zap" ) @@ -70,6 +71,32 @@ func (c *BridgeClient) GetDatabaseInfo() (*DatabaseInfo, error) { return info, nil } +// CheckIntegrity checks the database integrity using PRAGMA integrity_check +func (c *BridgeClient) CheckIntegrity() (bool, string, error) { + c.Logger.Debug("Checking database integrity", zap.String("path", c.Config.DatabasePath)) + + if _, err := os.Stat(c.Config.DatabasePath); os.IsNotExist(err) { + return false, "", fmt.Errorf("database file does not exist: %s", c.Config.DatabasePath) + } + + fmt.Println("🐛 CHECKING INTEGRITY BEFORE PUSH") + isOk, errorMsg, err := CheckIntegrity(c.Config.DatabasePath) + if err != nil { + c.Logger.Error("Failed to check database integrity", zap.Error(err)) + return false, "", err + } + + if !isOk { + c.Logger.Warn("Database integrity check failed", + zap.String("database", c.Config.DatabasePath), + zap.String("error", errorMsg)) + return false, errorMsg, nil + } + + c.Logger.Debug("Database integrity check passed", zap.String("database", c.Config.DatabasePath)) + return true, "", nil +} + // RunPushSync runs the origin-side synchronization with provided I/O functions func (c *BridgeClient) RunPushSync(readFunc ReadFunc, writeFunc WriteFunc) error { c.Logger.Info("Starting origin sync", zap.String("database", c.Config.DatabasePath)) @@ -117,7 +144,7 @@ func (c *BridgeClient) RunPullSync(readFunc ReadFunc, writeFunc WriteFunc) error return err } - c.Logger.Info("Replica sync completed successfully") + c.Logger.Info("Replica sync completed") return nil } diff --git a/bridge/sqlite_rsync_wrapper.c b/bridge/sqlite_rsync_wrapper.c index eeba020..771e4c5 100644 --- a/bridge/sqlite_rsync_wrapper.c +++ b/bridge/sqlite_rsync_wrapper.c @@ -132,6 +132,63 @@ int sqlite_rsync_get_db_info(const char *db_path, sqlite_db_info_t *info) sqlite3_close(db); return 0; } +// SQLRSYNC +// Check database integrity using PRAGMA integrity_check +int sqlite_rsync_check_integrity(const char *db_path, char *error_msg, int error_msg_size) +{ + if (!db_path || !error_msg) + { + return -1; + } + + // Initialize error message + error_msg[0] = '\0'; + + sqlite3 *db; + int rc = sqlite3_open_v2(db_path, &db, SQLITE_OPEN_READONLY, NULL); + if (rc != SQLITE_OK) + { + if (error_msg_size > 0) + { + snprintf(error_msg, error_msg_size, "Cannot open database: %s", sqlite3_errmsg(db)); + } + if (db) + sqlite3_close(db); + return -1; + } + + sqlite3_stmt *stmt; + rc = sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL); + if (rc != SQLITE_OK) + { + if (error_msg_size > 0) + { + snprintf(error_msg, error_msg_size, "Cannot prepare integrity check: %s", sqlite3_errmsg(db)); + } + sqlite3_close(db); + return -1; + } + + int result = 0; // Assume OK + while (sqlite3_step(stmt) == SQLITE_ROW) + { + const char *result_text = (const char *)sqlite3_column_text(stmt, 0); + if (result_text && strcmp(result_text, "ok") != 0) + { + // Database is corrupted + result = 1; + if (error_msg_size > 0) + { + snprintf(error_msg, error_msg_size, "Integrity check failed: %s", result_text); + } + break; + } + } + + sqlite3_finalize(stmt); + sqlite3_close(db); + return result; +} // Cleanup resources void sqlite_rsync_cleanup(void) diff --git a/bridge/sqlite_rsync_wrapper.h b/bridge/sqlite_rsync_wrapper.h index 431eb7c..d6f874f 100644 --- a/bridge/sqlite_rsync_wrapper.h +++ b/bridge/sqlite_rsync_wrapper.h @@ -78,6 +78,10 @@ extern "C" int sqlite_rsync_get_db_info(const char *db_path, sqlite_db_info_t *info); + // SQLRSYNC: Check database integrity using PRAGMA integrity_check + // Returns 0 if OK, 1 if corrupted, -1 on error + int sqlite_rsync_check_integrity(const char *db_path, char *error_msg, int error_msg_size); + // Cleanup resources void sqlite_rsync_cleanup(void); diff --git a/client/sync/coordinator.go b/client/sync/coordinator.go index 1b25c05..027eb63 100644 --- a/client/sync/coordinator.go +++ b/client/sync/coordinator.go @@ -427,6 +427,16 @@ func (c *Coordinator) executePull(isSubscription bool) error { if err := c.performPullSync(localClient, remoteClient); err != nil { return fmt.Errorf("pull synchronization failed: %w", err) } + + // Check database integrity after pull + isOk, errorMsg, err := localClient.CheckIntegrity() + if err != nil { + return fmt.Errorf("failed to check local database integrity: %w", err) + } + if !isOk { + return fmt.Errorf("database is corrupt: %s", errorMsg) + } + c.config.Version = remoteClient.GetVersion() // Save pull result if needed if remoteClient.GetNewPullKey() != "" && c.authResolver.CheckNeedsDashFile(c.config.LocalPath, remotePath) { @@ -450,9 +460,25 @@ func (c *Coordinator) executePull(isSubscription bool) error { // executePush performs a push sync operation func (c *Coordinator) executePush() error { - // Validate that database file exists - if _, err := os.Stat(c.config.LocalPath); os.IsNotExist(err) { - return fmt.Errorf("database file does not exist: %s", c.config.LocalPath) + // Create local client for SQLite operations + localClient, err := bridge.New(&bridge.BridgeConfig{ + DatabasePath: c.config.LocalPath, + DryRun: c.config.DryRun, + Logger: c.logger.Named("local"), + EnableSQLiteRsyncLogging: c.config.Verbose, + }) + if err != nil { + return fmt.Errorf("failed to create local client: %w", err) + } + defer localClient.Close() + + // Check database integrity before pushing + isOk, errorMsg, err := localClient.CheckIntegrity() + if err != nil { + return fmt.Errorf("failed to check local database integrity: %w", err) + } + if !isOk { + return fmt.Errorf("cannot create local client because integrity check failed: %s", errorMsg) } // Resolve authentication @@ -473,18 +499,6 @@ func (c *Coordinator) executePush() error { remotePath = c.config.RemotePath } - // Create local client for SQLite operations - localClient, err := bridge.New(&bridge.BridgeConfig{ - DatabasePath: c.config.LocalPath, - DryRun: c.config.DryRun, - Logger: c.logger.Named("local"), - EnableSQLiteRsyncLogging: c.config.Verbose, - }) - if err != nil { - return fmt.Errorf("failed to create local client: %w", err) - } - defer localClient.Close() - // Get absolute path for the local database absLocalPath, err := filepath.Abs(c.config.LocalPath) if err != nil { From e263778f39d6c4c43e4ce3ae631ef90d688d4be3 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Sat, 4 Oct 2025 18:03:21 -0700 Subject: [PATCH 10/20] fix: cleanup integrity check code --- bridge/client.go | 12 ++++-------- client/sync/coordinator.go | 17 +++-------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/bridge/client.go b/bridge/client.go index 0d5cd27..1bbc925 100644 --- a/bridge/client.go +++ b/bridge/client.go @@ -72,29 +72,25 @@ func (c *BridgeClient) GetDatabaseInfo() (*DatabaseInfo, error) { } // CheckIntegrity checks the database integrity using PRAGMA integrity_check -func (c *BridgeClient) CheckIntegrity() (bool, string, error) { +func (c *BridgeClient) CheckIntegrity() { c.Logger.Debug("Checking database integrity", zap.String("path", c.Config.DatabasePath)) if _, err := os.Stat(c.Config.DatabasePath); os.IsNotExist(err) { - return false, "", fmt.Errorf("database file does not exist: %s", c.Config.DatabasePath) + c.Logger.Fatal("database file does not exist", zap.String("path", c.Config.DatabasePath)) } - fmt.Println("🐛 CHECKING INTEGRITY BEFORE PUSH") isOk, errorMsg, err := CheckIntegrity(c.Config.DatabasePath) if err != nil { - c.Logger.Error("Failed to check database integrity", zap.Error(err)) - return false, "", err + c.Logger.Fatal("fatal error while checking integrity", zap.Error(err)) } if !isOk { - c.Logger.Warn("Database integrity check failed", + c.Logger.Fatal("database integrity check failed", zap.String("database", c.Config.DatabasePath), zap.String("error", errorMsg)) - return false, errorMsg, nil } c.Logger.Debug("Database integrity check passed", zap.String("database", c.Config.DatabasePath)) - return true, "", nil } // RunPushSync runs the origin-side synchronization with provided I/O functions diff --git a/client/sync/coordinator.go b/client/sync/coordinator.go index 027eb63..0af5335 100644 --- a/client/sync/coordinator.go +++ b/client/sync/coordinator.go @@ -429,13 +429,7 @@ func (c *Coordinator) executePull(isSubscription bool) error { } // Check database integrity after pull - isOk, errorMsg, err := localClient.CheckIntegrity() - if err != nil { - return fmt.Errorf("failed to check local database integrity: %w", err) - } - if !isOk { - return fmt.Errorf("database is corrupt: %s", errorMsg) - } + localClient.CheckIntegrity() c.config.Version = remoteClient.GetVersion() // Save pull result if needed @@ -473,13 +467,8 @@ func (c *Coordinator) executePush() error { defer localClient.Close() // Check database integrity before pushing - isOk, errorMsg, err := localClient.CheckIntegrity() - if err != nil { - return fmt.Errorf("failed to check local database integrity: %w", err) - } - if !isOk { - return fmt.Errorf("cannot create local client because integrity check failed: %s", errorMsg) - } + localClient.CheckIntegrity() + // Resolve authentication authResult, err := c.resolveAuth("push") From cfc4c7d7ed11e78bc662889a949cfaefe688a338 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Sun, 5 Oct 2025 13:07:14 -0700 Subject: [PATCH 11/20] fix: attach a commit message to sqlrsync --- .gitignore | 1 + examples/earthquakes/gov.usgs.earthquakes.sh | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 91a2606..f81f746 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ client/sqlrsync client/sqlrsync client/sqlrsync_simple asciinema/ +examples/earthquakes/nohup.out diff --git a/examples/earthquakes/gov.usgs.earthquakes.sh b/examples/earthquakes/gov.usgs.earthquakes.sh index 3a3b7ab..8c667aa 100755 --- a/examples/earthquakes/gov.usgs.earthquakes.sh +++ b/examples/earthquakes/gov.usgs.earthquakes.sh @@ -61,7 +61,10 @@ sync_data() { -- Use INSERT OR REPLACE to merge data from in-memory temp to main table ATTACH DATABASE "$FILE" AS c; -INSERT OR REPLACE INTO c.earthquakes SELECT * FROM incoming; + +CREATE TABLE IF NOT EXISTS c.$TABLE ($SCHEMA); + +INSERT OR REPLACE INTO c.$TABLE SELECT * FROM incoming; EOF if [ $? -eq 0 ]; then @@ -76,7 +79,7 @@ EOF # Sync to SQLRsync server if path is configured if [ -n "$SQLRSYNC_PATH" ] && command -v sqlrsync >/dev/null 2>&1; then echo "Syncing to SQLRsync server: $SQLRSYNC_PATH" - sqlrsync "$FILE" "$SQLRSYNC_PATH" + sqlrsync "$FILE" "$SQLRSYNC_PATH" -m "Added/updated $new_records records from USGS" if [ $? -eq 0 ]; then echo "Successfully synced to server" else From 228278b2b65c77cf36bae214a04bc6c7e38fc235 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Sun, 5 Oct 2025 16:57:55 -0700 Subject: [PATCH 12/20] fix: tweaks to usgs --- examples/earthquakes/gov.usgs.earthquakes.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/earthquakes/gov.usgs.earthquakes.sh b/examples/earthquakes/gov.usgs.earthquakes.sh index 8c667aa..f34a27c 100755 --- a/examples/earthquakes/gov.usgs.earthquakes.sh +++ b/examples/earthquakes/gov.usgs.earthquakes.sh @@ -6,7 +6,7 @@ # Configuration FILE=earthquakes.db TABLE=earthquakes -UPDATES=50m +UPDATES=5m URL=https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_month.csv URL=https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.csv SQLRSYNC_PATH=usgs.gov/earthquakes.db @@ -65,6 +65,9 @@ ATTACH DATABASE "$FILE" AS c; CREATE TABLE IF NOT EXISTS c.$TABLE ($SCHEMA); INSERT OR REPLACE INTO c.$TABLE SELECT * FROM incoming; + +SELECT CHANGES() AS CHANGED; +DETACH DATABASE c; EOF if [ $? -eq 0 ]; then @@ -79,7 +82,7 @@ EOF # Sync to SQLRsync server if path is configured if [ -n "$SQLRSYNC_PATH" ] && command -v sqlrsync >/dev/null 2>&1; then echo "Syncing to SQLRsync server: $SQLRSYNC_PATH" - sqlrsync "$FILE" "$SQLRSYNC_PATH" -m "Added/updated $new_records records from USGS" + sqlrsync "$FILE" "$SQLRSYNC_PATH" -m "Added $new_records records, others updated" if [ $? -eq 0 ]; then echo "Successfully synced to server" else From ee75384e5507265644716a0239b2936102c96d68 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Mon, 6 Oct 2025 20:02:04 -0700 Subject: [PATCH 13/20] add e2e testing --- .github/workflows/build.yml | 349 ++++++++++++++++++------------------ .github/workflows/test.yml | 79 ++++++++ 2 files changed, 256 insertions(+), 172 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b05c4f..e19d3fa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,13 +2,26 @@ name: Multi-Platform Build on: push: - branches: [main, develop] + branches: + - main + - develop + - "v*" # matches any branch that starts with 'v' pull_request: branches: [main] release: types: [published] jobs: + securityIntention: + name: Security Intention + runs-on: ubuntu-latest + steps: + - name: Security Intention + run: | + echo "This workflow is intended to build the project in a secure manner:" + echo " - Only installs absolutely essential and trusted dependencies. (steps \"Install *\")" + echo " - Uses HTTPS for direct package downloads" + echo " - Only uses official Github Actions \"actions/*\"" build: name: Build for ${{ matrix.os }} runs-on: ${{ matrix.runs-on }} @@ -24,145 +37,137 @@ jobs: - os: linux runs-on: ubuntu-latest arch: x86_64 - - steps: - - name: Security Intention - run: | - echo "This workflow is intended to build the project in a secure manner:" - echo " - Only installs absolutely essential and trusted dependencies. (steps \"Install *\")" - echo " - Uses HTTPS for direct package downloads" - echo " - Only uses official Github Actions \"actions/*\"" - - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: "1.21" - - - name: Install dependencies (Ubuntu) - if: matrix.os == 'linux' - run: | - sudo apt-get update - sudo apt-get install -y sqlite3 libsqlite3-dev build-essential - - - name: Install dependencies (macOS) - if: matrix.os == 'darwin' - run: | - brew install sqlite3 - - - name: Install dependencies (Windows) - if: matrix.os == 'windows' - run: | - choco install sqlite - - - name: Build SQLite (Unix) - if: matrix.os != 'windows' - run: | - cd sqlite - if [ ! -d "sqlite-latest" ]; then - # Download and extract SQLite source if not present - curl -O https://www.sqlite.org/2024/sqlite-autoconf-3450100.tar.gz - tar xzf sqlite-autoconf-3450100.tar.gz - mv sqlite-autoconf-3450100 sqlite-latest - fi - cd sqlite-latest - ./configure --prefix=$(pwd)/../install --enable-static --disable-shared "CFLAGS=-DSQLITE_ENABLE_DBPAGE_VTAB=1 -O2" - make - make install - - - name: Build SQLite (Windows) - if: matrix.os == 'windows' - run: | - cd sqlite - if (!(Test-Path "sqlite-latest")) { - Invoke-WebRequest -Uri "https://www.sqlite.org/2024/sqlite-autoconf-3450100.tar.gz" -OutFile "sqlite-autoconf-3450100.tar.gz" - tar -xzf sqlite-autoconf-3450100.tar.gz - Rename-Item sqlite-autoconf-3450100 sqlite-latest - } - cd sqlite-latest - # Use MSYS2/MinGW for Windows build - bash -c "./configure --prefix=$(pwd)/../install --enable-static --disable-shared 'CFLAGS=-DSQLITE_ENABLE_DBPAGE_VTAB=1 -O2'" - bash -c "make" - bash -c "make install" - - - name: Build Bridge - run: | - cd bridge - make - - - name: Set build environment (Darwin ARM64) - if: matrix.os == 'darwin' && matrix.arch == 'arm64' - run: | - echo "GOOS=darwin" >> $GITHUB_ENV - echo "GOARCH=arm64" >> $GITHUB_ENV - echo "CGO_ENABLED=1" >> $GITHUB_ENV - - - name: Set build environment (Darwin AMD64) - if: matrix.os == 'darwin' && matrix.arch == 'amd64' - run: | - echo "GOOS=darwin" >> $GITHUB_ENV - echo "GOARCH=amd64" >> $GITHUB_ENV - echo "CGO_ENABLED=1" >> $GITHUB_ENV - - - name: Set build environment (Linux x86_64) - if: matrix.os == 'linux' && matrix.arch == 'x86_64' - run: | - echo "GOOS=linux" >> $GITHUB_ENV - echo "GOARCH=amd64" >> $GITHUB_ENV - echo "CGO_ENABLED=1" >> $GITHUB_ENV - - - name: Set build environment (Windows) - if: matrix.os == 'windows' - run: | - echo "GOOS=windows" >> $env:GITHUB_ENV - echo "GOARCH=amd64" >> $env:GITHUB_ENV - echo "CGO_ENABLED=1" >> $env:GITHUB_ENV - - - name: Build Client (Unix) - if: matrix.os != 'windows' - run: | - cd client - make build - - - name: Build Client (Windows) - if: matrix.os == 'windows' - run: | - cd client - # Use make with MSYS2/MinGW - bash -c "make build" - - - name: Create release directory - run: | - mkdir -p release - - - name: Package binary (Unix) - if: matrix.os != 'windows' - run: | - if [ "${{ matrix.os }}" = "darwin" ] && [ "${{ matrix.arch }}" = "arm64" ]; then - BINARY_NAME="sqlrsync-${{ matrix.os }}-${{ matrix.arch }}" - else - BINARY_NAME="sqlrsync-${{ matrix.os }}-${{ matrix.arch }}" - fi - cp client/bin/sqlrsync release/${BINARY_NAME} - - - name: Package binary (Windows) - if: matrix.os == 'windows' - run: | - $BINARY_NAME = "sqlrsync-${{ matrix.os }}-${{ matrix.arch }}.exe" - Copy-Item "client/bin/sqlrsync.exe" "release/$BINARY_NAME" - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: sqlrsync-${{ matrix.os }}-${{ matrix.arch }} - path: release/* + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: "1.21" + + - name: Install dependencies (Ubuntu) + if: matrix.os == 'linux' + run: | + sudo apt-get update + sudo apt-get install -y sqlite3 libsqlite3-dev build-essential + + - name: Install dependencies (macOS) + if: matrix.os == 'darwin' + run: | + brew install sqlite3 + + - name: Install dependencies (Windows) + if: matrix.os == 'windows' + run: | + choco install sqlite + + - name: Build SQLite (Unix) + if: matrix.os != 'windows' + run: | + cd sqlite + if [ ! -d "sqlite-latest" ]; then + # Download and extract SQLite source if not present + curl -O https://www.sqlite.org/2024/sqlite-autoconf-3450100.tar.gz + tar xzf sqlite-autoconf-3450100.tar.gz + mv sqlite-autoconf-3450100 sqlite-latest + fi + cd sqlite-latest + ./configure --prefix=$(pwd)/../install --enable-static --disable-shared "CFLAGS=-DSQLITE_ENABLE_DBPAGE_VTAB=1 -O2" + make + make install + + - name: Build SQLite (Windows) + if: matrix.os == 'windows' + run: | + cd sqlite + if (!(Test-Path "sqlite-latest")) { + Invoke-WebRequest -Uri "https://www.sqlite.org/2024/sqlite-autoconf-3450100.tar.gz" -OutFile "sqlite-autoconf-3450100.tar.gz" + tar -xzf sqlite-autoconf-3450100.tar.gz + Rename-Item sqlite-autoconf-3450100 sqlite-latest + } + cd sqlite-latest + # Use MSYS2/MinGW for Windows build + bash -c "./configure --prefix=$(pwd)/../install --enable-static --disable-shared 'CFLAGS=-DSQLITE_ENABLE_DBPAGE_VTAB=1 -O2'" + bash -c "make" + bash -c "make install" + + - name: Build Bridge + run: | + cd bridge + make + + - name: Set build environment (Darwin ARM64) + if: matrix.os == 'darwin' && matrix.arch == 'arm64' + run: | + echo "GOOS=darwin" >> $GITHUB_ENV + echo "GOARCH=arm64" >> $GITHUB_ENV + echo "CGO_ENABLED=1" >> $GITHUB_ENV + + - name: Set build environment (Darwin AMD64) + if: matrix.os == 'darwin' && matrix.arch == 'amd64' + run: | + echo "GOOS=darwin" >> $GITHUB_ENV + echo "GOARCH=amd64" >> $GITHUB_ENV + echo "CGO_ENABLED=1" >> $GITHUB_ENV + + - name: Set build environment (Linux x86_64) + if: matrix.os == 'linux' && matrix.arch == 'x86_64' + run: | + echo "GOOS=linux" >> $GITHUB_ENV + echo "GOARCH=amd64" >> $GITHUB_ENV + echo "CGO_ENABLED=1" >> $GITHUB_ENV + + - name: Set build environment (Windows) + if: matrix.os == 'windows' + run: | + echo "GOOS=windows" >> $env:GITHUB_ENV + echo "GOARCH=amd64" >> $env:GITHUB_ENV + echo "CGO_ENABLED=1" >> $env:GITHUB_ENV + + - name: Build Client (Unix) + if: matrix.os != 'windows' + run: | + cd client + make build + + - name: Build Client (Windows) + if: matrix.os == 'windows' + run: | + cd client + # Use make with MSYS2/MinGW + bash -c "make build" + + - name: Create release directory + run: | + mkdir -p release + + - name: Package binary (Unix) + if: matrix.os != 'windows' + run: | + if [ "${{ matrix.os }}" = "darwin" ] && [ "${{ matrix.arch }}" = "arm64" ]; then + BINARY_NAME="sqlrsync-${{ matrix.os }}-${{ matrix.arch }}" + else + BINARY_NAME="sqlrsync-${{ matrix.os }}-${{ matrix.arch }}" + fi + cp client/bin/sqlrsync release/${BINARY_NAME} + + - name: Package binary (Windows) + if: matrix.os == 'windows' + run: | + $BINARY_NAME = "sqlrsync-${{ matrix.os }}-${{ matrix.arch }}.exe" + Copy-Item "client/bin/sqlrsync.exe" "release/$BINARY_NAME" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: sqlrsync-${{ matrix.os }}-${{ matrix.arch }} + path: release/* release: if: github.ref == 'refs/heads/main' && github.event_name == 'push' needs: build - permissions: + permissions: contents: write packages: write issues: write @@ -170,39 +175,39 @@ jobs: actions: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - - name: Extract version from main.go - id: extract-version - run: | - VERSION=$(grep 'var VERSION = ' client/main.go | sed 's/var VERSION = "\(.*\)"/\1/') - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "Extracted version: $VERSION" - - - name: Check if tag exists - id: tag-check - run: | - VERSION=${{ steps.extract-version.outputs.version }} - if git rev-parse "v$VERSION" >/dev/null 2>&1; then - echo "Tag v$VERSION already exists" - echo "tag-created=false" >> $GITHUB_OUTPUT - else - echo "Tag v$VERSION does not exist, will create" - echo "tag-created=true" >> $GITHUB_OUTPUT - fi - - - name: Download all release artifacts - if: steps.tag-check.outputs.tag-created == 'true' - uses: actions/download-artifact@v5 - - - name: Create tag and GitHub Release, attach artifact - env: - GH_TOKEN: ${{ github.token }} - run: | - TAG=v${{ steps.extract-version.outputs.version }} - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - git tag -a $TAG -m "Release $TAG" - git push origin $TAG - # create the release and attach the artifact (gh CLI) - gh release create $TAG --generate-notes sqlrsync-*/sqlrsync-* \ No newline at end of file + - uses: actions/checkout@v5 + + - name: Extract version from main.go + id: extract-version + run: | + VERSION=$(grep 'var VERSION = ' client/main.go | sed 's/var VERSION = "\(.*\)"/\1/') + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + + - name: Check if tag exists + id: tag-check + run: | + VERSION=${{ steps.extract-version.outputs.version }} + if git rev-parse "v$VERSION" >/dev/null 2>&1; then + echo "Tag v$VERSION already exists" + echo "tag-created=false" >> $GITHUB_OUTPUT + else + echo "Tag v$VERSION does not exist, will create" + echo "tag-created=true" >> $GITHUB_OUTPUT + fi + + - name: Download all release artifacts + if: steps.tag-check.outputs.tag-created == 'true' + uses: actions/download-artifact@v5 + + - name: Create tag and GitHub Release, attach artifact + env: + GH_TOKEN: ${{ github.token }} + run: | + TAG=v${{ steps.extract-version.outputs.version }} + git config user.name "${{ github.actor }}" + git config user.email "${{ github.actor }}@users.noreply.github.com" + git tag -a $TAG -m "Release $TAG" + git push origin $TAG + # create the release and attach the artifact (gh CLI) + gh release create $TAG --generate-notes sqlrsync-*/sqlrsync-* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..67f3075 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,79 @@ +name: End-to-End Tests + +on: + workflow_run: + workflows: ["Multi-Platform Build"] + types: + - completed + branches: + - main + - develop + - "v*" + workflow_dispatch: + inputs: + build_run_id: + description: "Build workflow run ID to download artifacts from" + required: false + type: string + +jobs: + securityIntention: + name: Security Intention + runs-on: ubuntu-latest + steps: + - name: Security Intention + run: | + echo "This workflow is intended to test the project in a secure manner:" + echo " - Downloads artifacts from the build workflow" + echo " - Only uses official Github Actions \"actions/*\"" + echo " - Performs end-to-end testing with production data sources" + + e2e-test: + name: End-to-End Test + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + steps: + - name: Download Linux artifact from build workflow + uses: actions/download-artifact@v4 + with: + name: sqlrsync-linux-x86_64 + run-id: ${{ github.event.workflow_run.id || github.event.inputs.build_run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Make sqlrsync executable + run: | + chmod +x sqlrsync-linux-x86_64 + sudo cp sqlrsync-linux-x86_64 /usr/local/bin/sqlrsync + + - name: Test sqlrsync --version + run: | + echo "Testing sqlrsync --version..." + sqlrsync --version + + - name: Test sqlrsync help + run: | + echo "Testing sqlrsync help..." + sqlrsync || true + + - name: Test sqlrsync with sqlrsync.com/usgs.gov/earthquakes.db + run: | + echo "Testing sqlrsync usgs.gov/earthquakes.db..." + sqlrsync usgs.gov/earthquakes.db + + - name: Test sqlrsync with subscribe for 10 seconds + run: | + echo "Testing sqlrsync usgs.gov/earthquakes.db --subscribe for 10 seconds..." + timeout 10s sqlrsync usgs.gov/earthquakes.db --subscribe > subscribe_output.log 2>&1 || true + + - name: Verify subscribe output + run: | + echo "Checking for 'Sync completed at' in output..." + cat subscribe_output.log + if grep -q "Sync completed at" subscribe_output.log; then + echo "✅ SUCCESS: Found 'Sync completed at' in output" + else + echo "❌ FAILURE: 'Sync completed at' not found in output" + echo "Full output:" + cat subscribe_output.log + exit 1 + fi From 89550dbe1fdbdc732bbf26bfe2491dc0bbdea7fc Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Mon, 6 Oct 2025 20:04:19 -0700 Subject: [PATCH 14/20] fix indent --- .github/workflows/build.yml | 252 ++++++++++++++++++------------------ 1 file changed, 126 insertions(+), 126 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e19d3fa..a1ea8e7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,132 +37,132 @@ jobs: - os: linux runs-on: ubuntu-latest arch: x86_64 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: "1.21" - - - name: Install dependencies (Ubuntu) - if: matrix.os == 'linux' - run: | - sudo apt-get update - sudo apt-get install -y sqlite3 libsqlite3-dev build-essential - - - name: Install dependencies (macOS) - if: matrix.os == 'darwin' - run: | - brew install sqlite3 - - - name: Install dependencies (Windows) - if: matrix.os == 'windows' - run: | - choco install sqlite - - - name: Build SQLite (Unix) - if: matrix.os != 'windows' - run: | - cd sqlite - if [ ! -d "sqlite-latest" ]; then - # Download and extract SQLite source if not present - curl -O https://www.sqlite.org/2024/sqlite-autoconf-3450100.tar.gz - tar xzf sqlite-autoconf-3450100.tar.gz - mv sqlite-autoconf-3450100 sqlite-latest - fi - cd sqlite-latest - ./configure --prefix=$(pwd)/../install --enable-static --disable-shared "CFLAGS=-DSQLITE_ENABLE_DBPAGE_VTAB=1 -O2" - make - make install - - - name: Build SQLite (Windows) - if: matrix.os == 'windows' - run: | - cd sqlite - if (!(Test-Path "sqlite-latest")) { - Invoke-WebRequest -Uri "https://www.sqlite.org/2024/sqlite-autoconf-3450100.tar.gz" -OutFile "sqlite-autoconf-3450100.tar.gz" - tar -xzf sqlite-autoconf-3450100.tar.gz - Rename-Item sqlite-autoconf-3450100 sqlite-latest - } - cd sqlite-latest - # Use MSYS2/MinGW for Windows build - bash -c "./configure --prefix=$(pwd)/../install --enable-static --disable-shared 'CFLAGS=-DSQLITE_ENABLE_DBPAGE_VTAB=1 -O2'" - bash -c "make" - bash -c "make install" - - - name: Build Bridge - run: | - cd bridge - make - - - name: Set build environment (Darwin ARM64) - if: matrix.os == 'darwin' && matrix.arch == 'arm64' - run: | - echo "GOOS=darwin" >> $GITHUB_ENV - echo "GOARCH=arm64" >> $GITHUB_ENV - echo "CGO_ENABLED=1" >> $GITHUB_ENV - - - name: Set build environment (Darwin AMD64) - if: matrix.os == 'darwin' && matrix.arch == 'amd64' - run: | - echo "GOOS=darwin" >> $GITHUB_ENV - echo "GOARCH=amd64" >> $GITHUB_ENV - echo "CGO_ENABLED=1" >> $GITHUB_ENV - - - name: Set build environment (Linux x86_64) - if: matrix.os == 'linux' && matrix.arch == 'x86_64' - run: | - echo "GOOS=linux" >> $GITHUB_ENV - echo "GOARCH=amd64" >> $GITHUB_ENV - echo "CGO_ENABLED=1" >> $GITHUB_ENV - - - name: Set build environment (Windows) - if: matrix.os == 'windows' - run: | - echo "GOOS=windows" >> $env:GITHUB_ENV - echo "GOARCH=amd64" >> $env:GITHUB_ENV - echo "CGO_ENABLED=1" >> $env:GITHUB_ENV - - - name: Build Client (Unix) - if: matrix.os != 'windows' - run: | - cd client - make build - - - name: Build Client (Windows) - if: matrix.os == 'windows' - run: | - cd client - # Use make with MSYS2/MinGW - bash -c "make build" - - - name: Create release directory - run: | - mkdir -p release - - - name: Package binary (Unix) - if: matrix.os != 'windows' - run: | - if [ "${{ matrix.os }}" = "darwin" ] && [ "${{ matrix.arch }}" = "arm64" ]; then - BINARY_NAME="sqlrsync-${{ matrix.os }}-${{ matrix.arch }}" - else - BINARY_NAME="sqlrsync-${{ matrix.os }}-${{ matrix.arch }}" - fi - cp client/bin/sqlrsync release/${BINARY_NAME} - - - name: Package binary (Windows) - if: matrix.os == 'windows' - run: | - $BINARY_NAME = "sqlrsync-${{ matrix.os }}-${{ matrix.arch }}.exe" - Copy-Item "client/bin/sqlrsync.exe" "release/$BINARY_NAME" - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: sqlrsync-${{ matrix.os }}-${{ matrix.arch }} - path: release/* + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: "1.21" + + - name: Install dependencies (Ubuntu) + if: matrix.os == 'linux' + run: | + sudo apt-get update + sudo apt-get install -y sqlite3 libsqlite3-dev build-essential + + - name: Install dependencies (macOS) + if: matrix.os == 'darwin' + run: | + brew install sqlite3 + + - name: Install dependencies (Windows) + if: matrix.os == 'windows' + run: | + choco install sqlite + + - name: Build SQLite (Unix) + if: matrix.os != 'windows' + run: | + cd sqlite + if [ ! -d "sqlite-latest" ]; then + # Download and extract SQLite source if not present + curl -O https://www.sqlite.org/2024/sqlite-autoconf-3450100.tar.gz + tar xzf sqlite-autoconf-3450100.tar.gz + mv sqlite-autoconf-3450100 sqlite-latest + fi + cd sqlite-latest + ./configure --prefix=$(pwd)/../install --enable-static --disable-shared "CFLAGS=-DSQLITE_ENABLE_DBPAGE_VTAB=1 -O2" + make + make install + + - name: Build SQLite (Windows) + if: matrix.os == 'windows' + run: | + cd sqlite + if (!(Test-Path "sqlite-latest")) { + Invoke-WebRequest -Uri "https://www.sqlite.org/2024/sqlite-autoconf-3450100.tar.gz" -OutFile "sqlite-autoconf-3450100.tar.gz" + tar -xzf sqlite-autoconf-3450100.tar.gz + Rename-Item sqlite-autoconf-3450100 sqlite-latest + } + cd sqlite-latest + # Use MSYS2/MinGW for Windows build + bash -c "./configure --prefix=$(pwd)/../install --enable-static --disable-shared 'CFLAGS=-DSQLITE_ENABLE_DBPAGE_VTAB=1 -O2'" + bash -c "make" + bash -c "make install" + + - name: Build Bridge + run: | + cd bridge + make + + - name: Set build environment (Darwin ARM64) + if: matrix.os == 'darwin' && matrix.arch == 'arm64' + run: | + echo "GOOS=darwin" >> $GITHUB_ENV + echo "GOARCH=arm64" >> $GITHUB_ENV + echo "CGO_ENABLED=1" >> $GITHUB_ENV + + - name: Set build environment (Darwin AMD64) + if: matrix.os == 'darwin' && matrix.arch == 'amd64' + run: | + echo "GOOS=darwin" >> $GITHUB_ENV + echo "GOARCH=amd64" >> $GITHUB_ENV + echo "CGO_ENABLED=1" >> $GITHUB_ENV + + - name: Set build environment (Linux x86_64) + if: matrix.os == 'linux' && matrix.arch == 'x86_64' + run: | + echo "GOOS=linux" >> $GITHUB_ENV + echo "GOARCH=amd64" >> $GITHUB_ENV + echo "CGO_ENABLED=1" >> $GITHUB_ENV + + - name: Set build environment (Windows) + if: matrix.os == 'windows' + run: | + echo "GOOS=windows" >> $env:GITHUB_ENV + echo "GOARCH=amd64" >> $env:GITHUB_ENV + echo "CGO_ENABLED=1" >> $env:GITHUB_ENV + + - name: Build Client (Unix) + if: matrix.os != 'windows' + run: | + cd client + make build + + - name: Build Client (Windows) + if: matrix.os == 'windows' + run: | + cd client + # Use make with MSYS2/MinGW + bash -c "make build" + + - name: Create release directory + run: | + mkdir -p release + + - name: Package binary (Unix) + if: matrix.os != 'windows' + run: | + if [ "${{ matrix.os }}" = "darwin" ] && [ "${{ matrix.arch }}" = "arm64" ]; then + BINARY_NAME="sqlrsync-${{ matrix.os }}-${{ matrix.arch }}" + else + BINARY_NAME="sqlrsync-${{ matrix.os }}-${{ matrix.arch }}" + fi + cp client/bin/sqlrsync release/${BINARY_NAME} + + - name: Package binary (Windows) + if: matrix.os == 'windows' + run: | + $BINARY_NAME = "sqlrsync-${{ matrix.os }}-${{ matrix.arch }}.exe" + Copy-Item "client/bin/sqlrsync.exe" "release/$BINARY_NAME" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: sqlrsync-${{ matrix.os }}-${{ matrix.arch }} + path: release/* release: if: github.ref == 'refs/heads/main' && github.event_name == 'push' From 1655d09eb049431ce85c050b191111f8e700ad14 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Mon, 6 Oct 2025 22:11:06 -0700 Subject: [PATCH 15/20] add tests to build --- .github/workflows/build.yml | 54 ++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a1ea8e7..bb41438 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,13 +1,9 @@ name: Multi-Platform Build - on: - push: + pull_request: branches: - main - - develop - - "v*" # matches any branch that starts with 'v' - pull_request: - branches: [main] + - "v*" release: types: [published] @@ -138,6 +134,52 @@ jobs: # Use make with MSYS2/MinGW bash -c "make build" + - name: Test sqlrsync --version + run: | + echo "Testing sqlrsync --version..." + ./client/bin/sqlrsync --version + + - name: Test sqlrsync help + run: | + echo "Testing sqlrsync help..." + ./client/bin/sqlrsync || true + + - name: Test sqlrsync with usgs.gov/earthquakes.db + run: | + echo "Testing sqlrsync usgs.gov/earthquakes.db..." + ./client/bin/sqlrsync usgs.gov/earthquakes.db + + - name: Test sqlrsync with subscribe for 10 seconds (Unix) + if: matrix.os != 'windows' + run: | + echo "Testing sqlrsync usgs.gov/earthquakes.db --subscribe for 10 seconds..." + timeout 10s ./client/bin/sqlrsync usgs.gov/earthquakes.db --subscribe > subscribe_output.log 2>&1 || true + + - name: Test sqlrsync with subscribe for 10 seconds (Windows) + if: matrix.os == 'windows' + run: | + echo "Testing sqlrsync usgs.gov/earthquakes.db --subscribe for 10 seconds..." + # Windows doesn't have timeout, use PowerShell equivalent + $job = Start-Job { ./client/bin/sqlrsync.exe usgs.gov/earthquakes.db --subscribe } + Wait-Job $job -Timeout 10 + Stop-Job $job + Receive-Job $job > subscribe_output.log 2>&1 || $true + + - name: Verify subscribe output (Unix) + if: matrix.os != 'windows' + run: | + echo "Checking for 'Sync completed at' in output..." + cat subscribe_output.log + if grep -q "Sync completed at" subscribe_output.log; then + echo "✅ SUCCESS: Found 'Sync completed at' in output" + else + echo "❌ FAILURE: 'Sync completed at' not found in output" + echo "Full output:" + cat subscribe_output.log + exit 1 + fi + + - name: Create release directory run: | mkdir -p release From e55c63026340297abc3e0618952f91afa03c0d10 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Mon, 6 Oct 2025 22:11:10 -0700 Subject: [PATCH 16/20] remove unused test.yml file --- .github/workflows/test.yml | 79 -------------------------------------- 1 file changed, 79 deletions(-) delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 67f3075..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: End-to-End Tests - -on: - workflow_run: - workflows: ["Multi-Platform Build"] - types: - - completed - branches: - - main - - develop - - "v*" - workflow_dispatch: - inputs: - build_run_id: - description: "Build workflow run ID to download artifacts from" - required: false - type: string - -jobs: - securityIntention: - name: Security Intention - runs-on: ubuntu-latest - steps: - - name: Security Intention - run: | - echo "This workflow is intended to test the project in a secure manner:" - echo " - Downloads artifacts from the build workflow" - echo " - Only uses official Github Actions \"actions/*\"" - echo " - Performs end-to-end testing with production data sources" - - e2e-test: - name: End-to-End Test - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} - steps: - - name: Download Linux artifact from build workflow - uses: actions/download-artifact@v4 - with: - name: sqlrsync-linux-x86_64 - run-id: ${{ github.event.workflow_run.id || github.event.inputs.build_run_id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Make sqlrsync executable - run: | - chmod +x sqlrsync-linux-x86_64 - sudo cp sqlrsync-linux-x86_64 /usr/local/bin/sqlrsync - - - name: Test sqlrsync --version - run: | - echo "Testing sqlrsync --version..." - sqlrsync --version - - - name: Test sqlrsync help - run: | - echo "Testing sqlrsync help..." - sqlrsync || true - - - name: Test sqlrsync with sqlrsync.com/usgs.gov/earthquakes.db - run: | - echo "Testing sqlrsync usgs.gov/earthquakes.db..." - sqlrsync usgs.gov/earthquakes.db - - - name: Test sqlrsync with subscribe for 10 seconds - run: | - echo "Testing sqlrsync usgs.gov/earthquakes.db --subscribe for 10 seconds..." - timeout 10s sqlrsync usgs.gov/earthquakes.db --subscribe > subscribe_output.log 2>&1 || true - - - name: Verify subscribe output - run: | - echo "Checking for 'Sync completed at' in output..." - cat subscribe_output.log - if grep -q "Sync completed at" subscribe_output.log; then - echo "✅ SUCCESS: Found 'Sync completed at' in output" - else - echo "❌ FAILURE: 'Sync completed at' not found in output" - echo "Full output:" - cat subscribe_output.log - exit 1 - fi From e9089409d2d25e59c6d5049db32fd216f623710f Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Mon, 6 Oct 2025 22:19:50 -0700 Subject: [PATCH 17/20] implement timeout bypass --- .github/workflows/build.yml | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bb41438..ec07154 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -149,12 +149,28 @@ jobs: echo "Testing sqlrsync usgs.gov/earthquakes.db..." ./client/bin/sqlrsync usgs.gov/earthquakes.db - - name: Test sqlrsync with subscribe for 10 seconds (Unix) - if: matrix.os != 'windows' + - name: Test sqlrsync with subscribe for 10 seconds (Linux) + if: matrix.os == 'linux' run: | echo "Testing sqlrsync usgs.gov/earthquakes.db --subscribe for 10 seconds..." timeout 10s ./client/bin/sqlrsync usgs.gov/earthquakes.db --subscribe > subscribe_output.log 2>&1 || true + - name: Test sqlrsync with subscribe for 10 seconds (macOS) + if: matrix.os == 'darwin' + run: | + echo "Testing sqlrsync usgs.gov/earthquakes.db --subscribe for 10 seconds..." + # macOS doesn't have timeout, use gtimeout or alternative + if command -v gtimeout &> /dev/null; then + gtimeout 10s ./client/bin/sqlrsync usgs.gov/earthquakes.db --subscribe > subscribe_output.log 2>&1 || true + else + # Fallback: run in background and kill after 10 seconds + ./client/bin/sqlrsync usgs.gov/earthquakes.db --subscribe > subscribe_output.log 2>&1 & + PID=$! + sleep 10 + kill $PID 2>/dev/null || true + wait $PID 2>/dev/null || true + fi + - name: Test sqlrsync with subscribe for 10 seconds (Windows) if: matrix.os == 'windows' run: | @@ -178,6 +194,16 @@ jobs: cat subscribe_output.log exit 1 fi + echo "Checking for 'Sync completed at' in output..." + cat subscribe_output.log + if grep -q "Sync completed at" subscribe_output.log; then + echo "✅ SUCCESS: Found 'Sync completed at' in output" + else + echo "❌ FAILURE: 'Sync completed at' not found in output" + echo "Full output:" + cat subscribe_output.log + exit 1 + fi - name: Create release directory From 5013be30d7ecf2442511597648fdaf2cfee3b350 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Tue, 7 Oct 2025 10:13:41 -0700 Subject: [PATCH 18/20] generalize the sync complete statement --- .github/workflows/build.yml | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec07154..c97bd5b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -184,28 +184,16 @@ jobs: - name: Verify subscribe output (Unix) if: matrix.os != 'windows' run: | - echo "Checking for 'Sync completed at' in output..." + echo "Checking for 'Sync complete' in output..." cat subscribe_output.log - if grep -q "Sync completed at" subscribe_output.log; then - echo "✅ SUCCESS: Found 'Sync completed at' in output" + if grep -q "Sync complete" subscribe_output.log; then + echo "✅ SUCCESS: Found 'Sync complete' in output" else - echo "❌ FAILURE: 'Sync completed at' not found in output" + echo "❌ FAILURE: 'Sync complete' not found in output" echo "Full output:" cat subscribe_output.log exit 1 fi - echo "Checking for 'Sync completed at' in output..." - cat subscribe_output.log - if grep -q "Sync completed at" subscribe_output.log; then - echo "✅ SUCCESS: Found 'Sync completed at' in output" - else - echo "❌ FAILURE: 'Sync completed at' not found in output" - echo "Full output:" - cat subscribe_output.log - exit 1 - fi - - - name: Create release directory run: | mkdir -p release From 1af51fef6278a003749a0b55d4b201fb2d7b5042 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Tue, 7 Oct 2025 10:30:07 -0700 Subject: [PATCH 19/20] fix: tweak to title of build --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c97bd5b..93b3d0c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ jobs: echo " - Uses HTTPS for direct package downloads" echo " - Only uses official Github Actions \"actions/*\"" build: - name: Build for ${{ matrix.os }} + name: Build for ${{ matrix.os }}-${{matrix.arch}} runs-on: ${{ matrix.runs-on }} strategy: matrix: From 0f829a0a2b2521c16a7c04342ec9f8a9d3134ff4 Mon Sep 17 00:00:00 2001 From: pnwmatt <180812017+pnwmatt@users.noreply.github.com> Date: Tue, 7 Oct 2025 13:53:04 -0700 Subject: [PATCH 20/20] Configure Dependabot for Go modules in client directory --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8e8323f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "gomod" # See documentation for possible values + directory: "/client" # Location of package manifests + schedule: + interval: "weekly"