diff --git a/CHANGELOG.md b/CHANGELOG.md index 29aaa8e..0ed91f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - A TypeScript SDK for convenient integration in TypeScript projects. +- Secret redaction via the TEST_SERVER_SECRETS environment variable. ## [0.0.1] - 2025-05-05 diff --git a/cmd/record.go b/cmd/record.go index fa14063..cc86fe7 100644 --- a/cmd/record.go +++ b/cmd/record.go @@ -16,8 +16,12 @@ limitations under the License. package cmd import ( + "os" + "strings" + "github.com/google/test-server/internal/config" "github.com/google/test-server/internal/record" + "github.com/google/test-server/internal/redact" "github.com/spf13/cobra" ) @@ -33,7 +37,14 @@ target server, and all requests and responses will be recorded.`, if err != nil { panic(err) } - err = record.Record(config, recordingDir) + + secrets := os.Getenv("TEST_SERVER_SECRETS") + redactor, err := redact.NewRedact(strings.Split(secrets, ",")) + if err != nil { + panic(err) + } + + err = record.Record(config, recordingDir, redactor) if err != nil { panic(err) } diff --git a/cmd/replay.go b/cmd/replay.go index 3a898a7..a5c1103 100644 --- a/cmd/replay.go +++ b/cmd/replay.go @@ -17,7 +17,11 @@ limitations under the License. package cmd import ( + "os" + "strings" + "github.com/google/test-server/internal/config" + "github.com/google/test-server/internal/redact" "github.com/google/test-server/internal/replay" "github.com/spf13/cobra" ) @@ -37,7 +41,14 @@ recording is found.`, if err != nil { panic(err) } - err = replay.Replay(config, replayRecordingDir) + + secrets := os.Getenv("TEST_SERVER_SECRETS") + redactor, err := redact.NewRedact(strings.Split(secrets, ",")) + if err != nil { + panic(err) + } + + err = replay.Replay(config, replayRecordingDir, redactor) if err != nil { panic(err) } diff --git a/internal/record/record.go b/internal/record/record.go index ca04f21..91fa154 100644 --- a/internal/record/record.go +++ b/internal/record/record.go @@ -22,9 +22,10 @@ import ( "sync" "github.com/google/test-server/internal/config" + "github.com/google/test-server/internal/redact" ) -func Record(cfg *config.TestServerConfig, recordingDir string) error { +func Record(cfg *config.TestServerConfig, recordingDir string, redactor *redact.Redact) error { // Create recording directory if it doesn't exist if err := os.MkdirAll(recordingDir, 0755); err != nil { return fmt.Errorf("failed to create recording directory: %w", err) @@ -41,7 +42,7 @@ func Record(cfg *config.TestServerConfig, recordingDir string) error { defer wg.Done() fmt.Printf("Starting server for %v\n", endpoint) - proxy := NewRecordingHTTPSProxy(&endpoint, recordingDir) + proxy := NewRecordingHTTPSProxy(&endpoint, recordingDir, redactor) err := proxy.Start() if err != nil { diff --git a/internal/record/recording_https_proxy.go b/internal/record/recording_https_proxy.go index 2fc06f4..68dbbd2 100644 --- a/internal/record/recording_https_proxy.go +++ b/internal/record/recording_https_proxy.go @@ -26,6 +26,7 @@ import ( "regexp" "github.com/google/test-server/internal/config" + "github.com/google/test-server/internal/redact" "github.com/google/test-server/internal/store" "github.com/gorilla/websocket" ) @@ -34,13 +35,15 @@ type RecordingHTTPSProxy struct { prevRequestSHA string config *config.EndpointConfig recordingDir string + redactor *redact.Redact } -func NewRecordingHTTPSProxy(cfg *config.EndpointConfig, recordingDir string) *RecordingHTTPSProxy { +func NewRecordingHTTPSProxy(cfg *config.EndpointConfig, recordingDir string, redactor *redact.Redact) *RecordingHTTPSProxy { return &RecordingHTTPSProxy{ prevRequestSHA: store.HeadSHA, config: cfg, recordingDir: recordingDir, + redactor: redactor, } } @@ -98,7 +101,12 @@ func (r *RecordingHTTPSProxy) recordRequest(req *http.Request) (string, error) { return "", err } + // Redact headers by key recordedRequest.RedactHeaders(r.config.RedactRequestHeaders) + // Redacts secrets from header values + r.redactor.Headers(recordedRequest.Header) + recordedRequest.Request = r.redactor.String(recordedRequest.Request) + recordedRequest.Body = r.redactor.Bytes(recordedRequest.Body) reqHash, err := recordedRequest.ComputeSum() if err != nil { @@ -166,6 +174,8 @@ func (r *RecordingHTTPSProxy) recordResponse(resp *http.Response, reqHash string return err } + recordedResponse.Body = r.redactor.Bytes(recordedResponse.Body) + recordPath := filepath.Join(r.recordingDir, reqHash+".resp") fmt.Printf("Writing response to: %s\n", recordPath) err = os.WriteFile(recordPath, []byte(recordedResponse.Serialize()), 0644) diff --git a/internal/redact/redact.go b/internal/redact/redact.go new file mode 100644 index 0000000..5baf9a9 --- /dev/null +++ b/internal/redact/redact.go @@ -0,0 +1,83 @@ +/* +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package redact + +import ( + "regexp" + "strings" +) + +// REDACTED is the string used to replace redacted secrets. +const REDACTED = "REDACTED" + +// Redact holds the compiled regex for redacting secrets. +type Redact struct { + regex *regexp.Regexp +} + +// NewRedact creates a new Redact instance with the given secrets. +func NewRedact(secrets []string) (*Redact, error) { + filteredSecrets := []string{} + for _, secret := range secrets { + if secret != "" { + filteredSecrets = append(filteredSecrets, regexp.QuoteMeta(secret)) + } + } + + if len(filteredSecrets) == 0 { + return &Redact{regex: nil}, nil // No secrets to redact + } + + regexPattern := strings.Join(filteredSecrets, "|") + re, err := regexp.Compile(regexPattern) + if err != nil { + return nil, err + } + + return &Redact{regex: re}, nil +} + +// Headers redacts the secrets in the values of the http.Header. +func (r *Redact) Headers(headers map[string][]string) { + if r == nil || r.regex == nil { + return // No redactor or no secrets configured + } + for name, values := range headers { + for i, value := range values { + headers[name][i] = r.regex.ReplaceAllString(value, REDACTED) + } + } +} + +// String redacts the secrets in the input string. +func (r *Redact) String(input string) string { + if r == nil || r.regex == nil { + return input // No redactor or no secrets configured + } + return r.regex.ReplaceAllString(input, REDACTED) +} + +// Bytes redacts the secrets in the input byte slice. +func (r *Redact) Bytes(input []byte) []byte { + if r == nil || r.regex == nil { + return input // No redactor or no secrets configured + } + if input == nil { + return nil // Return nil if input is nil + } + return r.regex.ReplaceAll(input, []byte(REDACTED)) +} diff --git a/internal/redact/redact_test.go b/internal/redact/redact_test.go new file mode 100644 index 0000000..ee6952b --- /dev/null +++ b/internal/redact/redact_test.go @@ -0,0 +1,195 @@ +/* +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package redact + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRedact_String(t *testing.T) { + testCases := []struct { + name string + input string + secrets []string + expectedOutput string + }{ + { + name: "Redact single secret", + input: "This is a secret: abc", + secrets: []string{"abc"}, + expectedOutput: "This is a secret: REDACTED", + }, + { + name: "Redact multiple secrets", + input: "Secret1: 123, Secret2: xyz", + secrets: []string{"123", "xyz"}, + expectedOutput: "Secret1: REDACTED, Secret2: REDACTED", + }, + { + name: "No secrets to redact", + input: "No secrets here", + secrets: []string{}, + expectedOutput: "No secrets here", + }, + { + name: "Empty input string", + input: "", + secrets: []string{"abc"}, + expectedOutput: "", + }, + { + name: "Empty secret in list", + input: "This is a secret: abc", + secrets: []string{"", "abc"}, + expectedOutput: "This is a secret: REDACTED", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + redactor, err := NewRedact(tc.secrets) + require.NoError(t, err) + actualOutput := redactor.String(tc.input) + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} + +func TestRedact_Bytes(t *testing.T) { + testCases := []struct { + name string + input []byte + secrets []string + expectedOutput []byte + }{ + { + name: "Redact single secret", + input: []byte("This is a secret: abc"), + secrets: []string{"abc"}, + expectedOutput: []byte("This is a secret: REDACTED"), + }, + { + name: "Redact multiple secrets", + input: []byte("Secret1: 123, Secret2: xyz"), + secrets: []string{"123", "xyz"}, + expectedOutput: []byte("Secret1: REDACTED, Secret2: REDACTED"), + }, + { + name: "No secrets to redact", + input: []byte("No secrets here"), + secrets: []string{}, + expectedOutput: []byte("No secrets here")}, + { + name: "Empty input bytes", + input: []byte{}, + secrets: []string{"abc"}, + expectedOutput: nil, + }, + { + name: "Empty secret in list", + input: []byte("This is a secret: abc"), + secrets: []string{"", "abc"}, + expectedOutput: []byte("This is a secret: REDACTED"), + }, + { + name: "Nil input bytes", + input: nil, + secrets: []string{"abc"}, + expectedOutput: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + redactor, err := NewRedact(tc.secrets) + require.NoError(t, err) + actualOutput := redactor.Bytes(tc.input) + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} + +func TestRedact_Headers(t *testing.T) { + testCases := []struct { + name string + headers http.Header + secrets []string + expectedHeaders http.Header + }{ + { + name: "Redact secret in single header value", + headers: http.Header{ + "Authorization": []string{"Bearer secret_token_123"}, + "Content-Type": []string{"application/json"}, + }, + secrets: []string{"secret_token_123"}, + expectedHeaders: http.Header{ + "Authorization": []string{"Bearer REDACTED"}, + "Content-Type": []string{"application/json"}, + }, + }, + { + name: "Redact secret in multiple header values", + headers: http.Header{ + "Set-Cookie": []string{"sessionid=secret_session_id_789", "other=value"}, + "X-Api-Key": []string{"key_value_xyz"}, + }, + secrets: []string{"secret_session_id_789", "key_value_xyz"}, + expectedHeaders: http.Header{ + "Set-Cookie": []string{"sessionid=REDACTED", "other=value"}, + "X-Api-Key": []string{"REDACTED"}, + }, + }, + { + name: "No secrets to redact", + headers: http.Header{ + "Authorization": []string{"Bearer token"}, + }, + secrets: []string{}, + expectedHeaders: http.Header{ + "Authorization": []string{"Bearer token"}, + }, + }, + { + name: "Empty secret in list", + headers: http.Header{ + "Authorization": []string{"Bearer secret_token_123"}, + }, + secrets: []string{"", "secret_token_123"}, + expectedHeaders: http.Header{ + "Authorization": []string{"Bearer REDACTED"}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + redactor, err := NewRedact(tc.secrets) + require.NoError(t, err) + // Create a copy of the headers to avoid modifying the original test case data + headersCopy := make(http.Header) + for k, v := range tc.headers { + headersCopy[k] = append([]string{}, v...) + } + redactor.Headers(headersCopy) + require.Equal(t, tc.expectedHeaders, headersCopy) + }) + } +} diff --git a/internal/replay/replay.go b/internal/replay/replay.go index 814d9d2..502642b 100644 --- a/internal/replay/replay.go +++ b/internal/replay/replay.go @@ -21,10 +21,11 @@ import ( "os" "github.com/google/test-server/internal/config" + "github.com/google/test-server/internal/redact" ) // Replay serves recorded responses for HTTP requests -func Replay(cfg *config.TestServerConfig, recordingDir string) error { +func Replay(cfg *config.TestServerConfig, recordingDir string, redactor *redact.Redact) error { // Validate recording directory exists if _, err := os.Stat(recordingDir); os.IsNotExist(err) { return fmt.Errorf("recording directory does not exist: %s", recordingDir) @@ -37,7 +38,7 @@ func Replay(cfg *config.TestServerConfig, recordingDir string) error { for _, endpoint := range cfg.Endpoints { go func(ep config.EndpointConfig) { - server := NewReplayHTTPServer(&endpoint, recordingDir) + server := NewReplayHTTPServer(&endpoint, recordingDir, redactor) err := server.Start() if err != nil { errChan <- fmt.Errorf("replay error for %s:%d: %w", diff --git a/internal/replay/replay_http_server.go b/internal/replay/replay_http_server.go index 32fb1db..b9ee2ad 100644 --- a/internal/replay/replay_http_server.go +++ b/internal/replay/replay_http_server.go @@ -23,6 +23,7 @@ import ( "path/filepath" "github.com/google/test-server/internal/config" + "github.com/google/test-server/internal/redact" "github.com/google/test-server/internal/store" ) @@ -30,13 +31,15 @@ type ReplayHTTPServer struct { prevRequestSHA string config *config.EndpointConfig recordingDir string + redactor *redact.Redact } -func NewReplayHTTPServer(cfg *config.EndpointConfig, recordingDir string) *ReplayHTTPServer { +func NewReplayHTTPServer(cfg *config.EndpointConfig, recordingDir string, redactor *redact.Redact) *ReplayHTTPServer { return &ReplayHTTPServer{ prevRequestSHA: store.HeadSHA, config: cfg, recordingDir: recordingDir, + redactor: redactor, } } @@ -53,8 +56,15 @@ func (r *ReplayHTTPServer) Start() error { } func (r *ReplayHTTPServer) handleRequest(w http.ResponseWriter, req *http.Request) { - fmt.Printf("Replaying request: %s %s\n", req.Method, req.URL.String()) - reqHash, err := r.computeRequestHash(req) + redactedReq, err := r.createRedactedRequest(req) + if err != nil { + fmt.Printf("Error processing request") + http.Error(w, fmt.Sprintf("Error processing request: %v", err), http.StatusInternalServerError) + return + } + fmt.Printf("Replaying request: %ss\n", redactedReq.Request) + + reqHash, err := redactedReq.ComputeSum() if err != nil { fmt.Printf("Error computing request sum: %v\n", err) http.Error(w, fmt.Sprintf("Error computing request sum: %v", err), http.StatusInternalServerError) @@ -75,20 +85,19 @@ func (r *ReplayHTTPServer) handleRequest(w http.ResponseWriter, req *http.Reques } } -func (r *ReplayHTTPServer) computeRequestHash(req *http.Request) (string, error) { +func (r *ReplayHTTPServer) createRedactedRequest(req *http.Request) (*store.RecordedRequest, error) { recordedRequest, err := store.NewRecordedRequest(req, r.prevRequestSHA, *r.config) if err != nil { - return "", err + return nil, err } + // Redact headers by key recordedRequest.RedactHeaders(r.config.RedactRequestHeaders) + // Redacts secrets from header values + r.redactor.Headers(recordedRequest.Header) + recordedRequest.Body = r.redactor.Bytes(recordedRequest.Body) - reqHash, err := recordedRequest.ComputeSum() - if err != nil { - return "", err - } - - return reqHash, nil + return recordedRequest, nil } func (r *ReplayHTTPServer) loadResponse(sha string) (*store.RecordedResponse, error) {