From 41f704256f602d3829f66b49af9c340b2d781ca0 Mon Sep 17 00:00:00 2001 From: annhiluc Date: Thu, 14 Aug 2025 17:48:18 +0000 Subject: [PATCH 1/3] feat: Change recording format to JSON serializable recording --- internal/record/recording_https_proxy.go | 60 +++--- internal/redact/redact.go | 30 ++- internal/redact/redact_test.go | 122 ++++++++--- internal/replay/replay_http_server.go | 99 ++++----- internal/store/store.go | 249 +++++++---------------- internal/store/store_test.go | 176 ++++++---------- 6 files changed, 324 insertions(+), 412 deletions(-) diff --git a/internal/record/recording_https_proxy.go b/internal/record/recording_https_proxy.go index 105188a..076c55e 100644 --- a/internal/record/recording_https_proxy.go +++ b/internal/record/recording_https_proxy.go @@ -18,6 +18,7 @@ package record import ( "bytes" + "encoding/json" "fmt" "io" "net/http" @@ -33,7 +34,7 @@ import ( type RecordingHTTPSProxy struct { prevRequestSHA string - seenFiles map[string]struct{} + seenFiles map[string]store.RecordFile config *config.EndpointConfig recordingDir string redactor *redact.Redact @@ -42,7 +43,7 @@ type RecordingHTTPSProxy struct { func NewRecordingHTTPSProxy(cfg *config.EndpointConfig, recordingDir string, redactor *redact.Redact) *RecordingHTTPSProxy { return &RecordingHTTPSProxy{ prevRequestSHA: store.HeadSHA, - seenFiles: make(map[string]struct{}), + seenFiles: make(map[string]store.RecordFile), config: cfg, recordingDir: recordingDir, redactor: redactor, @@ -86,7 +87,7 @@ func (r *RecordingHTTPSProxy) handleRequest(w http.ResponseWriter, req *http.Req } if _, ok := r.seenFiles[fileName]; !ok { // Reset to HeadSHA when first time seen a request from the given file. - recReq.PreviousRequest=store.HeadSHA + recReq.PreviousRequest = store.HeadSHA } if req.Header.Get("Upgrade") == "websocket" { @@ -108,10 +109,9 @@ func (r *RecordingHTTPSProxy) handleRequest(w http.ResponseWriter, req *http.Req http.Error(w, fmt.Sprintf("Error recording response: %v", err), http.StatusInternalServerError) return } - if (fileName != shaSum) { + if fileName != shaSum { r.prevRequestSHA = shaSum } - r.seenFiles[fileName] = struct{}{} } func (r *RecordingHTTPSProxy) redactRequest(req *http.Request) (*store.RecordedRequest, error) { @@ -123,9 +123,13 @@ func (r *RecordingHTTPSProxy) redactRequest(req *http.Request) (*store.RecordedR // Redact headers by key recordedRequest.RedactHeaders(r.config.RedactRequestHeaders) // Redacts secrets from header values - r.redactor.Headers(recordedRequest.Header) + r.redactor.Headers(recordedRequest.Headers) recordedRequest.Request = r.redactor.String(recordedRequest.Request) - recordedRequest.Body = r.redactor.Bytes(recordedRequest.Body) + var redactedBodySegments []map[string]any + for _, bodySegment := range recordedRequest.BodySegments { + redactedBodySegments = append(redactedBodySegments, r.redactor.Map(bodySegment)) + } + recordedRequest.BodySegments = redactedBodySegments return recordedRequest, nil } @@ -181,42 +185,40 @@ func (r *RecordingHTTPSProxy) recordResponse(recReq *store.RecordedRequest, resp if err != nil { return err } - recordPath := filepath.Join(r.recordingDir, fileName+".http.log") - // Default to overwriting the file, assuming it's a new file to record. - fileMode := os.O_TRUNC - // If we've seen requests with the same file name before, change the mode to append. - if _, ok := r.seenFiles[fileName]; ok { - fileMode = os.O_APPEND + recordFile, ok := r.seenFiles[fileName] + if !ok { + r.seenFiles[fileName] = store.RecordFile{RecordID: fileName} + recordFile = r.seenFiles[fileName] } - file, err := os.OpenFile(recordPath, fileMode|os.O_CREATE|os.O_WRONLY , 0644) + + var recordInteraction store.RecordInteraction + recordInteraction.Request = recReq + recordInteraction.ShaSum = shaSum + recordInteraction.Response = recordedResponse + + recordFile.Interactions = append(recordFile.Interactions, &recordInteraction) + + recordPath := filepath.Join(r.recordingDir, fileName+".json") + + // Default to overwriting the file. + fileMode := os.O_TRUNC + file, err := os.OpenFile(recordPath, fileMode|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return err } defer file.Close() - fmt.Printf("Writing request to: %s\n", recordPath) - serializedReq := recReq.Serialize() - _, err = file.WriteString(fmt.Sprintf("%s.req %d\n", shaSum, len(serializedReq))) - if err != nil { - return err - } - _, err = file.WriteString(serializedReq) + interaction, err := json.MarshalIndent(recordFile, "", " ") if err != nil { return err } - fmt.Printf("Writing response to: %s\n", recordPath) - recordedResponse.Body = r.redactor.Bytes(recordedResponse.Body) - serializedResp := recordedResponse.Serialize() - _, err = file.WriteString(fmt.Sprintf("\n%s.resp %d\n", shaSum, len(serializedResp))) - if err != nil { - return err - } - _, err = file.WriteString(serializedResp) + _, err = file.WriteString(string(interaction)) if err != nil { return err } + return nil } diff --git a/internal/redact/redact.go b/internal/redact/redact.go index 5baf9a9..35677ff 100644 --- a/internal/redact/redact.go +++ b/internal/redact/redact.go @@ -17,6 +17,7 @@ limitations under the License. package redact import ( + "encoding/json" "regexp" "strings" ) @@ -52,14 +53,12 @@ func NewRedact(secrets []string) (*Redact, error) { } // Headers redacts the secrets in the values of the http.Header. -func (r *Redact) Headers(headers map[string][]string) { +func (r *Redact) Headers(headers map[string]string) { if r == nil || r.regex == nil { return // No redactor or no secrets configured } - for name, values := range headers { - for i, value := range values { - headers[name][i] = r.regex.ReplaceAllString(value, REDACTED) - } + for name, value := range headers { + headers[name] = r.regex.ReplaceAllString(value, REDACTED) } } @@ -81,3 +80,24 @@ func (r *Redact) Bytes(input []byte) []byte { } return r.regex.ReplaceAll(input, []byte(REDACTED)) } + +func (r *Redact) Map(input map[string]any) map[string]any { + if r == nil || r.regex == nil { + return input // No redactor or no secrets configured + } + if input == nil { + return nil // Return nil if input is nil + } + jsonBytes, err := json.Marshal(input) + if err != nil { + return nil + } + redactedJsonBytes := r.Bytes(jsonBytes) + var redactedMap map[string]any + err = json.Unmarshal(redactedJsonBytes, &redactedMap) + if err != nil { + return nil + } + + return redactedMap +} diff --git a/internal/redact/redact_test.go b/internal/redact/redact_test.go index ee6952b..6affefd 100644 --- a/internal/redact/redact_test.go +++ b/internal/redact/redact_test.go @@ -17,7 +17,6 @@ limitations under the License. package redact import ( - "net/http" "testing" "github.com/stretchr/testify/require" @@ -129,52 +128,52 @@ func TestRedact_Bytes(t *testing.T) { func TestRedact_Headers(t *testing.T) { testCases := []struct { name string - headers http.Header + headers map[string]string secrets []string - expectedHeaders http.Header + expectedHeaders map[string]string }{ { name: "Redact secret in single header value", - headers: http.Header{ - "Authorization": []string{"Bearer secret_token_123"}, - "Content-Type": []string{"application/json"}, + headers: map[string]string{ + "Authorization": "Bearer secret_token_123", + "Content-Type": "application/json", }, secrets: []string{"secret_token_123"}, - expectedHeaders: http.Header{ - "Authorization": []string{"Bearer REDACTED"}, - "Content-Type": []string{"application/json"}, + expectedHeaders: map[string]string{ + "Authorization": "Bearer REDACTED", + "Content-Type": "application/json", }, }, { name: "Redact secret in multiple header values", - headers: http.Header{ - "Set-Cookie": []string{"sessionid=secret_session_id_789", "other=value"}, - "X-Api-Key": []string{"key_value_xyz"}, + headers: map[string]string{ + "Set-Cookie": "sessionid=secret_session_id_789 other=value", + "X-Api-Key": "key_value_xyz", }, secrets: []string{"secret_session_id_789", "key_value_xyz"}, - expectedHeaders: http.Header{ - "Set-Cookie": []string{"sessionid=REDACTED", "other=value"}, - "X-Api-Key": []string{"REDACTED"}, + expectedHeaders: map[string]string{ + "Set-Cookie": "sessionid=REDACTED other=value", + "X-Api-Key": "REDACTED", }, }, { name: "No secrets to redact", - headers: http.Header{ - "Authorization": []string{"Bearer token"}, + headers: map[string]string{ + "Authorization": "Bearer token", }, secrets: []string{}, - expectedHeaders: http.Header{ - "Authorization": []string{"Bearer token"}, + expectedHeaders: map[string]string{ + "Authorization": "Bearer token", }, }, { name: "Empty secret in list", - headers: http.Header{ - "Authorization": []string{"Bearer secret_token_123"}, + headers: map[string]string{ + "Authorization": "Bearer secret_token_123", }, secrets: []string{"", "secret_token_123"}, - expectedHeaders: http.Header{ - "Authorization": []string{"Bearer REDACTED"}, + expectedHeaders: map[string]string{ + "Authorization": "Bearer REDACTED", }, }, } @@ -184,12 +183,85 @@ func TestRedact_Headers(t *testing.T) { redactor, err := NewRedact(tc.secrets) require.NoError(t, err) // Create a copy of the headers to avoid modifying the original test case data - headersCopy := make(http.Header) + headersCopy := make(map[string]string) for k, v := range tc.headers { - headersCopy[k] = append([]string{}, v...) + headersCopy[k] = v } redactor.Headers(headersCopy) require.Equal(t, tc.expectedHeaders, headersCopy) }) } } + +func TestRedact_Map(t *testing.T) { + testCases := []struct { + name string + input map[string]any + secrets []string + expectedOutput map[string]any + }{ + { + name: "Redact single secret", + input: map[string]any{ + "key": "This is a secret: abc", + }, + secrets: []string{"abc"}, + expectedOutput: map[string]any{ + "key": "This is a secret: REDACTED", + }, + }, + { + name: "Redact multiple secrets", + input: map[string]any{ + "Secret1": "123", + "Secret2": "xyz", + }, + secrets: []string{"123", "xyz"}, + expectedOutput: map[string]any{ + "Secret1": "REDACTED", + "Secret2": "REDACTED", + }, + }, + { + name: "No secrets to redact", + input: map[string]any{ + "key": "No secrets here", + }, + secrets: []string{}, + expectedOutput: map[string]any{ + "key": "No secrets here", + }, + }, + { + name: "Empty input map", + input: map[string]any{}, + secrets: []string{"abc"}, + expectedOutput: map[string]any{}, + }, + { + name: "Empty secret in list", + input: map[string]any{ + "key": "This is a secret: abc", + }, + secrets: []string{"", "abc"}, + expectedOutput: map[string]any{ + "key": "This is a secret: REDACTED", + }, + }, + { + name: "Nil input map", + input: nil, + secrets: []string{"abc"}, + expectedOutput: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + redactor, err := NewRedact(tc.secrets) + require.NoError(t, err) + actualOutput := redactor.Map(tc.input) + require.Equal(t, tc.expectedOutput, actualOutput) + }) + } +} diff --git a/internal/replay/replay_http_server.go b/internal/replay/replay_http_server.go index 1eff395..1fc6075 100644 --- a/internal/replay/replay_http_server.go +++ b/internal/replay/replay_http_server.go @@ -17,15 +17,16 @@ limitations under the License. package replay import ( + "bufio" + "encoding/json" "fmt" + "io" "net/http" "os" "path/filepath" "strconv" "strings" "unicode" - "bufio" - "io" "github.com/google/test-server/internal/config" "github.com/google/test-server/internal/redact" @@ -35,7 +36,7 @@ import ( type ReplayHTTPServer struct { prevRequestSHA string - seenFiles map[string]struct{} + seenFiles map[string]struct{} config *config.EndpointConfig recordingDir string redactor *redact.Redact @@ -44,7 +45,7 @@ type ReplayHTTPServer struct { func NewReplayHTTPServer(cfg *config.EndpointConfig, recordingDir string, redactor *redact.Redact) *ReplayHTTPServer { return &ReplayHTTPServer{ prevRequestSHA: store.HeadSHA, - seenFiles: make(map[string]struct{}), + seenFiles: make(map[string]struct{}), config: cfg, recordingDir: recordingDir, redactor: redactor, @@ -84,7 +85,7 @@ func (r *ReplayHTTPServer) handleRequest(w http.ResponseWriter, req *http.Reques } if _, ok := r.seenFiles[fileName]; !ok { // Reset to HeadSHA when first time seen request from the given file. - redactedReq.PreviousRequest=store.HeadSHA + redactedReq.PreviousRequest = store.HeadSHA } if req.Header.Get("Upgrade") == "websocket" { fmt.Printf("Upgrading connection to websocket...\n") @@ -113,7 +114,7 @@ func (r *ReplayHTTPServer) handleRequest(w http.ResponseWriter, req *http.Reques fmt.Printf("Error writing response: %v\n", err) panic(err) } - if (fileName != shaSum) { + if fileName != shaSum { r.prevRequestSHA = shaSum } r.seenFiles[fileName] = struct{}{} @@ -128,16 +129,19 @@ func (r *ReplayHTTPServer) createRedactedRequest(req *http.Request) (*store.Reco // Redact headers by key recordedRequest.RedactHeaders(r.config.RedactRequestHeaders) // Redacts secrets from header values - r.redactor.Headers(recordedRequest.Header) + r.redactor.Headers(recordedRequest.Headers) recordedRequest.Request = r.redactor.String(recordedRequest.Request) - recordedRequest.Body = r.redactor.Bytes(recordedRequest.Body) - + var redactedBodySegments []map[string]any + for _, bodySegment := range recordedRequest.BodySegments { + redactedBodySegments = append(redactedBodySegments, r.redactor.Map(bodySegment)) + } + recordedRequest.BodySegments = redactedBodySegments return recordedRequest, nil } func (r *ReplayHTTPServer) loadResponse(fileName string, shaSum string) (*store.RecordedResponse, error) { - // 1. Open the replay log file for reading. - filePath := filepath.Join(r.recordingDir, fileName+".http.log") + // Open the replay log file for reading. + filePath := filepath.Join(r.recordingDir, fileName+".json") fmt.Printf("loading response from : %s with shaSum: %s\n", filePath, shaSum) file, err := os.Open(filePath) if err != nil { @@ -146,63 +150,44 @@ func (r *ReplayHTTPServer) loadResponse(fileName string, shaSum string) (*store. defer file.Close() reader := bufio.NewReader(file) - expectedKey := shaSum + ".resp" - // 2. Scan the file line by line using the reader directly. - for { - // Read one line, including the newline character. - line, err := reader.ReadString('\n') - if err != nil { - if err == io.EOF { - return nil, fmt.Errorf("response with shaSum %s not found in file", shaSum) - } - return nil, fmt.Errorf("error while reading file: %w", err) - } - trimmedLine := strings.TrimSpace(line) - parts := strings.Fields(trimmedLine) - if len(parts) != 2 { - continue - } - - fileKey := parts[0] - sizeStr := parts[1] - - size, err := strconv.Atoi(sizeStr) - if err != nil { - return nil, fmt.Errorf("invalid size format on delimiter line: '%s'", trimmedLine) - } - fmt.Printf("Bytes to load: %d\n", size) - if size < 0 { - return nil, fmt.Errorf("invalid negative size on delimiter line: '%s'", trimmedLine) - } + body, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + var recordFile store.RecordFile + err = json.Unmarshal(body, &recordFile) + if err != nil { + return nil, fmt.Errorf("unable to deserialize data to RecordFile: %w", err) + } - // 3. Read the exact number of bytes for the payload. - data := make([]byte, size) - if _, err := io.ReadFull(reader, data); err != nil { - return nil, fmt.Errorf("failed to read %d bytes after delimiter: %w", size, err) + for _, interaction := range recordFile.Interactions { + if interaction.Request.PreviousRequest == shaSum { + return interaction.Response, nil } - - // 4. Return the response when it matches our target shaSum. - if fileKey == expectedKey { - return store.DeserializeResponse(data) - } else { - continue + if interaction.ShaSum == shaSum { + return interaction.Response, nil } } + + return nil, fmt.Errorf("response with shaSum %s not found in file", shaSum) } func (r *ReplayHTTPServer) writeResponse(w http.ResponseWriter, resp *store.RecordedResponse) error { - for key, values := range resp.Header { - for _, value := range values { - if key == "Content-Length" || key == "Content-Encoding" { - continue - } - w.Header().Add(key, value) + for key, value := range resp.Headers { + if key == "Content-Length" || key == "Content-Encoding" { + continue } + w.Header().Add(key, value) } - w.WriteHeader(resp.StatusCode) + w.WriteHeader(int(resp.StatusCode)) + + jsonBytes, err := json.Marshal(resp.BodySegments[0]) + if err != nil { + return err + } - _, err := w.Write(resp.Body) + _, err = w.Write(jsonBytes) return err } diff --git a/internal/store/store.go b/internal/store/store.go index 60e5997..002a0ea 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -21,10 +21,11 @@ import ( "compress/gzip" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" + "log" "net/http" - "sort" "strings" "github.com/google/test-server/internal/config" @@ -32,14 +33,37 @@ import ( const HeadSHA = "b4d6e60a9b97e7b98c63df9308728c5c88c0b40c398046772c63447b94608b4d" +// Represents a single interaction, request and response in a replay. +type RecordInteraction struct { + Request *RecordedRequest `json:"request,omitempty"` + ShaSum string `json:"shaSum,omitempty"` + Response *RecordedResponse `json:"response,omitempty"` +} + +// Represents a recorded session. +type RecordFile struct { + RecordID string `json:"recordID,omitempty"` + Interactions []*RecordInteraction `json:"interactions,omitempty"` +} + type RecordedRequest struct { - Request string - Header http.Header - Body []byte - PreviousRequest string // The sha256 sum of the previous request in the chain. - ServerAddress string - Port int64 - Protocol string + Method string `json:"method,omitempty"` + URL string `json:"url,omitempty"` + Request string `json:"request,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + BodySegments []map[string]any `json:"bodySegments,omitempty"` + // The sha256 sum of the previous request in the chain. + PreviousRequest string `json:"previousRequest,omitempty"` + ServerAddress string `json:"serverAddress,omitempty"` + Port int64 `json:"port,omitempty"` + Protocol string `json:"protocol,omitempty"` +} + +type RecordedResponse struct { + StatusCode int32 `json:"statusCode,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + BodySegments []map[string]any `json:"bodySegments,omitempty"` + SDKResponseSegments []map[string]any `json:"sdkResponseSegments,omitempty"` } // NewRecordedRequest creates a RecordedRequest from an http.Request. @@ -58,9 +82,11 @@ func NewRecordedRequest(req *http.Request, previousRequest string, cfg config.En // Create the RecordedRequest. recordedRequest := &RecordedRequest{ + Method: req.Method, + URL: req.URL.String(), Request: request, - Header: header, - Body: body, + Headers: GetHeadersMap(&header), + BodySegments: []map[string]any{body}, PreviousRequest: previousRequest, ServerAddress: cfg.TargetHost, Port: cfg.TargetPort, @@ -70,17 +96,23 @@ func NewRecordedRequest(req *http.Request, previousRequest string, cfg config.En return recordedRequest, nil } -func readBody(req *http.Request) ([]byte, error) { +func readBody(req *http.Request) (map[string]any, error) { if req.Body == nil { - return []byte{}, nil + return map[string]any{}, nil } body, err := io.ReadAll(req.Body) if err != nil { return nil, err } + var resultMap map[string]any + err = json.Unmarshal(body, &resultMap) + if err != nil { + log.Fatalf("Error unmarshaling JSON: %v", err) + return nil, err + } // Restore the request body for further use. req.Body = io.NopCloser(bytes.NewBuffer(body)) - return body, nil + return resultMap, nil } // ComputeSum computes the SHA256 sum of a RecordedRequest. @@ -96,7 +128,7 @@ func (r *RecordedRequest) ComputeSum() string { // It returns error when test name contains illegal sequence. // If the TEST_NAME header is not present, it falls back to computed SHA256 sum. func (r *RecordedRequest) GetRecordingFileName() (string, error) { - testName := r.Header.Get("Test-Name") + testName := r.Headers["Test-Name"] if strings.Contains(testName, "../") { return "", fmt.Errorf("test name: %s contains illegal sequence '../'", testName) } @@ -108,127 +140,23 @@ func (r *RecordedRequest) GetRecordingFileName() (string, error) { } // Serialize the request. -// -// The serialization format is as follows: -// - The first line is the sha256 of the previous request as a hex string. -// - Next is the server address. -// - Next is the port. -// - Next is the protocol. -// - Next is a line of 80 asterisks. -// - Next is the HTTP request. -// - Next, a single line for each header formatted as "{key}: {value}". -// - Next, there are 2 empty lines. -// - The rest of the file is the body content. func (r *RecordedRequest) Serialize() string { - var builder strings.Builder - - // Format the SHA256 sum of the previous request. - builder.WriteString(r.PreviousRequest) - builder.WriteString("\n") - - builder.WriteString(fmt.Sprintf("Server Address: %s\n", r.ServerAddress)) - - builder.WriteString(fmt.Sprintf("Port: %d\n", r.Port)) - - builder.WriteString(fmt.Sprintf("Protocol: %s\n", r.Protocol)) - - builder.WriteString(strings.Repeat("*", 80) + "\n") - - // Format the HTTP request line. - builder.WriteString(r.Request) - builder.WriteString("\n") - - // Format the headers in sorted order. - keys := make([]string, 0, len(r.Header)) - for key := range r.Header { - keys = append(keys, key) - } - sort.Strings(keys) - - for _, key := range keys { - for _, value := range r.Header[key] { - builder.WriteString(fmt.Sprintf("%s: %s\n", key, value)) - } - } - - builder.WriteString("\n\n") - builder.WriteString(string(r.Body)) - - return builder.String() -} - -// Deserialize the request. -func Deserialize(data string) (*RecordedRequest, error) { - lines := strings.Split(data, "\n") - if len(lines) < 6 { - return nil, fmt.Errorf("invalid serialized data: not enough lines") - } - - previousRequest := lines[0] - - serverAddress := strings.TrimPrefix(lines[1], "Server Address: ") - portString := strings.TrimPrefix(lines[2], "Port: ") - protocol := strings.TrimPrefix(lines[3], "Protocol: ") - - port := 0 - if portString != "" { - _, err := fmt.Sscan(portString, &port) - if err != nil { - return nil, fmt.Errorf("invalid port: %w", err) - } - } - - request := lines[5] - - headerStart := 6 - bodyStart := -1 - headers := make(http.Header) - - for i := headerStart; i < len(lines); i++ { - if lines[i] == "" && lines[i+1] == "" { - bodyStart = i + 2 - break - } - parts := strings.SplitN(lines[i], ": ", 2) - if len(parts) != 2 { - continue - } - key := parts[0] - value := parts[1] - headers.Add(key, value) - } - - var body []byte - if bodyStart != -1 && bodyStart < len(lines) { - body = []byte(strings.Join(lines[bodyStart:], "\n")) - } - - recordedRequest := &RecordedRequest{ - Request: request, - Header: headers, - Body: body, - PreviousRequest: previousRequest, - ServerAddress: serverAddress, - Port: int64(port), - Protocol: protocol, + req, err := json.MarshalIndent(r, "", " ") + if err != nil { + fmt.Printf("unable to serialize recorded request: %s", err) + return "" } - return recordedRequest, nil + return string(req) } // RedactHeaders removes the specified headers from the RecordedRequest. func (r *RecordedRequest) RedactHeaders(headers []string) { for _, header := range headers { - r.Header.Del(header) + delete(r.Headers, header) } } -type RecordedResponse struct { - StatusCode int - Header http.Header - Body []byte -} - func NewRecordedResponse(resp *http.Response, body []byte) (*RecordedResponse, error) { if resp.Header.Get("Content-Encoding") == "gzip" { gzipReader, err := gzip.NewReader(bytes.NewReader(body)) @@ -247,69 +175,28 @@ func NewRecordedResponse(resp *http.Response, body []byte) (*RecordedResponse, e } - recordedResponse := &RecordedResponse{ - StatusCode: resp.StatusCode, - Header: resp.Header, - Body: body, - } - return recordedResponse, nil -} - -func (r *RecordedResponse) Serialize() string { - var buffer bytes.Buffer - - buffer.WriteString(fmt.Sprintf("Status code: %d \n", r.StatusCode)) - for name, values := range r.Header { - for _, value := range values { - buffer.WriteString(fmt.Sprintf("%s: %s\n", name, value)) - } - } - buffer.WriteString("\n") - buffer.Write(r.Body) - - return buffer.String() -} - -// DeserializeResponse deserializes the response. -func DeserializeResponse(data []byte) (*RecordedResponse, error) { - lines := bytes.SplitN(data, []byte("\n"), 2) - if len(lines) < 2 { - return nil, fmt.Errorf("invalid serialized data: not enough lines") - } - - statusCodeLine := lines[0] - statusCode := 0 - - _, err := fmt.Sscanf(string(statusCodeLine), "Status code: %d", &statusCode) + var bodySegment map[string]any + err := json.Unmarshal(body, &bodySegment) if err != nil { - return nil, fmt.Errorf("invalid status code: %w", err) - } - - headerBodySplit := bytes.SplitN(lines[1], []byte("\n\n"), 2) - if len(headerBodySplit) < 2 { - return nil, fmt.Errorf("invalid serialized data: no body separator") + return nil, err } - headerLines := bytes.Split(headerBodySplit[0], []byte("\n")) - headers := make(http.Header) - - for _, line := range headerLines { - parts := bytes.SplitN(line, []byte(": "), 2) - if len(parts) != 2 { - continue - } - key := string(parts[0]) - value := string(parts[1]) - headers.Add(key, value) + recordedResponse := &RecordedResponse{ + StatusCode: int32(resp.StatusCode), + Headers: GetHeadersMap(&resp.Header), + BodySegments: []map[string]any{bodySegment}, } + return recordedResponse, nil +} - body := headerBodySplit[1] +func GetHeadersMap(header *http.Header) map[string]string { + // Create a new map[string]string + headerMap := make(map[string]string) - recordedResponse := &RecordedResponse{ - StatusCode: statusCode, - Header: headers, - Body: body, + // Iterate over the http.Header and populate the new map + for key, values := range *header { + headerMap[key] = strings.Join(values, ", ") } - return recordedResponse, nil + return headerMap } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 0552807..32d9f3d 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -36,56 +36,56 @@ func TestRecordedRequest_Serialize(t *testing.T) { name: "Empty request", request: RecordedRequest{ Request: "", - Header: http.Header{}, - Body: []byte{}, + Headers: map[string]string{}, + BodySegments: []map[string]any{}, PreviousRequest: HeadSHA, ServerAddress: "", Port: 0, Protocol: "", }, - expected: HeadSHA + "\nServer Address: \nPort: 0\nProtocol: \n********************************************************************************\n\n\n\n", + expected: "{\n \"previousRequest\": \"b4d6e60a9b97e7b98c63df9308728c5c88c0b40c398046772c63447b94608b4d\"\n}", }, { name: "Request with headers", request: RecordedRequest{ Request: "GET / HTTP/1.1", - Header: http.Header{ - "Accept": []string{"application/xml"}, - "Content-Type": []string{"application/json"}, + Headers: map[string]string{ + "Accept": "application/xml", + "Content-Type": "application/json", }, - Body: []byte{}, + BodySegments: []map[string]any{}, PreviousRequest: HeadSHA, ServerAddress: "", Port: 0, Protocol: "", }, - expected: HeadSHA + "\nServer Address: \nPort: 0\nProtocol: \n********************************************************************************\nGET / HTTP/1.1\nAccept: application/xml\nContent-Type: application/json\n\n\n", + expected: "{\n \"request\": \"GET / HTTP/1.1\",\n \"headers\": {\n \"Accept\": \"application/xml\",\n \"Content-Type\": \"application/json\"\n },\n \"previousRequest\": \"b4d6e60a9b97e7b98c63df9308728c5c88c0b40c398046772c63447b94608b4d\"\n}", }, { name: "Request with body", request: RecordedRequest{ Request: "POST /data HTTP/1.1", - Header: http.Header{}, - Body: []byte("{\"key\": \"value\"}"), + Headers: map[string]string{}, + BodySegments: []map[string]any{{"key": "value"}}, PreviousRequest: HeadSHA, ServerAddress: "", Port: 0, Protocol: "", }, - expected: HeadSHA + "\nServer Address: \nPort: 0\nProtocol: \n********************************************************************************\nPOST /data HTTP/1.1\n\n\n{\"key\": \"value\"}", + expected: "{\n \"request\": \"POST /data HTTP/1.1\",\n \"bodySegments\": [\n {\n \"key\": \"value\"\n }\n ],\n \"previousRequest\": \"b4d6e60a9b97e7b98c63df9308728c5c88c0b40c398046772c63447b94608b4d\"\n}", }, { name: "Request with previous request SHA256 sum", request: RecordedRequest{ Request: "GET / HTTP/1.1", - Header: http.Header{}, - Body: []byte{}, + Headers: map[string]string{}, + BodySegments: []map[string]any{}, PreviousRequest: "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", ServerAddress: "", Port: 0, Protocol: "", }, - expected: "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20\nServer Address: \nPort: 0\nProtocol: \n********************************************************************************\nGET / HTTP/1.1\n\n\n", + expected: "{\n \"request\": \"GET / HTTP/1.1\",\n \"previousRequest\": \"0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20\"\n}", }, } @@ -108,7 +108,7 @@ func TestNewRecordedRequest(t *testing.T) { { name: "Test with body", request: func() *http.Request { - req, _ := http.NewRequest("POST", "http://example.com/test", bytes.NewBuffer([]byte("test body"))) + req, _ := http.NewRequest("POST", "http://example.com/test", bytes.NewBuffer([]byte("{\"test body\": \"\"}"))) req.Header.Set("Content-Type", "application/json") return req }(), @@ -119,8 +119,8 @@ func TestNewRecordedRequest(t *testing.T) { }, expected: &RecordedRequest{ Request: "POST http://example.com/test HTTP/1.1", - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: []byte("test body"), + Headers: map[string]string{"Content-Type": "application/json"}, + BodySegments: []map[string]any{{"test body": ""}}, PreviousRequest: HeadSHA, ServerAddress: "example.com", Port: 443, @@ -141,8 +141,8 @@ func TestNewRecordedRequest(t *testing.T) { }, expected: &RecordedRequest{ Request: "GET http://example.com/test HTTP/1.1", - Header: http.Header{}, - Body: []byte{}, + Headers: map[string]string{}, + BodySegments: []map[string]any{{}}, PreviousRequest: HeadSHA, ServerAddress: "example.com", Port: 443, @@ -177,8 +177,8 @@ func TestNewRecordedRequest(t *testing.T) { require.NoError(t, err) require.Equal(t, tc.expected.Request, recordedRequest.Request) - require.Equal(t, tc.expected.Header, recordedRequest.Header) - require.Equal(t, tc.expected.Body, recordedRequest.Body) + require.Equal(t, tc.expected.Headers, recordedRequest.Headers) + require.Equal(t, tc.expected.BodySegments, recordedRequest.BodySegments) require.Equal(t, tc.expected.PreviousRequest, recordedRequest.PreviousRequest) }) } @@ -189,142 +189,88 @@ func TestRecordedRequest_RedactHeaders(t *testing.T) { name string request RecordedRequest headersToRedact []string - expectedHeaders http.Header + expectedHeaders map[string]string }{ { name: "Redact single header", request: RecordedRequest{ Request: "GET / HTTP/1.1", - Header: http.Header{ - "Accept": []string{"application/xml"}, - "Content-Type": []string{"application/json"}, + Headers: map[string]string{ + "Accept": "application/xml", + "Content-Type": "application/json", }, - Body: []byte{}, + BodySegments: []map[string]any{}, PreviousRequest: HeadSHA, ServerAddress: "", Port: 0, Protocol: "", }, headersToRedact: []string{"Content-Type"}, - expectedHeaders: http.Header{ - "Accept": []string{"application/xml"}, + expectedHeaders: map[string]string{ + "Accept": "application/xml", }, }, { name: "Redact multiple headers", request: RecordedRequest{ Request: "GET / HTTP/1.1", - Header: http.Header{ - "Accept": []string{"application/xml"}, - "Content-Type": []string{"application/json"}, - "Authorization": []string{"Bearer token"}, + Headers: map[string]string{ + "Accept": "application/xml", + "Content-Type": "application/json", + "Authorization": "Bearer token", }, - Body: []byte{}, + BodySegments: []map[string]any{}, PreviousRequest: HeadSHA, ServerAddress: "", Port: 0, Protocol: "", }, headersToRedact: []string{"Content-Type", "Authorization"}, - expectedHeaders: http.Header{ - "Accept": []string{"application/xml"}, + expectedHeaders: map[string]string{ + "Accept": "application/xml", }, }, { name: "Redact non-existent header", request: RecordedRequest{ Request: "GET / HTTP/1.1", - Header: http.Header{ - "Accept": []string{"application/xml"}, + Headers: map[string]string{ + "Accept": "application/xml", }, - Body: []byte{}, + BodySegments: []map[string]any{}, PreviousRequest: HeadSHA, ServerAddress: "", Port: 0, Protocol: "", }, headersToRedact: []string{"Non-Existent"}, - expectedHeaders: http.Header{ - "Accept": []string{"application/xml"}, + expectedHeaders: map[string]string{ + "Accept": "application/xml", }, }, { name: "Redact all headers", request: RecordedRequest{ Request: "GET / HTTP/1.1", - Header: http.Header{ - "Accept": []string{"application/xml"}, - "Content-Type": []string{"application/json"}, + Headers: map[string]string{ + "Accept": "application/xml", + "Content-Type": "application/json", }, - Body: []byte{}, + BodySegments: []map[string]any{}, PreviousRequest: HeadSHA, ServerAddress: "", Port: 0, Protocol: "", }, headersToRedact: []string{"Accept", "Content-Type"}, - expectedHeaders: http.Header{}, + expectedHeaders: map[string]string{}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { tc.request.RedactHeaders(tc.headersToRedact) - require.Equal(t, tc.expectedHeaders, tc.request.Header, "RedactHeaders() result mismatch") - }) - } -} - -func TestRecordedRequest_Deserialize(t *testing.T) { - testCases := []struct { - name string - input string - expected *RecordedRequest - expectedErr bool - }{ - { - name: "Valid serialized request", - input: "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20\nServer Address: example.com\nPort: 8080\nProtocol: http\n********************************************************************************\nGET / HTTP/1.1\nAccept: application/xml\nContent-Type: application/json\n\n\n{\"key\": \"value\"}", - expected: &RecordedRequest{ - Request: "GET / HTTP/1.1", - Header: http.Header{"Accept": []string{"application/xml"}, "Content-Type": []string{"application/json"}}, - Body: []byte("{\"key\": \"value\"}"), - PreviousRequest: "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20", - ServerAddress: "example.com", - Port: 8080, - Protocol: "http", - }, - expectedErr: false, - }, - { - name: "Invalid serialized request - missing separator", - input: "GET / HTTP/1.1\nAccept: application/xml", - expected: nil, - expectedErr: true, - }, - { - name: "Empty input", - input: "", - expected: nil, - expectedErr: true, - }, - { - name: "Invalid serialized request - invalid port", - input: "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20\nServer Address: example.com\nPort: invalid\nProtocol: http\n********************************************************************************\nGET / HTTP/1.1\nAccept: application/xml\nContent-Type: application/json\n\n\n{\"key\": \"value\"}", - expected: nil, - expectedErr: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actual, err := Deserialize(tc.input) - if tc.expectedErr { - require.Error(t, err) - return - } - require.NoError(t, err) - require.Equal(t, tc.expected, actual) + require.Equal(t, tc.expectedHeaders, tc.request.Headers, "RedactHeaders() result mismatch") }) } } @@ -340,10 +286,10 @@ func TestRecordedRequest_GetRecordFileName(t *testing.T) { name: "Request with test name header", request: RecordedRequest{ Request: "GET / HTTP/1.1", - Header: http.Header{ - "Test-Name": []string{"random test name"}, + Headers: map[string]string{ + "Test-Name": "random test name", }, - Body: []byte{}, + BodySegments: []map[string]any{}, PreviousRequest: HeadSHA, ServerAddress: "", Port: 0, @@ -356,26 +302,26 @@ func TestRecordedRequest_GetRecordFileName(t *testing.T) { name: "Request with empty test name header", request: RecordedRequest{ Request: "GET / HTTP/1.1", - Header: http.Header{ - "Test-Name": []string{""}, + Headers: map[string]string{ + "Test-Name": "", }, - Body: []byte{}, + BodySegments: []map[string]any{}, PreviousRequest: HeadSHA, ServerAddress: "", Port: 0, Protocol: "", }, - expected: "f824dd099907ed4549822de827b075a7578baadebf08c5bc7303ead90a8f9ff7", + expected: "28f9ed309209353577523abbe4224d54aacf62c8f7cb2368b66a35088d830f4d", expectedErr: false, }, { name: "Request with invalid test name header", request: RecordedRequest{ Request: "GET / HTTP/1.1", - Header: http.Header{ - "Test-Name": []string{"../invalid_name"}, + Headers: map[string]string{ + "Test-Name": "../invalid_name", }, - Body: []byte{}, + BodySegments: []map[string]any{}, PreviousRequest: HeadSHA, ServerAddress: "", Port: 0, @@ -388,17 +334,17 @@ func TestRecordedRequest_GetRecordFileName(t *testing.T) { name: "Request without test name header", request: RecordedRequest{ Request: "GET / HTTP/1.1", - Header: http.Header{ - "Accept": []string{"application/xml"}, - "Content-Type": []string{"application/json"}, + Headers: map[string]string{ + "Accept": "application/xml", + "Content-Type": "application/json", }, - Body: []byte{}, + BodySegments: []map[string]any{}, PreviousRequest: HeadSHA, ServerAddress: "", Port: 0, Protocol: "", }, - expected: "fc060aea9a2bf35da16ed18c6be577ca64d0f91d681d5db385082df61ecf4ccf", + expected: "cf125193d9ada2cc07b684455524cc5d61e39269892178db1a8046273f3268d1", expectedErr: false, }, } From 3096743c48ac507fadba3c8cf425273844231c6c Mon Sep 17 00:00:00 2001 From: annhiluc Date: Thu, 14 Aug 2025 17:48:18 +0000 Subject: [PATCH 2/3] feat: Change recording format to JSON serializable recording --- internal/record/recording_https_proxy.go | 2 +- internal/replay/replay_http_server.go | 2 +- internal/store/store.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/record/recording_https_proxy.go b/internal/record/recording_https_proxy.go index 076c55e..5b916dd 100644 --- a/internal/record/recording_https_proxy.go +++ b/internal/record/recording_https_proxy.go @@ -194,7 +194,7 @@ func (r *RecordingHTTPSProxy) recordResponse(recReq *store.RecordedRequest, resp var recordInteraction store.RecordInteraction recordInteraction.Request = recReq - recordInteraction.ShaSum = shaSum + recordInteraction.SHASum = shaSum recordInteraction.Response = recordedResponse recordFile.Interactions = append(recordFile.Interactions, &recordInteraction) diff --git a/internal/replay/replay_http_server.go b/internal/replay/replay_http_server.go index 1fc6075..ebadc23 100644 --- a/internal/replay/replay_http_server.go +++ b/internal/replay/replay_http_server.go @@ -164,7 +164,7 @@ func (r *ReplayHTTPServer) loadResponse(fileName string, shaSum string) (*store. if interaction.Request.PreviousRequest == shaSum { return interaction.Response, nil } - if interaction.ShaSum == shaSum { + if interaction.SHASum == shaSum { return interaction.Response, nil } } diff --git a/internal/store/store.go b/internal/store/store.go index 002a0ea..1476caa 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -36,7 +36,7 @@ const HeadSHA = "b4d6e60a9b97e7b98c63df9308728c5c88c0b40c398046772c63447b94608b4 // Represents a single interaction, request and response in a replay. type RecordInteraction struct { Request *RecordedRequest `json:"request,omitempty"` - ShaSum string `json:"shaSum,omitempty"` + SHASum string `json:"shaSum,omitempty"` Response *RecordedResponse `json:"response,omitempty"` } From e80bd84a2c099ce190d553e76c8a93ff7731c6a0 Mon Sep 17 00:00:00 2001 From: annhiluc Date: Thu, 14 Aug 2025 17:48:18 +0000 Subject: [PATCH 3/3] feat: Change recording format to JSON serializable recording --- internal/record/recording_https_proxy.go | 3 ++- internal/replay/replay_http_server.go | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/record/recording_https_proxy.go b/internal/record/recording_https_proxy.go index 5b916dd..3f8449e 100644 --- a/internal/record/recording_https_proxy.go +++ b/internal/record/recording_https_proxy.go @@ -188,7 +188,7 @@ func (r *RecordingHTTPSProxy) recordResponse(recReq *store.RecordedRequest, resp recordFile, ok := r.seenFiles[fileName] if !ok { - r.seenFiles[fileName] = store.RecordFile{RecordID: fileName} + r.seenFiles[fileName] = store.RecordFile{RecordID: fileName, Interactions: []*store.RecordInteraction{}} recordFile = r.seenFiles[fileName] } @@ -198,6 +198,7 @@ func (r *RecordingHTTPSProxy) recordResponse(recReq *store.RecordedRequest, resp recordInteraction.Response = recordedResponse recordFile.Interactions = append(recordFile.Interactions, &recordInteraction) + r.seenFiles[fileName] = recordFile recordPath := filepath.Join(r.recordingDir, fileName+".json") diff --git a/internal/replay/replay_http_server.go b/internal/replay/replay_http_server.go index ebadc23..0beaa89 100644 --- a/internal/replay/replay_http_server.go +++ b/internal/replay/replay_http_server.go @@ -161,9 +161,6 @@ func (r *ReplayHTTPServer) loadResponse(fileName string, shaSum string) (*store. } for _, interaction := range recordFile.Interactions { - if interaction.Request.PreviousRequest == shaSum { - return interaction.Response, nil - } if interaction.SHASum == shaSum { return interaction.Response, nil }