From c6291f6f89bd593ed31e1394dc25891a0c1e66ac Mon Sep 17 00:00:00 2001 From: fishonamos Date: Fri, 30 May 2025 10:03:20 +0100 Subject: [PATCH 01/14] feat(telemetry server): create Go Telemetry server and update cli to env --- server/common.go | 18 ++++ server/events.go | 124 ++++++++++++++++++++++++++ server/scripts.go | 215 ++++++++++++++++++++++++++++++++++++++++++++++ server/server.go | 57 ++++++++++++ server/status.go | 61 +++++++++++++ 5 files changed, 475 insertions(+) create mode 100644 server/common.go create mode 100644 server/events.go create mode 100644 server/scripts.go create mode 100644 server/server.go create mode 100644 server/status.go diff --git a/server/common.go b/server/common.go new file mode 100644 index 0000000..dc90d32 --- /dev/null +++ b/server/common.go @@ -0,0 +1,18 @@ +package server + +import ( + "net/http" + + "pyrevittelemetryserver/cli" +) + +const OkMessage = "[ {g}OK{!} ]" + +func respondError(err error, w http.ResponseWriter, logger *cli.Logger) { + message := err.Error() + logger.Debug("validation error: ", message) + _, responseErr := w.Write([]byte(message)) + if responseErr != nil { + logger.Debug(responseErr) + } +} diff --git a/server/events.go b/server/events.go new file mode 100644 index 0000000..fd47568 --- /dev/null +++ b/server/events.go @@ -0,0 +1,124 @@ +package server + +import ( + "encoding/json" + "fmt" + "net/http" + "strconv" + + "pyrevittelemetryserver/cli" + "pyrevittelemetryserver/persistence" + + "github.com/gorilla/mux" +) + +func dumpEventAndRespond(logrec interface{}, w http.ResponseWriter, logger *cli.Logger) { + jsonData, responseDataErr := json.Marshal(logrec) + if responseDataErr == nil { + jsonString := string(jsonData) + if logger.PrintTrace { + logger.Trace(jsonString) + } + w.Header().Set("Content-Type", "application/json") + _, responseErr := w.Write([]byte(jsonString)) + if responseErr != nil { + logger.Debug(responseErr) + } + } else { + logger.Debug(responseDataErr) + } +} + +func RouteEvents(router *mux.Router, opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) { + // POST events/ + // create new script telemetry record + // https://stackoverflow.com/a/26212073 + router.HandleFunc("/api/v2/events/", func(w http.ResponseWriter, r *http.Request) { + // parse given json data into a new record + logrec := persistence.EventTelemetryRecordV2{} + decodeErr := json.NewDecoder(r.Body).Decode(&logrec) + if decodeErr != nil { + logger.Debug(decodeErr) + return + } + + err := logrec.Validate() + if err != nil { + // log error + logrec.PrintRecordInfo(logger, fmt.Sprintf("[ {r}%s{!} ]", err.Error())) + // respond with error + w.WriteHeader(http.StatusBadRequest) + respondError(err, w, logger) + } else { + // now write to db + _, dbWriteErr := dbConn.WriteEventTelemetryV2(&logrec, logger) + if dbWriteErr != nil { + logger.Debug(dbWriteErr) + logrec.PrintRecordInfo(logger, fmt.Sprintf("[ {r}%s{!} ]", dbWriteErr)) + } else { + logrec.PrintRecordInfo(logger, OkMessage) + } + // respond with the created data + dumpEventAndRespond(logrec, w, logger) + } + + }).Methods("POST") + + // GET events/ + // get recorded telemetry record + router.HandleFunc("/api/v2/events/", func(w http.ResponseWriter, r *http.Request) { + // Parse query parameters + limit := 100 // Default limit + offset := 0 // Default offset + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 { + limit = l + } + } + if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" { + if o, err := strconv.Atoi(offsetStr); err == nil && o >= 0 { + offset = o + } + } + + // Parse search query if provided + var searchQuery map[string]interface{} + if searchStr := r.URL.Query().Get("q"); searchStr != "" { + if err := json.Unmarshal([]byte(searchStr), &searchQuery); err != nil { + w.WriteHeader(http.StatusBadRequest) + respondError(fmt.Errorf("invalid search query: %v", err), w, logger) + return + } + } + + // Get records + var records []persistence.EventTelemetryRecordV2 + var err error + if searchQuery != nil { + records, err = dbConn.SearchEventTelemetryV2(searchQuery, limit, offset, logger) + } else { + records, err = dbConn.ReadEventTelemetryV2(limit, offset, logger) + } + + if err != nil { + logger.Debug(err) + w.WriteHeader(http.StatusInternalServerError) + respondError(err, w, logger) + return + } + + // Write response + w.Header().Set("Content-Type", "application/json") + jsonData, err := json.Marshal(records) + if err != nil { + logger.Debug(err) + w.WriteHeader(http.StatusInternalServerError) + respondError(err, w, logger) + return + } + + if _, err := w.Write(jsonData); err != nil { + logger.Debug(err) + } + }).Methods("GET") +} diff --git a/server/scripts.go b/server/scripts.go new file mode 100644 index 0000000..4b12370 --- /dev/null +++ b/server/scripts.go @@ -0,0 +1,215 @@ +package server + +import ( + "encoding/json" + "fmt" + "net/http" + "strconv" + + "pyrevittelemetryserver/cli" + "pyrevittelemetryserver/persistence" + + "github.com/gorilla/mux" +) + +func dumpScriptAndRespond(logrec interface{}, w http.ResponseWriter, logger *cli.Logger) { + // dump the telemetry record json data if requested + jsonData, responseDataErr := json.Marshal(logrec) + if responseDataErr == nil { + jsonString := string(jsonData) + if logger.PrintTrace { + logger.Trace(jsonString) + } + + // write response + w.Header().Set("Content-Type", "application/json") + _, responseErr := w.Write([]byte(jsonString)) + if responseErr != nil { + logger.Debug(responseErr) + } + } else { + logger.Debug(responseDataErr) + } +} + +func RouteScripts(router *mux.Router, opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) { + // POST scripts/ + // create new script telemetry record + // https://stackoverflow.com/a/26212073 + router.HandleFunc("/api/v1/scripts/", func(w http.ResponseWriter, r *http.Request) { + // parse given json data into a new record + logrec := persistence.ScriptTelemetryRecordV1{} + decodeErr := json.NewDecoder(r.Body).Decode(&logrec) + if decodeErr != nil { + logger.Debug(decodeErr) + return + } + + err := logrec.Validate() + if err != nil { + // log error + logrec.PrintRecordInfo(logger, fmt.Sprintf("[ {r}%s{!} ]", err.Error())) + // respond with error + w.WriteHeader(http.StatusBadRequest) + respondError(err, w, logger) + } else { + // now write to db + _, dbWriteErr := dbConn.WriteScriptTelemetryV1(&logrec, logger) + if dbWriteErr != nil { + logger.Debug(dbWriteErr) + logrec.PrintRecordInfo(logger, fmt.Sprintf("[ {r}%s{!} ]", dbWriteErr)) + } else { + logrec.PrintRecordInfo(logger, OkMessage) + } + // respond with the created data + dumpScriptAndRespond(logrec, w, logger) + } + + }).Methods("POST") + + router.HandleFunc("/api/v2/scripts/", func(w http.ResponseWriter, r *http.Request) { + // parse given json data into a new record + logrec := persistence.ScriptTelemetryRecordV2{} + decodeErr := json.NewDecoder(r.Body).Decode(&logrec) + if decodeErr != nil { + logger.Debug(decodeErr) + return + } + + // validate + err := logrec.Validate() + if err != nil { + // log error + logrec.PrintRecordInfo(logger, fmt.Sprintf("[ {r}%s{!} ]", err.Error())) + // respond with error + w.WriteHeader(http.StatusBadRequest) + respondError(err, w, logger) + } else { + // now write to db + _, dbWriteErr := dbConn.WriteScriptTelemetryV2(&logrec, logger) + if dbWriteErr != nil { + logger.Debug(dbWriteErr) + logrec.PrintRecordInfo(logger, fmt.Sprintf("[ {r}%s{!} ]", dbWriteErr)) + } else { + logrec.PrintRecordInfo(logger, OkMessage) + } + // respond with the created data + dumpScriptAndRespond(logrec, w, logger) + } + + }).Methods("POST") + + // GET scripts/ + // get recorded telemetry record + router.HandleFunc("/api/v1/scripts/", func(w http.ResponseWriter, r *http.Request) { + // Parse query parameters + limit := 100 // Default limit + offset := 0 // Default offset + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 { + limit = l + } + } + if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" { + if o, err := strconv.Atoi(offsetStr); err == nil && o >= 0 { + offset = o + } + } + + // Parse search query if provided + var searchQuery map[string]interface{} + if searchStr := r.URL.Query().Get("q"); searchStr != "" { + if err := json.Unmarshal([]byte(searchStr), &searchQuery); err != nil { + w.WriteHeader(http.StatusBadRequest) + respondError(fmt.Errorf("invalid search query: %v", err), w, logger) + return + } + } + + // Get records + var records []persistence.ScriptTelemetryRecordV1 + var err error + if searchQuery != nil { + records, err = dbConn.SearchScriptTelemetryV1(searchQuery, limit, offset, logger) + } else { + records, err = dbConn.ReadScriptTelemetryV1(limit, offset, logger) + } + + if err != nil { + logger.Debug(err) + w.WriteHeader(http.StatusInternalServerError) + respondError(err, w, logger) + return + } + + // Write response + w.Header().Set("Content-Type", "application/json") + jsonData, err := json.Marshal(records) + if err != nil { + logger.Debug(err) + w.WriteHeader(http.StatusInternalServerError) + respondError(err, w, logger) + return + } + + if _, err := w.Write(jsonData); err != nil { + logger.Debug(err) + } + }).Methods("GET") + + router.HandleFunc("/api/v2/scripts/", func(w http.ResponseWriter, r *http.Request) { + // Parse query parameters + limit := 100 // Default limit + offset := 0 // Default offset + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 { + limit = l + } + } + if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" { + if o, err := strconv.Atoi(offsetStr); err == nil && o >= 0 { + offset = o + } + } + + // Parse search query if provided + var searchQuery map[string]interface{} + if searchStr := r.URL.Query().Get("q"); searchStr != "" { + if err := json.Unmarshal([]byte(searchStr), &searchQuery); err != nil { + w.WriteHeader(http.StatusBadRequest) + respondError(fmt.Errorf("invalid search query: %v", err), w, logger) + return + } + } + + // Get records + var records []persistence.ScriptTelemetryRecordV2 + var err error + if searchQuery != nil { + records, err = dbConn.SearchScriptTelemetryV2(searchQuery, limit, offset, logger) + } else { + records, err = dbConn.ReadScriptTelemetryV2(limit, offset, logger) + } + + if err != nil { + logger.Debug(err) + w.WriteHeader(http.StatusInternalServerError) + respondError(err, w, logger) + return + } + + // Write response + w.Header().Set("Content-Type", "application/json") + jsonData, err := json.Marshal(records) + if err != nil { + logger.Debug(err) + w.WriteHeader(http.StatusInternalServerError) + respondError(err, w, logger) + return + } + + if _, err := w.Write(jsonData); err != nil { + logger.Debug(err) + } + }).Methods("GET") +} diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..bb8ef30 --- /dev/null +++ b/server/server.go @@ -0,0 +1,57 @@ +package server + +import ( + "fmt" + "net/http" + + "pyrevittelemetryserver/cli" + "pyrevittelemetryserver/persistence" + + "github.com/gofrs/uuid" + "github.com/gorilla/mux" +) + +var ServerId uuid.UUID + +func Start(opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) { + //create new id for ths server instance + ServerId = uuid.Must(uuid.NewV4()) + + // http server router + router := mux.NewRouter().StrictSlash(true) + + // create routes + // create scripts routes + if opts.ScriptsTable != "" { + RouteScripts(router, opts, dbConn, logger) + } + // create events routes + if opts.EventsTable != "" { + RouteEvents(router, opts, dbConn, logger) + } + + RouteStatus(router, opts, dbConn, logger) + + // start listening now + logger.Print(fmt.Sprintf("Server listening on %d...", opts.Port)) + if opts.Https { + logger.Fatal( + http.ListenAndServeTLS( + fmt.Sprintf(":%d", opts.Port), + fmt.Sprintf("%s.crt", opts.ExeName), + fmt.Sprintf("%s.key", opts.ExeName), + router, + )) + + } else { + logger.Fatal( + http.ListenAndServe( + fmt.Sprintf(":%d", opts.Port), + router, + )) + } +} + +func GetStatus() string { + return "pass" // "pass", "fail" or "warn" +} diff --git a/server/status.go b/server/status.go new file mode 100644 index 0000000..91498c9 --- /dev/null +++ b/server/status.go @@ -0,0 +1,61 @@ +// https://inadarei.github.io/rfc-healthcheck/ +package server + +import ( + "encoding/json" + "net/http" + + "pyrevittelemetryserver/cli" + "pyrevittelemetryserver/persistence" + + "github.com/gorilla/mux" +) + +type ServerStatus struct { + Status string `json:"status"` + Version string `json:"version"` + Output string `json:"output"` + ServiceId string `json:"serviceid"` + Checks map[string]persistence.ConnectionStatus `json:"checks"` +} + +func prepareAndReportStatus(w http.ResponseWriter, opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) { + // create status report data + jsonData, responseDataErr := json.Marshal( + ServerStatus{ + Status: GetStatus(), + Version: opts.Version, + ServiceId: ServerId.String(), + Checks: map[string]persistence.ConnectionStatus{ + string(dbConn.GetType()): dbConn.GetStatus(logger), + }, + }) + if responseDataErr == nil { + jsonString := string(jsonData) + if logger.PrintTrace { + logger.Trace(jsonString) + } + + // write response + w.Header().Set("Content-Type", "application/health+json") + _, responseErr := w.Write([]byte(jsonString)) + if responseErr != nil { + logger.Debug(responseErr) + } + } else { + logger.Debug(responseDataErr) + } + +} + +func RouteStatus(router *mux.Router, opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) { + // GET scripts/ + // get recorded telemetry record + router.HandleFunc("/api/v1/status", func(w http.ResponseWriter, r *http.Request) { + prepareAndReportStatus(w, opts, dbConn, logger) + }).Methods("GET") + + router.HandleFunc("/api/v2/status", func(w http.ResponseWriter, r *http.Request) { + prepareAndReportStatus(w, opts, dbConn, logger) + }).Methods("GET") +} From 3eae1d278aa4dd57fc362e79a6afd87da9ea1196 Mon Sep 17 00:00:00 2001 From: fishonamos Date: Fri, 30 May 2025 14:05:34 +0100 Subject: [PATCH 02/14] go tidy and update main --- go.mod | 38 +++++++++++++++++++++++++ go.sum | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 35 +++++++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..85042af --- /dev/null +++ b/go.mod @@ -0,0 +1,38 @@ +module pyrevittelemetryserver + +go 1.23.0 + +toolchain go1.24.2 + +require ( + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d + github.com/denisenkom/go-mssqldb v0.11.0 + github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 + github.com/go-sql-driver/mysql v1.6.0 + github.com/gofrs/uuid v4.3.1+incompatible + github.com/gorilla/mux v1.8.0 + github.com/lib/pq v1.10.3 + github.com/mattn/go-sqlite3 v1.14.8 + github.com/pkg/errors v0.9.1 + github.com/satori/go.uuid v1.2.0 + go.mongodb.org/mongo-driver v1.11.1 + pkg.re/essentialkaos/ek.v10 v12.32.0+incompatible +) + +require ( + // Indirect dependencies + github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/klauspost/compress v1.13.6 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.1 // indirect + github.com/xdg-go/stringprep v1.0.3 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + golang.org/x/crypto v0.35.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/text v0.22.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + pkg.re/essentialkaos/check.v1 v1.0.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..2ac5ef4 --- /dev/null +++ b/go.sum @@ -0,0 +1,87 @@ +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= +github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI= +github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg= +github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU= +github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8= +go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +pkg.re/essentialkaos/check.v1 v1.0.0 h1:2V++mhtm9yHqvW7gtXqcU1D+98vTICGnXmaZloLsZVY= +pkg.re/essentialkaos/check.v1 v1.0.0/go.mod h1:B7CoMnGFRnruw7X2Z45kWNvoCW+5OhUsLUm1EBM1aJs= +pkg.re/essentialkaos/ek.v10 v12.32.0+incompatible h1:MSnAZgf9WxV/kBpmPpD7md3ajOSXrugvbGIqRd9AWTI= +pkg.re/essentialkaos/ek.v10 v12.32.0+incompatible/go.mod h1:QhFbmORYfukHQjR05vj21bPWmCRLYlSy0tNGGCQgGnI= diff --git a/main.go b/main.go new file mode 100644 index 0000000..c32e553 --- /dev/null +++ b/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + + "pyrevittelemetryserver/cli" + "pyrevittelemetryserver/persistence" + "pyrevittelemetryserver/server" +) + +func main() { + // process command line arguments + options := cli.NewOptions() + + // Then log options if requested + logger := cli.NewLogger(options) + logger.Trace(options) + for key, value := range *options.Opts { + logger.Debug(fmt.Sprintf("%s=%v", key, value)) + } + + dbcfg, cErr := persistence.NewConfig(options) + if cErr != nil { + panic(cErr) + } + + // request a db connection to read and write + dbConn, nErr := persistence.NewConnection(dbcfg) + if nErr != nil { + panic(nErr) + } + + // ask server to start and pass db writer interface + server.Start(options, dbConn, logger) +} From d8318920922248bcbaf4d6fb21e77738ba7ad238 Mon Sep 17 00:00:00 2001 From: fishonamos Date: Fri, 30 May 2025 16:30:48 +0100 Subject: [PATCH 03/14] add support os env variables and refac cli --- cli/args.go | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++ cli/logger.go | 39 ++++++++++++++++ cli/usage.go | 51 +++++++++++++++++++++ 3 files changed, 211 insertions(+) create mode 100644 cli/args.go create mode 100644 cli/logger.go create mode 100644 cli/usage.go diff --git a/cli/args.go b/cli/args.go new file mode 100644 index 0000000..dc9ff6d --- /dev/null +++ b/cli/args.go @@ -0,0 +1,121 @@ +package cli + +import ( + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/docopt/docopt-go" +) + +// Environment variable support: +// PYREVT_TELEMETRY_DB_CONNSTRING +// PYREVT_TELEMETRY_SCRIPTS_TABLE +// PYREVT_TELEMETRY_EVENTS_TABLE +// PYREVT_TELEMETRY_PORT +// PYREVT_TELEMETRY_HTTPS +// PYREVT_TELEMETRY_DEBUG +// PYREVT_TELEMETRY_TRACE +// PYREVT_TELEMETRY_EXENAME +// PYREVT_TELEMETRY_VERSION + +type Options struct { + ExeName string `json:"exe_name"` + Version string `json:"version"` + Opts *docopt.Opts + ConnString string `json:"connection_string"` + ScriptsTable string `json:"script_table"` + EventsTable string `json:"events_table"` + Port int `json:"server_port"` + Https bool `json:"https"` + Debug bool `json:"debug_mode"` + Trace bool `json:"trace_mode"` +} + +func getExeName() string { + return strings.TrimSuffix( + filepath.Base(os.Args[0]), + filepath.Ext(os.Args[0]), + ) +} + +func NewOptions() *Options { + argv := os.Args[1:] + + parser := &docopt.Parser{ + HelpHandler: printHelpAndExit, + } + + opts, _ := parser.ParseArgs(help, argv, version) + + connString, _ := opts.String("") + scriptTable, _ := opts.String("--scripts") + eventTable, _ := opts.String("--events") + port, _ := opts.Int("--port") + https, _ := opts.Bool("--https") + + debug, _ := opts.Bool("--debug") + trace, _ := opts.Bool("--trace") + + // Environment variable fallback + if connString == "" { + connString = os.Getenv("PYREVT_TELEMETRY_DB_CONNSTRING") + } + if scriptTable == "" { + scriptTable = os.Getenv("PYREVT_TELEMETRY_SCRIPTS_TABLE") + } + if eventTable == "" { + eventTable = os.Getenv("PYREVT_TELEMETRY_EVENTS_TABLE") + } + if port == 0 { + if portStr := os.Getenv("PYREVT_TELEMETRY_PORT"); portStr != "" { + if p, err := strconv.Atoi(portStr); err == nil { + port = p + } + } + } + if !https { + if httpsStr := os.Getenv("PYREVT_TELEMETRY_HTTPS"); httpsStr != "" { + if httpsStr == "1" || strings.ToLower(httpsStr) == "true" { + https = true + } + } + } + if !debug { + if debugStr := os.Getenv("PYREVT_TELEMETRY_DEBUG"); debugStr != "" { + if debugStr == "1" || strings.ToLower(debugStr) == "true" { + debug = true + } + } + } + if !trace { + if traceStr := os.Getenv("PYREVT_TELEMETRY_TRACE"); traceStr != "" { + if traceStr == "1" || strings.ToLower(traceStr) == "true" { + trace = true + } + } + } + + exeName := getExeName() + if envExe := os.Getenv("PYREVT_TELEMETRY_EXENAME"); envExe != "" { + exeName = envExe + } + ver := version + if envVer := os.Getenv("PYREVT_TELEMETRY_VERSION"); envVer != "" { + ver = envVer + } + + return &Options{ + ExeName: exeName, + Version: ver, + Opts: &opts, + ConnString: connString, + ScriptsTable: scriptTable, + EventsTable: eventTable, + Port: port, + Https: https, + Debug: debug, + Trace: trace, + } +} diff --git a/cli/logger.go b/cli/logger.go new file mode 100644 index 0000000..dfab80b --- /dev/null +++ b/cli/logger.go @@ -0,0 +1,39 @@ +package cli + +import ( + "log" + + "pkg.re/essentialkaos/ek.v10/fmtc" +) + +type Logger struct { + PrintDebug bool + PrintTrace bool +} + +func NewLogger(options *Options) *Logger { + return &Logger{ + PrintDebug: options.Debug, + PrintTrace: options.Trace, + } +} + +func (m *Logger) Fatal(args ...interface{}) { + log.Fatal(args...) +} + +func (m *Logger) Debug(args ...interface{}) { + if m.PrintDebug { + log.Print(args...) + } +} + +func (m *Logger) Trace(args ...interface{}) { + if m.PrintTrace { + log.Print(args...) + } +} + +func (m *Logger) Print(args ...interface{}) { + fmtc.Println(args...) +} diff --git a/cli/usage.go b/cli/usage.go new file mode 100644 index 0000000..55c7091 --- /dev/null +++ b/cli/usage.go @@ -0,0 +1,51 @@ +package cli + +import ( + "fmt" + "os" +) + +const version string = "0.19" +const help string = `Record pyRevit usage logs to database + +Usage: + pyrevit-telemetryserver [] [--scripts=] [--events=] --port= [--https] [--debug] [--trace] + +Options: + -h --help show this screen + -V --version show version + --scripts= target table or collection for script logs + --events= target table or collection for app event logs + --port= server port number to listen on + --https secure connection, expects ./pyrevit-telemetryserver.key and ./pyrevit-telemetryserver.crt + --debug print debug info + --trace print trace info e.g. full json logs and sql queries + +Supports: + postgresql: using github.com/lib/pq + mongodb: using gopkg.in/mgo.v2 + mysql: using github.com/go-sql-driver/mysql + sqlserver: using github.com/denisenkom/go-mssqldb + sqlite3: using github.com/mattn/go-sqlite3 + +Examples: + pyrevit-telemetryserver postgres://user:pass@data.mycompany.com/mydb --scripts="pyrevitlogs" --events="appevents" --port=8080 --debug + pyrevit-telemetryserver mongodb://user:pass@localhost:27017/mydb --scripts="pyrevitlogs" --events="appevents" --port=8080 + pyrevit-telemetryserver "mysql:user:pass@tcp(localhost:3306)/tests" --scripts="pyrevitlogs" --port=8080 + pyrevit-telemetryserver sqlserver://user:pass@my-azure-db.database.windows.net?database=mydb --scripts="pyrevitlogs" --port=8080 + pyrevit-telemetryserver sqlite3:data.db --scripts="pyrevitlogs" --port=8080 +` + +var printHelpAndExit = func(err error, docoptMessage string) { + if err != nil { + // if err occured print full help + // docopt only includes usage section in its message + fmt.Fprintln(os.Stderr, help) + os.Exit(1) + } else { + // otherwise print whatever docopt says + // e.g. reporting version + fmt.Println(docoptMessage) + os.Exit(0) + } +} From ae665284314eb3c175904ab26dc942524e69c0d3 Mon Sep 17 00:00:00 2001 From: fishonamos Date: Fri, 30 May 2025 16:38:22 +0100 Subject: [PATCH 04/14] setup mongo --- persistence/config.go | 56 ++++++ persistence/connection.go | 64 ++++++ persistence/genericsql.go | 414 ++++++++++++++++++++++++++++++++++++++ persistence/models.go | 208 +++++++++++++++++++ persistence/mongo.go | 209 +++++++++++++++++++ persistence/utils.go | 35 ++++ 6 files changed, 986 insertions(+) create mode 100644 persistence/config.go create mode 100644 persistence/connection.go create mode 100644 persistence/genericsql.go create mode 100644 persistence/models.go create mode 100644 persistence/mongo.go create mode 100644 persistence/utils.go diff --git a/persistence/config.go b/persistence/config.go new file mode 100644 index 0000000..6ef8609 --- /dev/null +++ b/persistence/config.go @@ -0,0 +1,56 @@ +package persistence + +import ( + "strings" + + "pyrevittelemetryserver/cli" + + "github.com/pkg/errors" +) + +type DBBackend string + +const ( + Postgres DBBackend = "postgres" + MongoDB DBBackend = "mongodb" + MySql DBBackend = "mysql" + MSSql DBBackend = "sqlserver" + Sqlite DBBackend = "sqlite3" +) + +type Config struct { + Backend DBBackend `json:"backend"` + ConnString string `json:"connection_string"` + ScriptTarget string `json:"script_target"` + EventTarget string `json:"event_target"` +} + +func NewConfig(options *cli.Options) (*Config, error) { + backend, err := parseUri(options.ConnString) + if err != nil { + return nil, err + } + + return &Config{ + Backend: backend, + ConnString: options.ConnString, + ScriptTarget: options.ScriptsTable, + EventTarget: options.EventsTable, + }, nil +} + +func parseUri(connString string) (DBBackend, error) { + if strings.HasPrefix(connString, "postgres:") { + return Postgres, nil + } else if strings.HasPrefix(connString, "mongodb:") { + return MongoDB, nil + } else if strings.HasPrefix(connString, "mysql:") { + return MySql, nil + } else if strings.HasPrefix(connString, "sqlserver:") { + return MSSql, nil + } else if strings.HasPrefix(connString, "sqlite3:") { + return Sqlite, nil + } else { + return "", errors.New("db is not yet supported") + } +} diff --git a/persistence/connection.go b/persistence/connection.go new file mode 100644 index 0000000..6f67240 --- /dev/null +++ b/persistence/connection.go @@ -0,0 +1,64 @@ +package persistence + +import ( + "pyrevittelemetryserver/cli" +) + +type ConnectionStatus struct { + Status string `json:"status"` + Version string `json:"version"` + Output string `json:"output"` +} + +// ErroCodes +// 0: All OK +// 1: No data to write +// 2: data is available but did not get pushed under dry run +// 3: headers are required +type Result struct { + ResultCode int + Message string +} + +type DatabaseConnection struct { + Config *Config `json:"db_configs"` +} + +type Connection interface { + GetType() DBBackend + GetVersion(*cli.Logger) string + GetStatus(*cli.Logger) ConnectionStatus + WriteScriptTelemetryV1(*ScriptTelemetryRecordV1, *cli.Logger) (*Result, error) + WriteScriptTelemetryV2(*ScriptTelemetryRecordV2, *cli.Logger) (*Result, error) + WriteEventTelemetryV2(*EventTelemetryRecordV2, *cli.Logger) (*Result, error) + + // Read methods for retrieving telemetry data + ReadScriptTelemetryV1(limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV1, error) + ReadScriptTelemetryV2(limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV2, error) + ReadEventTelemetryV2(limit int, offset int, logger *cli.Logger) ([]EventTelemetryRecordV2, error) + + // Search methods for filtering telemetry data + SearchScriptTelemetryV1(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV1, error) + SearchScriptTelemetryV2(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV2, error) + SearchEventTelemetryV2(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]EventTelemetryRecordV2, error) +} + +func NewConnection(dbcfg *Config) (Connection, error) { + w := DatabaseConnection{ + Config: dbcfg, + } + if dbcfg.Backend == Postgres { + return GenericSQLConnection{w}, nil + } else if dbcfg.Backend == MongoDB { + return MongoDBConnection{w}, nil + } else if dbcfg.Backend == MySql { + return GenericSQLConnection{w}, nil + } else if dbcfg.Backend == MSSql { + return GenericSQLConnection{w}, nil + } else if dbcfg.Backend == Sqlite { + return GenericSQLConnection{w}, nil + } + // ... other writers + + panic("should not get here") +} diff --git a/persistence/genericsql.go b/persistence/genericsql.go new file mode 100644 index 0000000..579eb51 --- /dev/null +++ b/persistence/genericsql.go @@ -0,0 +1,414 @@ +package persistence + +import ( + "database/sql" + "encoding/json" + "fmt" + "log" + "reflect" + "regexp" + "strconv" + "strings" + + "pyrevittelemetryserver/cli" + + _ "github.com/denisenkom/go-mssqldb" + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" + _ "github.com/mattn/go-sqlite3" + + uuid "github.com/satori/go.uuid" +) + +type GenericSQLConnection struct { + DatabaseConnection +} + +func (w GenericSQLConnection) GetType() DBBackend { + return w.Config.Backend +} + +func (w GenericSQLConnection) GetVersion(logger *cli.Logger) string { + db, err := openConnection(w.Config.Backend, w.Config.ConnString, logger) + if err != nil { + logger.Debug("error opening connection") + return "" + } + defer db.Close() + + var version string + err = db.QueryRow("select version()").Scan(&version) + if err != nil { + err = db.QueryRow("select @@version").Scan(&version) + if err != nil { + log.Fatal(err) + } + } + return version +} + +func (w GenericSQLConnection) GetStatus(logger *cli.Logger) ConnectionStatus { + return ConnectionStatus{ + Status: "pass", + Version: w.GetVersion(logger), + } +} + +func (w GenericSQLConnection) WriteScriptTelemetryV1(logrec *ScriptTelemetryRecordV1, logger *cli.Logger) (*Result, error) { + // generate generic sql insert query + logger.Debug("generating query") + query, qErr := generateScriptInsertQueryV1(w.Config.ScriptTarget, logrec, logger) + if qErr != nil { + return nil, qErr + } + + return commitSQL(w.Config.Backend, w.Config.ConnString, query, logger) +} + +func (w GenericSQLConnection) WriteScriptTelemetryV2(logrec *ScriptTelemetryRecordV2, logger *cli.Logger) (*Result, error) { + // generate generic sql insert query + logger.Debug("generating query") + query, qErr := generateScriptInsertQueryV2(w.Config.ScriptTarget, logrec, logger) + if qErr != nil { + return nil, qErr + } + + return commitSQL(w.Config.Backend, w.Config.ConnString, query, logger) +} + +func (w GenericSQLConnection) WriteEventTelemetryV2(logrec *EventTelemetryRecordV2, logger *cli.Logger) (*Result, error) { + // generate generic sql insert query + logger.Debug("generating query") + query, qErr := generateEventInsertQueryV2(w.Config.EventTarget, logrec, logger) + if qErr != nil { + return nil, qErr + } + + return commitSQL(w.Config.Backend, w.Config.ConnString, query, logger) +} + +func commitSQL(backend DBBackend, connStr string, query string, logger *cli.Logger) (*Result, error) { + // open connection + db, err := openConnection(backend, connStr, logger) + if err != nil { + logger.Debug("error opening connection") + return nil, err + } + defer db.Close() + + // start transaction + logger.Debug("opening transaction") + tx, beginErr := db.Begin() + if beginErr != nil { + logger.Debug("error opening transaction") + return nil, beginErr + } + defer tx.Rollback() + + // run the insert query + logger.Debug("executing insert query") + _, eErr := db.Exec(query) + if eErr != nil { + return nil, eErr + } + + // commit transaction + logger.Debug("commiting transaction") + txnErr := tx.Commit() + if txnErr != nil { + return nil, txnErr + } + + logger.Debug("preparing report") + return &Result{ + Message: "successfully inserted usage record", + }, nil +} + +func openConnection(backend DBBackend, connStr string, logger *cli.Logger) (*sql.DB, error) { + // open connection + logger.Debug(fmt.Sprintf("opening %s connection", backend)) + cleanConnStr := connStr + if backend == Sqlite || backend == MySql { + cleanConnStr = strings.Replace(connStr, string(backend)+":", "", 1) + } + return sql.Open(string(backend), cleanConnStr) +} + +func generateScriptInsertQueryV1(table string, logrec *ScriptTelemetryRecordV1, logger *cli.Logger) (string, error) { + // read csv file and build sql insert query + var querystr strings.Builder + + logger.Debug("generating insert query with-out headers") + querystr.WriteString(fmt.Sprintf("INSERT INTO %s values ", table)) + + // build sql data info + logger.Debug("building insert query for data") + datalines := make([]string, 0) + + cresults, merr := json.Marshal(logrec.CommandResults) + if merr != nil { + logger.Debug("error logging command results") + } + + // create record based on schema + var record []string + + // generate record id, panic if error + recordId := uuid.NewV4() + + re := regexp.MustCompile(`(\d+:\d+:\d+)`) + record = []string{ + recordId.String(), + logrec.Date, + re.FindString(logrec.Time), + logrec.UserName, + logrec.RevitVersion, + logrec.RevitBuild, + logrec.SessionId, + logrec.PyRevitVersion, + strconv.FormatBool(logrec.IsDebugMode), + strconv.FormatBool(logrec.IsConfigMode), + logrec.CommandName, + logrec.BundleName, + logrec.ExtensionName, + logrec.CommandUniqueName, + strconv.Itoa(logrec.ResultCode), + string(cresults), + logrec.ScriptPath, + logrec.TraceInfo.EngineInfo.Version, + logrec.TraceInfo.IronPythonTraceDump, + logrec.TraceInfo.CLRTraceDump, + } + + datalines = append(datalines, ToSql(&record, true)) + + // add csv records to query string + all_datalines := strings.Join(datalines, ", ") + logger.Trace(all_datalines) + querystr.WriteString(all_datalines) + querystr.WriteString(";\n") + logger.Debug("building query completed") + + // execute query + full_query := querystr.String() + logger.Trace(full_query) + return full_query, nil +} + +func generateScriptInsertQueryV2(table string, logrec *ScriptTelemetryRecordV2, logger *cli.Logger) (string, error) { + // read csv file and build sql insert query + var querystr strings.Builder + + logger.Debug("generating insert query with-out headers") + querystr.WriteString(fmt.Sprintf("INSERT INTO %s values ", table)) + + // build sql data info + logger.Debug("building insert query for data") + datalines := make([]string, 0) + + // marshal json data + engineCfgs, merr := json.Marshal(logrec.TraceInfo.EngineInfo.Configs) + if merr != nil { + logger.Debug("error logging engine configs") + } + + // marshal json data + cresults, merr := json.Marshal(logrec.CommandResults) + if merr != nil { + logger.Debug("error logging command results") + } + + // create record based on schema + var record []string + + // generate record id, panic if error + recordId := uuid.NewV4() + + record = []string{ + recordId.String(), + logrec.TimeStamp, + logrec.UserName, + logrec.HostUserName, + logrec.RevitVersion, + logrec.RevitBuild, + logrec.SessionId, + logrec.PyRevitVersion, + logrec.Clone, + strconv.FormatBool(logrec.IsDebugMode), + strconv.FormatBool(logrec.IsConfigMode), + strconv.FormatBool(logrec.IsExecFromGUI), + logrec.ExecId, + logrec.ExecTimeStamp, + logrec.CommandName, + logrec.BundleName, + logrec.ExtensionName, + logrec.CommandUniqueName, + logrec.DocumentName, + logrec.DocumentPath, + strconv.Itoa(logrec.ResultCode), + string(cresults), + logrec.ScriptPath, + logrec.TraceInfo.EngineInfo.Type, + logrec.TraceInfo.EngineInfo.Version, + strings.Join(logrec.TraceInfo.EngineInfo.SysPaths, ";"), + string(engineCfgs), + logrec.TraceInfo.Message, + } + datalines = append(datalines, ToSql(&record, true)) + + // add csv records to query string + all_datalines := strings.Join(datalines, ", ") + logger.Trace(all_datalines) + querystr.WriteString(all_datalines) + querystr.WriteString(";\n") + logger.Debug("building query completed") + + // execute query + full_query := querystr.String() + logger.Trace(full_query) + return full_query, nil +} + +func generateEventInsertQueryV2(table string, logrec *EventTelemetryRecordV2, logger *cli.Logger) (string, error) { + // read csv file and build sql insert query + var querystr strings.Builder + + logger.Debug("generating insert query with-out headers") + querystr.WriteString(fmt.Sprintf("INSERT INTO %s values ", table)) + + // build sql data info + logger.Debug("building insert query for data") + datalines := make([]string, 0) + + // marshal json data + cresults, merr := json.Marshal(logrec.EventArgs) + if merr != nil { + logger.Debug("error logging command results") + } + + // create record based on schema + var record []string + + // generate record id, panic if error + recordId := uuid.NewV4() + + record = []string{ + recordId.String(), + logrec.TimeStamp, + logrec.HandlerId, + logrec.EventType, + string(cresults), + logrec.UserName, + logrec.HostUserName, + logrec.RevitVersion, + logrec.RevitBuild, + strconv.FormatBool(logrec.Cancellable), + strconv.FormatBool(logrec.Cancelled), + strconv.Itoa(logrec.DocumentId), + logrec.DocumentType, + logrec.DocumentTemplate, + logrec.DocumentName, + logrec.DocumentPath, + logrec.ProjectNumber, + logrec.ProjectName, + } + datalines = append(datalines, ToSql(&record, true)) + + // add csv records to query string + all_datalines := strings.Join(datalines, ", ") + logger.Trace(all_datalines) + querystr.WriteString(all_datalines) + querystr.WriteString(";\n") + logger.Debug("building query completed") + + // execute query + full_query := querystr.String() + logger.Trace(full_query) + return full_query, nil +} + +// Read methods for SQL databases +func (w GenericSQLConnection) ReadScriptTelemetryV1(limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV1, error) { + return readSQL[ScriptTelemetryRecordV1](w.Config.Backend, w.Config.ConnString, w.Config.ScriptTarget, nil, limit, offset, logger) +} + +func (w GenericSQLConnection) ReadScriptTelemetryV2(limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV2, error) { + return readSQL[ScriptTelemetryRecordV2](w.Config.Backend, w.Config.ConnString, w.Config.ScriptTarget, nil, limit, offset, logger) +} + +func (w GenericSQLConnection) ReadEventTelemetryV2(limit int, offset int, logger *cli.Logger) ([]EventTelemetryRecordV2, error) { + return readSQL[EventTelemetryRecordV2](w.Config.Backend, w.Config.ConnString, w.Config.EventTarget, nil, limit, offset, logger) +} + +// Search methods for SQL databases +func (w GenericSQLConnection) SearchScriptTelemetryV1(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV1, error) { + return readSQL[ScriptTelemetryRecordV1](w.Config.Backend, w.Config.ConnString, w.Config.ScriptTarget, query, limit, offset, logger) +} + +func (w GenericSQLConnection) SearchScriptTelemetryV2(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV2, error) { + return readSQL[ScriptTelemetryRecordV2](w.Config.Backend, w.Config.ConnString, w.Config.ScriptTarget, query, limit, offset, logger) +} + +func (w GenericSQLConnection) SearchEventTelemetryV2(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]EventTelemetryRecordV2, error) { + return readSQL[EventTelemetryRecordV2](w.Config.Backend, w.Config.ConnString, w.Config.EventTarget, query, limit, offset, logger) +} + +// Generic read function for SQL databases +func readSQL[T any](backend DBBackend, connStr string, table string, query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]T, error) { + // Open connection + db, err := openConnection(backend, connStr, logger) + if err != nil { + return nil, err + } + defer db.Close() + + // Build query + var sqlQuery strings.Builder + sqlQuery.WriteString(fmt.Sprintf("SELECT * FROM %s", table)) + + // Add WHERE clause if query parameters provided + args := make([]interface{}, 0) + if query != nil { + sqlQuery.WriteString(" WHERE ") + conditions := make([]string, 0) + for key, value := range query { + conditions = append(conditions, fmt.Sprintf("%s = ?", key)) + args = append(args, value) + } + sqlQuery.WriteString(strings.Join(conditions, " AND ")) + } + + // Add ORDER BY and LIMIT/OFFSET + sqlQuery.WriteString(" ORDER BY timestamp DESC") + sqlQuery.WriteString(fmt.Sprintf(" LIMIT %d OFFSET %d", limit, offset)) + + // Execute query + rows, err := db.Query(sqlQuery.String(), args...) + if err != nil { + return nil, err + } + defer rows.Close() + + // Scan results + var results []T + for rows.Next() { + var record T + // Use reflection to scan into struct fields + val := reflect.ValueOf(&record).Elem() + fields := make([]interface{}, val.NumField()) + for i := 0; i < val.NumField(); i++ { + fields[i] = val.Field(i).Addr().Interface() + } + if err := rows.Scan(fields...); err != nil { + return nil, err + } + results = append(results, record) + } + + if err = rows.Err(); err != nil { + return nil, err + } + + return results, nil +} diff --git a/persistence/models.go b/persistence/models.go new file mode 100644 index 0000000..0cc6a0c --- /dev/null +++ b/persistence/models.go @@ -0,0 +1,208 @@ +package persistence + +import ( + "fmt" + + "pyrevittelemetryserver/cli" + + "github.com/asaskevich/govalidator" +) + +// v1.0 +type EngineInfoV1 struct { + Version string `json:"version" bson:"version" valid:"-"` + SysPaths []string `json:"syspath" bson:"syspath" valid:"-"` +} + +type TraceInfoV1 struct { + EngineInfo EngineInfoV1 `json:"engine" bson:"engine" valid:"-"` + IronPythonTraceDump string `json:"ipy" bson:"ipy" valid:"-"` + CLRTraceDump string `json:"clr" bson:"clr" valid:"-"` +} + +type ScriptTelemetryRecordV1 struct { + Date string `json:"date" bson:"date" valid:"-"` + Time string `json:"time" bson:"time" valid:"-"` + UserName string `json:"username" bson:"username" valid:"-"` + RevitVersion string `json:"revit" bson:"revit" valid:"numeric~Invalid revit version"` + RevitBuild string `json:"revitbuild" bson:"revitbuild" valid:"matches(\\d{8}_\\d{4}\\(x\\d{2}\\))~Invalid revit build number"` + SessionId string `json:"sessionid" bson:"sessionid" valid:"uuidv4~Invalid session id"` + PyRevitVersion string `json:"pyrevit" bson:"pyrevit" valid:"-"` + IsDebugMode bool `json:"debug" bson:"debug"` + IsConfigMode bool `json:"config" bson:"config"` + CommandName string `json:"commandname" bson:"commandname" valid:"-"` + CommandUniqueName string `json:"commanduniquename" bson:"commanduniquename" valid:"-"` + BundleName string `json:"commandbundle" bson:"commandbundle" valid:"-"` + ExtensionName string `json:"commandextension" bson:"commandextension" valid:"-"` + ResultCode int `json:"resultcode" bson:"resultcode" valid:"numeric~Invalid result code"` + CommandResults map[string]string `json:"commandresults" bson:"commandresults" valid:"-"` + ScriptPath string `json:"scriptpath" bson:"scriptpath" valid:"-"` + TraceInfo TraceInfoV1 `json:"trace" bson:"trace"` +} + +func (logrec ScriptTelemetryRecordV1) PrintRecordInfo(logger *cli.Logger, message string) { + logger.Print(fmt.Sprintf( + "%s %s-%s %q @ %s:%s [%s.%s] code=%d info=%v", + message, + logrec.Date, + logrec.Time, + logrec.UserName, + logrec.RevitBuild, + logrec.TraceInfo.EngineInfo.Version, + logrec.ExtensionName, + logrec.CommandName, + logrec.ResultCode, + logrec.CommandResults, + )) +} + +func (logrec ScriptTelemetryRecordV1) Validate() error { + // govalidator.SetFieldsRequiredByDefault(true) + + // validate now + _, err := govalidator.ValidateStruct(logrec) + return err +} + +// v2.0 +type EngineInfoV2 struct { + Type string `json:"type" bson:"type" valid:"engine~Invalid executor engine type"` + Version string `json:"version" bson:"version" valid:"-"` + SysPaths []string `json:"syspath" bson:"syspath" valid:"-"` + Configs map[string]interface{} `json:"configs" bson:"configs" valid:"-"` +} + +type TraceInfoV2 struct { + EngineInfo EngineInfoV2 `json:"engine" bson:"engine"` + Message string `json:"message" bson:"message" valid:"-"` +} + +type RecordMetaV2 struct { + SchemaVersion string `json:"schema" bson:"schema" valid:"schema~Invalid schema version"` +} + +type ScriptTelemetryRecordV2 struct { + RecordMeta RecordMetaV2 `json:"meta" bson:"meta"` + TimeStamp string `json:"timestamp" bson:"timestamp" valid:"rfc3339~Invalid timestamp"` + UserName string `json:"username" bson:"username" valid:"-"` + HostUserName string `json:"host_user" bson:"host_user" valid:"-"` + RevitVersion string `json:"revit" bson:"revit" valid:"numeric~Invalid revit version"` + RevitBuild string `json:"revitbuild" bson:"revitbuild" valid:"matches(\\d{8}_\\d{4}\\(x\\d{2}\\))~Invalid revit build number"` + SessionId string `json:"sessionid" bson:"sessionid" valid:"uuidv4~Invalid session id"` + PyRevitVersion string `json:"pyrevit" bson:"pyrevit" valid:"-"` + Clone string `json:"clone" bson:"clone" valid:"-"` + IsDebugMode bool `json:"debug" bson:"debug"` + IsConfigMode bool `json:"config" bson:"config"` + IsExecFromGUI bool `json:"from_gui" bson:"from_gui"` + ExecId string `json:"exec_id" bson:"exec_id" valid:"-"` + ExecTimeStamp string `json:"exec_timestamp" bson:"exec_timestamp" valid:"-"` + CommandName string `json:"commandname" bson:"commandname" valid:"-"` + CommandUniqueName string `json:"commanduniquename" bson:"commanduniquename" valid:"-"` + BundleName string `json:"commandbundle" bson:"commandbundle" valid:"-"` + ExtensionName string `json:"commandextension" bson:"commandextension" valid:"-"` + DocumentName string `json:"docname" bson:"docname" valid:"-"` + DocumentPath string `json:"docpath" bson:"docpath" valid:"-"` + ResultCode int `json:"resultcode" bson:"resultcode" valid:"numeric~Invalid result code"` + CommandResults map[string]interface{} `json:"commandresults" bson:"commandresults" valid:"-"` + ScriptPath string `json:"scriptpath" bson:"scriptpath" valid:"-"` + TraceInfo TraceInfoV2 `json:"trace" bson:"trace"` +} + +func (logrec ScriptTelemetryRecordV2) PrintRecordInfo(logger *cli.Logger, message string) { + logger.Print(fmt.Sprintf( + "%s %s %q %s:%s (%s) [%s.%s] code=%d info=%v", + message, + logrec.TimeStamp, + logrec.UserName, + logrec.RevitBuild, + logrec.TraceInfo.EngineInfo.Version, + logrec.TraceInfo.EngineInfo.Type, + logrec.ExtensionName, + logrec.CommandName, + logrec.ResultCode, + logrec.CommandResults, + )) +} + +func (logrec ScriptTelemetryRecordV2) Validate() error { + // govalidator.SetFieldsRequiredByDefault(true) + + // custom validators + govalidator.TagMap["schema"] = govalidator.Validator(func(str string) bool { + return str == "2.0" + }) + + govalidator.TagMap["engine"] = govalidator.Validator(func(str string) bool { + switch str { + case + "unknown", + "ironpython", + "cpython", + "csharp", + "invoke", + "visualbasic", + "ironruby", + "dynamobim", + "grasshopper", + "content", + "hyperlink": + return true + } + return false + }) + + // validate now + _, err := govalidator.ValidateStruct(logrec) + return err +} + +// introduced with api v2 +type EventTelemetryRecordV2 struct { + RecordMeta RecordMetaV2 `json:"meta" bson:"meta"` + TimeStamp string `json:"timestamp" bson:"timestamp" valid:"rfc3339~Invalid timestamp"` + HandlerId string `json:"handler_id" bson:"handler_id" valid:"-"` + EventType string `json:"type" bson:"type" valid:"-"` + EventArgs map[string]interface{} `json:"args" bson:"args" valid:"-"` + UserName string `json:"username" bson:"username" valid:"-"` + HostUserName string `json:"host_user" bson:"host_user" valid:"-"` + RevitVersion string `json:"revit" bson:"revit" valid:"numeric~Invalid revit version"` + RevitBuild string `json:"revitbuild" bson:"revitbuild" valid:"matches(\\d{8}_\\d{4}\\(x\\d{2}\\))~Invalid revit build number"` + + // general + Cancellable bool `json:"cancellable" bson:"cancellable"` + Cancelled bool `json:"cancelled" bson:"cancelled"` + DocumentId int `json:"docid" bson:"docid" valid:"-"` + DocumentType string `json:"doctype" bson:"doctype" valid:"-"` + DocumentTemplate string `json:"doctemplate" bson:"doctemplate" valid:"-"` + DocumentName string `json:"docname" bson:"docname" valid:"-"` + DocumentPath string `json:"docpath" bson:"docpath" valid:"-"` + ProjectNumber string `json:"projectnum" bson:"projectnum" valid:"-"` + ProjectName string `json:"projectname" bson:"projectname" valid:"-"` +} + +func (logrec EventTelemetryRecordV2) PrintRecordInfo(logger *cli.Logger, message string) { + if logrec.RecordMeta.SchemaVersion == "2.0" { + logger.Print(fmt.Sprintf( + "%s %s [%s] %q @ %s doc=%q @ %s", + message, + logrec.TimeStamp, + logrec.EventType, + logrec.HostUserName, + logrec.RevitBuild, + logrec.DocumentName, + logrec.DocumentPath, + )) + } +} + +func (logrec EventTelemetryRecordV2) Validate() error { + // govalidator.SetFieldsRequiredByDefault(true) + + // custom validators + govalidator.TagMap["schema"] = govalidator.Validator(func(str string) bool { + return str == "2.0" + }) + + _, err := govalidator.ValidateStruct(logrec) + return err +} diff --git a/persistence/mongo.go b/persistence/mongo.go new file mode 100644 index 0000000..16910ba --- /dev/null +++ b/persistence/mongo.go @@ -0,0 +1,209 @@ +package persistence + +import ( + "context" + "fmt" + "pyrevittelemetryserver/cli" + "time" + + _ "github.com/lib/pq" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readpref" + "go.mongodb.org/mongo-driver/x/mongo/driver/connstring" +) + +type MongoDBConnection struct { + DatabaseConnection +} + +func (w MongoDBConnection) GetType() DBBackend { + return w.Config.Backend +} + +func (w MongoDBConnection) GetVersion(logger *cli.Logger) string { + // parse and grab database name from uri + logger.Debug("grabbing db name from connection string") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + logger.Debug("opening mongodb session") + client, err := mongo.Connect(ctx, options.Client().ApplyURI(w.Config.ConnString)) + + defer func() { + if err = client.Disconnect(ctx); err != nil { + panic(err) + } + }() + + ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + pErr := client.Ping(ctx, readpref.Primary()) + + if pErr != nil { + return "" + } + + // get version from admin DB + logger.Debug("getting mongodb version") + var commandResult bson.M + command := bson.D{{"buildInfo", 1}} + vErr := client.Database("admin").RunCommand(ctx, command).Decode(&commandResult) + + if vErr != nil { + return "" + } + + // parse version field to get version information + ver := fmt.Sprintf("%+v", commandResult["version"]) + return ver +} + +func (w MongoDBConnection) GetStatus(logger *cli.Logger) ConnectionStatus { + return ConnectionStatus{ + Status: "pass", + Version: w.GetVersion(logger), + } +} + +func (w MongoDBConnection) WriteScriptTelemetryV1(logrec *ScriptTelemetryRecordV1, logger *cli.Logger) (*Result, error) { + return commitMongo(w.Config.ConnString, w.Config.ScriptTarget, logrec, logger) +} + +func (w MongoDBConnection) WriteScriptTelemetryV2(logrec *ScriptTelemetryRecordV2, logger *cli.Logger) (*Result, error) { + return commitMongo(w.Config.ConnString, w.Config.ScriptTarget, logrec, logger) +} + +func (w MongoDBConnection) WriteEventTelemetryV2(logrec *EventTelemetryRecordV2, logger *cli.Logger) (*Result, error) { + return commitMongo(w.Config.ConnString, w.Config.EventTarget, logrec, logger) +} + +func commitMongo(connStr string, targetCollection string, logrec interface{}, logger *cli.Logger) (*Result, error) { + // parse and grab database name from uri + logger.Debug("check connection string") + connStringInfo, err := connstring.ParseAndValidate(connStr) + + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + logger.Debug("opening mongodb session using connection string") + client, err := mongo.Connect(ctx, options.Client().ApplyURI(connStr)) + + if err != nil { + return nil, err + } + + logger.Trace(client) + + logger.Debug("getting target collection") + // db := session.DB(dialinfo.Database) + db := client.Database(connStringInfo.Database) + // c := db.C(targetCollection) + c := db.Collection(targetCollection) + logger.Trace(c) + + logger.Debug("opening bulk operation") + // bulkop := c.Bulk() + + // build sql data info + logger.Debug("building documents") + + iCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + logger.Debug("inserting new document") + _, txnErr := c.InsertOne(iCtx, logrec) + + if txnErr != nil { + return nil, txnErr + } + + // compact collection if requested + logger.Debug("preparing report") + return &Result{ + Message: "successfully inserted usage document", + }, nil +} + +// Read methods for MongoDB +func (w MongoDBConnection) ReadScriptTelemetryV1(limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV1, error) { + return readMongo[ScriptTelemetryRecordV1](w.Config.ConnString, w.Config.ScriptTarget, nil, limit, offset, logger) +} + +func (w MongoDBConnection) ReadScriptTelemetryV2(limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV2, error) { + return readMongo[ScriptTelemetryRecordV2](w.Config.ConnString, w.Config.ScriptTarget, nil, limit, offset, logger) +} + +func (w MongoDBConnection) ReadEventTelemetryV2(limit int, offset int, logger *cli.Logger) ([]EventTelemetryRecordV2, error) { + return readMongo[EventTelemetryRecordV2](w.Config.ConnString, w.Config.EventTarget, nil, limit, offset, logger) +} + +// Search methods for MongoDB +func (w MongoDBConnection) SearchScriptTelemetryV1(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV1, error) { + return readMongo[ScriptTelemetryRecordV1](w.Config.ConnString, w.Config.ScriptTarget, query, limit, offset, logger) +} + +func (w MongoDBConnection) SearchScriptTelemetryV2(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV2, error) { + return readMongo[ScriptTelemetryRecordV2](w.Config.ConnString, w.Config.ScriptTarget, query, limit, offset, logger) +} + +func (w MongoDBConnection) SearchEventTelemetryV2(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]EventTelemetryRecordV2, error) { + return readMongo[EventTelemetryRecordV2](w.Config.ConnString, w.Config.EventTarget, query, limit, offset, logger) +} + +// Generic read function for MongoDB +func readMongo[T any](connStr string, targetCollection string, query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]T, error) { + // Parse connection string + connStringInfo, err := connstring.ParseAndValidate(connStr) + if err != nil { + return nil, err + } + + // Connect to MongoDB + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + client, err := mongo.Connect(ctx, options.Client().ApplyURI(connStr)) + if err != nil { + return nil, err + } + defer client.Disconnect(ctx) + + // Get collection + db := client.Database(connStringInfo.Database) + c := db.Collection(targetCollection) + + // Prepare query options + findOptions := options.Find() + findOptions.SetLimit(int64(limit)) + findOptions.SetSkip(int64(offset)) + findOptions.SetSort(bson.D{{"timestamp", -1}}) // Sort by timestamp descending + + // Convert query map to bson.M + var filter bson.M + if query != nil { + filter = bson.M(query) + } else { + filter = bson.M{} + } + + // Execute query + cursor, err := c.Find(ctx, filter, findOptions) + if err != nil { + return nil, err + } + defer cursor.Close(ctx) + + // Decode results + var results []T + if err = cursor.All(ctx, &results); err != nil { + return nil, err + } + + return results, nil +} diff --git a/persistence/utils.go b/persistence/utils.go new file mode 100644 index 0000000..d460268 --- /dev/null +++ b/persistence/utils.go @@ -0,0 +1,35 @@ +package persistence + +import ( + "fmt" + "strings" +) + +func ToSql(values *[]string, wrap bool) string { + // wrap values in '' first + cleanedValues := make([]string, 0) + valueFormat := "%s" + if wrap { + for _, value := range *values { + if value != "" { + cleanedValues = append( + cleanedValues, + fmt.Sprintf("'%s'", strings.Replace(value, "'", "''", -1))) + } else { + cleanedValues = append(cleanedValues, "NULL") + } + } + } else { + for _, value := range *values { + cleanedValues = append( + cleanedValues, + fmt.Sprintf(valueFormat, value)) + } + } + // create the (,,,) sql value list + return fmt.Sprintf("(%s)", strings.Join(cleanedValues, ", ")) +} + +func ToMap(fields, values *[]string) map[string]string { + return make(map[string]string) +} From 7188b886cb47a47eca34270ae7848d6f2b6ce870 Mon Sep 17 00:00:00 2001 From: fishonamos Date: Fri, 30 May 2025 20:29:05 +0100 Subject: [PATCH 05/14] add documentation (readme) --- README.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3e09386..00b2f5f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,56 @@ -# telemetry-server -Telemetry Server for pyRevit +# pyRevit Telemetry Server + +## Quick Start + +```sh +# Build the server +go build -o pyrevit-telemetryserver + +# Run with environment variables +PYREVT_TELEMETRY_DB_CONNSTRING="mongodb://localhost:27017/atoma-test" \ +PYREVT_TELEMETRY_SCRIPTS_TABLE="scripts" \ +PYREVT_TELEMETRY_EVENTS_TABLE="events" \ +./pyrevit-telemetryserver --port=8080 +``` + +## Environment Variables + +| Variable | Description | Example | +|----------------------------------------|---------------------------------------------|-----------------------------------------| +| PYREVT_TELEMETRY_DB_BACKEND | Database backend (`mongo`, `postgres`, etc) | mongo | +| PYREVT_TELEMETRY_DB_CONNSTRING | Database connection string | mongodb://localhost:27017/atoma-test | +| PYREVT_TELEMETRY_SCRIPTS_TABLE | Name of the scripts table/collection | scripts | +| PYREVT_TELEMETRY_EVENTS_TABLE | Name of the events table/collection | events | +| PYREVT_TELEMETRY_PORT | Port to run the server on | 8080 | +| PYREVT_TELEMETRY_DEBUG | Enable debug logging (`true`/`false`) | true | +| PYREVT_TELEMETRY_TRACE | Enable trace logging (`true`/`false`) | false | + +> All CLI options can be set via environment variables. CLI flags override environment variables. + +## API Endpoints + +- `GET /api/v1/status` — Health check +- `GET /api/v1/scripts/` — List v1 script telemetry +- `POST /api/v1/scripts/` — Submit v1 script telemetry +- `GET /api/v2/scripts/` — List v2 script telemetry +- `POST /api/v2/scripts/` — Submit v2 script telemetry +- `GET /api/v2/events/` — List v2 event telemetry +- `POST /api/v2/events/` — Submit v2 event telemetry + +## Example: Submit Script Telemetry + +```sh +curl -X POST -H "Content-Type: application/json" \ + -d '{"date":"2024-03-30","time":"08:45:00", ... }' \ + http://localhost:8080/api/v1/scripts/ +``` + +## Docker Usage + +### Build and Run with Docker Compose + +```sh +docker-compose up --build +``` + +This will start both DB and the telemetry server. The server will be available at `http://localhost:8080`. \ No newline at end of file From 68e3db1b4b0c5716a1840f18e7e8842077af782f Mon Sep 17 00:00:00 2001 From: fishonamos Date: Mon, 2 Jun 2025 15:18:28 +0100 Subject: [PATCH 06/14] add github actions script --- .github/workflows/main.yml | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..8b3b151 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,53 @@ +name: CI/CD + +on: + workflow_dispatch: + push: + branches: [main] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # Setup Go + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.22" + + - name: Build + run: | + mkdir -p bin + go build -o bin/pyrevit-telemetryserver + + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + with: + name: bin + path: bin + + docker: + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub # TODO: uncomment if pushing + # uses: docker/login-action@v3 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build Docker image + run: docker build -t pyrevit-telemetryserver:latest . + + # - name: Push Docker image # Optional, uncomment if pushing + # run: docker push pyrevit-telemetryserver:latest From 3a09a3767fe97f1a727bbbf835a2403157ec620d Mon Sep 17 00:00:00 2001 From: fishonamos Date: Mon, 2 Jun 2025 15:19:53 +0100 Subject: [PATCH 07/14] fix yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8b3b151..ec3e0b0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,5 +49,5 @@ jobs: - name: Build Docker image run: docker build -t pyrevit-telemetryserver:latest . - # - name: Push Docker image # Optional, uncomment if pushing + # - name: Push Docker image # Will be uncommented if pushing # run: docker push pyrevit-telemetryserver:latest From 92a6663bb51fbb0a50e1054bdc8b833ca488624b Mon Sep 17 00:00:00 2001 From: fishonamos Date: Tue, 24 Jun 2025 10:59:48 +0100 Subject: [PATCH 08/14] refactor server for testing --- server/server.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/server/server.go b/server/server.go index bb8ef30..a743170 100644 --- a/server/server.go +++ b/server/server.go @@ -13,26 +13,25 @@ import ( var ServerId uuid.UUID -func Start(opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) { - //create new id for ths server instance - ServerId = uuid.Must(uuid.NewV4()) - - // http server router +func NewRouter(opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) http.Handler { router := mux.NewRouter().StrictSlash(true) - // create routes - // create scripts routes if opts.ScriptsTable != "" { RouteScripts(router, opts, dbConn, logger) } - // create events routes if opts.EventsTable != "" { RouteEvents(router, opts, dbConn, logger) } - RouteStatus(router, opts, dbConn, logger) - // start listening now + return router +} + +func Start(opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) { + ServerId = uuid.Must(uuid.NewV4()) + + router := NewRouter(opts, dbConn, logger) + logger.Print(fmt.Sprintf("Server listening on %d...", opts.Port)) if opts.Https { logger.Fatal( @@ -42,7 +41,6 @@ func Start(opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) fmt.Sprintf("%s.key", opts.ExeName), router, )) - } else { logger.Fatal( http.ListenAndServe( From 4ae08bfac1cb851bb04b4ed1e8cfc9370bfe922f Mon Sep 17 00:00:00 2001 From: fishonamos Date: Tue, 24 Jun 2025 11:25:31 +0100 Subject: [PATCH 09/14] fa- add server tests --- go.mod | 2 + server/server_test.go | 132 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 server/server_test.go diff --git a/go.mod b/go.mod index 85042af..48d7f4a 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,8 @@ require ( pkg.re/essentialkaos/ek.v10 v12.32.0+incompatible ) +require github.com/stretchr/testify v1.10.0 // indirect + require ( // Indirect dependencies github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect diff --git a/server/server_test.go b/server/server_test.go new file mode 100644 index 0000000..076fdce --- /dev/null +++ b/server/server_test.go @@ -0,0 +1,132 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "pyrevittelemetryserver/cli" + "pyrevittelemetryserver/persistence" + "strings" + "testing" +) + +type mockDB struct{ persistence.Connection } +type mockLogger struct{ cli.Logger } + +func TestStatusEndpoint(t *testing.T) { + opts := &cli.Options{ + ScriptsTable: "scripts", + EventsTable: "events", + } + db := &mockDB{} + logger := &cli.Logger{} + + router := NewRouter(opts, db, logger) + req := httptest.NewRequest("GET", "/api/v1/status", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200 OK, got %d", w.Code) + } + if w.Body.Len() == 0 { + t.Fatalf("expected non-empty body") + } +} + +func (m *mockDB) WriteScriptTelemetryV1(logrec *persistence.ScriptTelemetryRecordV1, logger *cli.Logger) (*persistence.Result, error) { + return &persistence.Result{ResultCode: 0, Message: "ok"}, nil +} +func (m *mockDB) WriteScriptTelemetryV2(logrec *persistence.ScriptTelemetryRecordV2, logger *cli.Logger) (*persistence.Result, error) { + return &persistence.Result{ResultCode: 0, Message: "ok"}, nil +} +func (m *mockDB) WriteEventTelemetryV2(logrec *persistence.EventTelemetryRecordV2, logger *cli.Logger) (*persistence.Result, error) { + return &persistence.Result{ResultCode: 0, Message: "ok"}, nil +} +func (m *mockDB) ReadScriptTelemetryV1(limit int, offset int, logger *cli.Logger) ([]persistence.ScriptTelemetryRecordV1, error) { + return []persistence.ScriptTelemetryRecordV1{}, nil +} +func (m *mockDB) ReadScriptTelemetryV2(limit int, offset int, logger *cli.Logger) ([]persistence.ScriptTelemetryRecordV2, error) { + return []persistence.ScriptTelemetryRecordV2{}, nil +} +func (m *mockDB) ReadEventTelemetryV2(limit int, offset int, logger *cli.Logger) ([]persistence.EventTelemetryRecordV2, error) { + return []persistence.EventTelemetryRecordV2{}, nil +} +func (m *mockDB) GetType() persistence.DBBackend { + return "mock" +} +func (m *mockDB) GetStatus(logger *cli.Logger) persistence.ConnectionStatus { + return persistence.ConnectionStatus{ + Status: "pass", + Version: "test", + Output: "mock", + } +} + +func TestScriptV1Endpoints(t *testing.T) { + router := NewRouter(&cli.Options{ScriptsTable: "scripts", EventsTable: "events"}, &mockDB{}, &cli.Logger{}) + // Valid payload + payload := `{"date":"2024-03-30","time":"08:45:00","username":"user","revit":"2021","revitbuild":"20240330_1234(x64)","sessionid":"b3b1a2e0-4b5c-4d2a-8c3e-2b1a2e04b5c4","pyrevit":"4.8","debug":false,"config":false,"commandname":"cmd","commanduniquename":"cmd.unique","commandbundle":"bundle","commandextension":"ext","resultcode":0,"commandresults":{},"scriptpath":"/path/to/script","trace":{"engine":{"version":"1.0","syspath":[]},"ipy":"","clr":""}}` + req := httptest.NewRequest("POST", "/api/v1/scripts/", strings.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200 OK, got %d", w.Code) + } + if w.Body.Len() == 0 { + t.Fatalf("expected non-empty body") + } + // GET + req = httptest.NewRequest("GET", "/api/v1/scripts/", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200 OK, got %d", w.Code) + } +} + +func TestScriptV2Endpoints(t *testing.T) { + router := NewRouter(&cli.Options{ScriptsTable: "scripts", EventsTable: "events"}, &mockDB{}, &cli.Logger{}) + // Valid payload + payload := `{"meta":{"schema":"2.0"},"timestamp":"2024-03-30T08:45:00Z","username":"user","host_user":"host","revit":"2021","revitbuild":"20240330_1234(x64)","sessionid":"b3b1a2e0-4b5c-4d2a-8c3e-2b1a2e04b5c4","pyrevit":"4.8","clone":"main","debug":false,"config":false,"from_gui":false,"exec_id":"execid","exec_timestamp":"2024-03-30T08:45:00Z","commandname":"cmd","commanduniquename":"cmd.unique","commandbundle":"bundle","commandextension":"ext","docname":"doc","docpath":"/path/to/doc","resultcode":0,"commandresults":{},"scriptpath":"/path/to/script","trace":{"engine":{"type":"ironpython","version":"1.0","syspath":[],"configs":{}},"message":""}}` + req := httptest.NewRequest("POST", "/api/v2/scripts/", strings.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200 OK, got %d", w.Code) + } + if w.Body.Len() == 0 { + t.Fatalf("expected non-empty body") + } + // GET + req = httptest.NewRequest("GET", "/api/v2/scripts/", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200 OK, got %d", w.Code) + } +} + +func TestEventV2Endpoints(t *testing.T) { + router := NewRouter(&cli.Options{ScriptsTable: "scripts", EventsTable: "events"}, &mockDB{}, &cli.Logger{}) + // Valid payload + payload := `{"meta":{"schema":"2.0"},"timestamp":"2024-03-30T08:45:00Z","handler_id":"handler","type":"eventtype","args":{},"username":"user","host_user":"host","revit":"2021","revitbuild":"20240330_1234(x64)","cancellable":false,"cancelled":false,"docid":1,"doctype":"type","doctemplate":"template","docname":"doc","docpath":"/path/to/doc","projectnum":"123","projectname":"proj"}` + req := httptest.NewRequest("POST", "/api/v2/events/", strings.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200 OK, got %d", w.Code) + } + if w.Body.Len() == 0 { + t.Fatalf("expected non-empty body") + } + // GET + req = httptest.NewRequest("GET", "/api/v2/events/", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200 OK, got %d", w.Code) + } +} From d461ee7ef4cf69b2814866b81e8cc988f63ff0ba Mon Sep 17 00:00:00 2001 From: fishonamos Date: Tue, 24 Jun 2025 11:31:33 +0100 Subject: [PATCH 10/14] add go test --- .github/workflows/main.yml | 5 ++++- cli/usage.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ec3e0b0..619a8c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,7 +3,7 @@ name: CI/CD on: workflow_dispatch: push: - branches: [main] + branches: ["**"] jobs: build: @@ -28,6 +28,9 @@ jobs: name: bin path: bin + - name: Run Tests + run: go test ./... + docker: runs-on: ubuntu-latest needs: build diff --git a/cli/usage.go b/cli/usage.go index 55c7091..d14b823 100644 --- a/cli/usage.go +++ b/cli/usage.go @@ -40,7 +40,7 @@ var printHelpAndExit = func(err error, docoptMessage string) { if err != nil { // if err occured print full help // docopt only includes usage section in its message - fmt.Fprintln(os.Stderr, help) + fmt.Fprint(os.Stderr, help) os.Exit(1) } else { // otherwise print whatever docopt says From 8d430cc9bb0d985d41aca0e3434c2275f18ed0bb Mon Sep 17 00:00:00 2001 From: fishonamos Date: Tue, 24 Jun 2025 11:34:12 +0100 Subject: [PATCH 11/14] update test --- server/server_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/server_test.go b/server/server_test.go index 076fdce..e9d7752 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -87,7 +87,7 @@ func TestScriptV1Endpoints(t *testing.T) { func TestScriptV2Endpoints(t *testing.T) { router := NewRouter(&cli.Options{ScriptsTable: "scripts", EventsTable: "events"}, &mockDB{}, &cli.Logger{}) - // Valid payload + // test payload payload := `{"meta":{"schema":"2.0"},"timestamp":"2024-03-30T08:45:00Z","username":"user","host_user":"host","revit":"2021","revitbuild":"20240330_1234(x64)","sessionid":"b3b1a2e0-4b5c-4d2a-8c3e-2b1a2e04b5c4","pyrevit":"4.8","clone":"main","debug":false,"config":false,"from_gui":false,"exec_id":"execid","exec_timestamp":"2024-03-30T08:45:00Z","commandname":"cmd","commanduniquename":"cmd.unique","commandbundle":"bundle","commandextension":"ext","docname":"doc","docpath":"/path/to/doc","resultcode":0,"commandresults":{},"scriptpath":"/path/to/script","trace":{"engine":{"type":"ironpython","version":"1.0","syspath":[],"configs":{}},"message":""}}` req := httptest.NewRequest("POST", "/api/v2/scripts/", strings.NewReader(payload)) req.Header.Set("Content-Type", "application/json") @@ -99,7 +99,7 @@ func TestScriptV2Endpoints(t *testing.T) { if w.Body.Len() == 0 { t.Fatalf("expected non-empty body") } - // GET + // GET scripts req = httptest.NewRequest("GET", "/api/v2/scripts/", nil) w = httptest.NewRecorder() router.ServeHTTP(w, req) @@ -110,7 +110,7 @@ func TestScriptV2Endpoints(t *testing.T) { func TestEventV2Endpoints(t *testing.T) { router := NewRouter(&cli.Options{ScriptsTable: "scripts", EventsTable: "events"}, &mockDB{}, &cli.Logger{}) - // Valid payload + // test payload payload := `{"meta":{"schema":"2.0"},"timestamp":"2024-03-30T08:45:00Z","handler_id":"handler","type":"eventtype","args":{},"username":"user","host_user":"host","revit":"2021","revitbuild":"20240330_1234(x64)","cancellable":false,"cancelled":false,"docid":1,"doctype":"type","doctemplate":"template","docname":"doc","docpath":"/path/to/doc","projectnum":"123","projectname":"proj"}` req := httptest.NewRequest("POST", "/api/v2/events/", strings.NewReader(payload)) req.Header.Set("Content-Type", "application/json") @@ -122,7 +122,7 @@ func TestEventV2Endpoints(t *testing.T) { if w.Body.Len() == 0 { t.Fatalf("expected non-empty body") } - // GET + // GET e req = httptest.NewRequest("GET", "/api/v2/events/", nil) w = httptest.NewRecorder() router.ServeHTTP(w, req) From bc190f4e32115feb49a3135a7baabc882fe57023 Mon Sep 17 00:00:00 2001 From: fishonamos Date: Tue, 24 Jun 2025 12:09:41 +0100 Subject: [PATCH 12/14] update test to units and verbose test actions --- server/server_test.go | 255 ++++++++++++++++++++++++++++++------------ 1 file changed, 181 insertions(+), 74 deletions(-) diff --git a/server/server_test.go b/server/server_test.go index e9d7752..dc13a4c 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -10,27 +10,15 @@ import ( ) type mockDB struct{ persistence.Connection } -type mockLogger struct{ cli.Logger } -func TestStatusEndpoint(t *testing.T) { +func setupTestRouter() http.Handler { opts := &cli.Options{ ScriptsTable: "scripts", EventsTable: "events", } db := &mockDB{} logger := &cli.Logger{} - - router := NewRouter(opts, db, logger) - req := httptest.NewRequest("GET", "/api/v1/status", nil) - w := httptest.NewRecorder() - router.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected 200 OK, got %d", w.Code) - } - if w.Body.Len() == 0 { - t.Fatalf("expected non-empty body") - } + return NewRouter(opts, db, logger) } func (m *mockDB) WriteScriptTelemetryV1(logrec *persistence.ScriptTelemetryRecordV1, logger *cli.Logger) (*persistence.Result, error) { @@ -62,71 +50,190 @@ func (m *mockDB) GetStatus(logger *cli.Logger) persistence.ConnectionStatus { } } +// ------------------------- +// Status Endpoint +// ------------------------- + +func TestStatusEndpoint(t *testing.T) { + router := setupTestRouter() + + t.Run("GET /api/v1/status returns 200", func(t *testing.T) { + req := httptest.NewRequest("GET", "/api/v1/status", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + if w.Body.Len() == 0 { + t.Errorf("expected non-empty body") + } + }) +} + +// ------------------------- +// Script V1 +// ------------------------- + func TestScriptV1Endpoints(t *testing.T) { - router := NewRouter(&cli.Options{ScriptsTable: "scripts", EventsTable: "events"}, &mockDB{}, &cli.Logger{}) - // Valid payload - payload := `{"date":"2024-03-30","time":"08:45:00","username":"user","revit":"2021","revitbuild":"20240330_1234(x64)","sessionid":"b3b1a2e0-4b5c-4d2a-8c3e-2b1a2e04b5c4","pyrevit":"4.8","debug":false,"config":false,"commandname":"cmd","commanduniquename":"cmd.unique","commandbundle":"bundle","commandextension":"ext","resultcode":0,"commandresults":{},"scriptpath":"/path/to/script","trace":{"engine":{"version":"1.0","syspath":[]},"ipy":"","clr":""}}` - req := httptest.NewRequest("POST", "/api/v1/scripts/", strings.NewReader(payload)) - req.Header.Set("Content-Type", "application/json") - w := httptest.NewRecorder() - router.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("expected 200 OK, got %d", w.Code) - } - if w.Body.Len() == 0 { - t.Fatalf("expected non-empty body") - } - // GET - req = httptest.NewRequest("GET", "/api/v1/scripts/", nil) - w = httptest.NewRecorder() - router.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("expected 200 OK, got %d", w.Code) - } + router := setupTestRouter() + + payload := `{ + "date":"2024-03-30", + "time":"08:45:00", + "username":"user", + "revit":"2021", + "revitbuild":"20240330_1234(x64)", + "sessionid":"b3b1a2e0-4b5c-4d2a-8c3e-2b1a2e04b5c4", + "pyrevit":"4.8", + "debug":false, + "config":false, + "commandname":"cmd", + "commanduniquename":"cmd.unique", + "commandbundle":"bundle", + "commandextension":"ext", + "resultcode":0, + "commandresults":{}, + "scriptpath":"/path/to/script", + "trace":{ + "engine":{"version":"1.0","syspath":[]}, + "ipy":"", + "clr":"" + } + }` + + t.Run("POST /api/v1/scripts/ returns 200", func(t *testing.T) { + req := httptest.NewRequest("POST", "/api/v1/scripts/", strings.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + }) + + t.Run("GET /api/v1/scripts/ returns 200", func(t *testing.T) { + req := httptest.NewRequest("GET", "/api/v1/scripts/", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + }) } +// ------------------------- +// Script V2 +// ------------------------- + func TestScriptV2Endpoints(t *testing.T) { - router := NewRouter(&cli.Options{ScriptsTable: "scripts", EventsTable: "events"}, &mockDB{}, &cli.Logger{}) - // test payload - payload := `{"meta":{"schema":"2.0"},"timestamp":"2024-03-30T08:45:00Z","username":"user","host_user":"host","revit":"2021","revitbuild":"20240330_1234(x64)","sessionid":"b3b1a2e0-4b5c-4d2a-8c3e-2b1a2e04b5c4","pyrevit":"4.8","clone":"main","debug":false,"config":false,"from_gui":false,"exec_id":"execid","exec_timestamp":"2024-03-30T08:45:00Z","commandname":"cmd","commanduniquename":"cmd.unique","commandbundle":"bundle","commandextension":"ext","docname":"doc","docpath":"/path/to/doc","resultcode":0,"commandresults":{},"scriptpath":"/path/to/script","trace":{"engine":{"type":"ironpython","version":"1.0","syspath":[],"configs":{}},"message":""}}` - req := httptest.NewRequest("POST", "/api/v2/scripts/", strings.NewReader(payload)) - req.Header.Set("Content-Type", "application/json") - w := httptest.NewRecorder() - router.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("expected 200 OK, got %d", w.Code) - } - if w.Body.Len() == 0 { - t.Fatalf("expected non-empty body") - } - // GET scripts - req = httptest.NewRequest("GET", "/api/v2/scripts/", nil) - w = httptest.NewRecorder() - router.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("expected 200 OK, got %d", w.Code) - } + router := setupTestRouter() + + payload := `{ + "meta":{"schema":"2.0"}, + "timestamp":"2024-03-30T08:45:00Z", + "username":"user", + "host_user":"host", + "revit":"2021", + "revitbuild":"20240330_1234(x64)", + "sessionid":"b3b1a2e0-4b5c-4d2a-8c3e-2b1a2e04b5c4", + "pyrevit":"4.8", + "clone":"main", + "debug":false, + "config":false, + "from_gui":false, + "exec_id":"execid", + "exec_timestamp":"2024-03-30T08:45:00Z", + "commandname":"cmd", + "commanduniquename":"cmd.unique", + "commandbundle":"bundle", + "commandextension":"ext", + "docname":"doc", + "docpath":"/path/to/doc", + "resultcode":0, + "commandresults":{}, + "scriptpath":"/path/to/script", + "trace":{ + "engine":{ + "type":"ironpython", + "version":"1.0", + "syspath":[], + "configs":{} + }, + "message":"" + } + }` + + t.Run("POST /api/v2/scripts/ returns 200", func(t *testing.T) { + req := httptest.NewRequest("POST", "/api/v2/scripts/", strings.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + }) + + t.Run("GET /api/v2/scripts/ returns 200", func(t *testing.T) { + req := httptest.NewRequest("GET", "/api/v2/scripts/", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + }) } +// ------------------------- +// Event V2 +// ------------------------- + func TestEventV2Endpoints(t *testing.T) { - router := NewRouter(&cli.Options{ScriptsTable: "scripts", EventsTable: "events"}, &mockDB{}, &cli.Logger{}) - // test payload - payload := `{"meta":{"schema":"2.0"},"timestamp":"2024-03-30T08:45:00Z","handler_id":"handler","type":"eventtype","args":{},"username":"user","host_user":"host","revit":"2021","revitbuild":"20240330_1234(x64)","cancellable":false,"cancelled":false,"docid":1,"doctype":"type","doctemplate":"template","docname":"doc","docpath":"/path/to/doc","projectnum":"123","projectname":"proj"}` - req := httptest.NewRequest("POST", "/api/v2/events/", strings.NewReader(payload)) - req.Header.Set("Content-Type", "application/json") - w := httptest.NewRecorder() - router.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("expected 200 OK, got %d", w.Code) - } - if w.Body.Len() == 0 { - t.Fatalf("expected non-empty body") - } - // GET e - req = httptest.NewRequest("GET", "/api/v2/events/", nil) - w = httptest.NewRecorder() - router.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("expected 200 OK, got %d", w.Code) - } + router := setupTestRouter() + + payload := `{ + "meta":{"schema":"2.0"}, + "timestamp":"2024-03-30T08:45:00Z", + "handler_id":"handler", + "type":"eventtype", + "args":{}, + "username":"user", + "host_user":"host", + "revit":"2021", + "revitbuild":"20240330_1234(x64)", + "cancellable":false, + "cancelled":false, + "docid":1, + "doctype":"type", + "doctemplate":"template", + "docname":"doc", + "docpath":"/path/to/doc", + "projectnum":"123", + "projectname":"proj" + }` + + t.Run("POST /api/v2/events/ returns 200", func(t *testing.T) { + req := httptest.NewRequest("POST", "/api/v2/events/", strings.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + }) + + t.Run("GET /api/v2/events/ returns 200", func(t *testing.T) { + req := httptest.NewRequest("GET", "/api/v2/events/", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + }) } From 3e4e4a70629b98757b98028b44590e5981d6ead4 Mon Sep 17 00:00:00 2001 From: fishonamos Date: Tue, 24 Jun 2025 12:11:18 +0100 Subject: [PATCH 13/14] workflow -v --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 619a8c3..92b8cfc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,7 +29,7 @@ jobs: path: bin - name: Run Tests - run: go test ./... + run: go test -v ./... docker: runs-on: ubuntu-latest From ec9ffda1410c4df956686b844a506bb0f5f76026 Mon Sep 17 00:00:00 2001 From: fishonamos Date: Tue, 24 Jun 2025 12:12:16 +0100 Subject: [PATCH 14/14] add go sum --- go.sum | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go.sum b/go.sum index 2ac5ef4..092d64b 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=