Improve response parsing: no ptypes, faster datetimes

DESCRIPTION

@@ -87,7 +87,6 @@ Collate:
     'page.R'
     'parse.R'
     'promote.R'
-    'ptype.R'
     'remote.R'
     'runtime-caches.R'
     'schedule.R'

NEWS.md

@@ -2,6 +2,13 @@
 
 - When using integrations, prefer to read from `CONNECT_CONTENT_SESSION_TOKEN_FILE` to find the session token. This helps long-running processes ensure that they can maintain fresh credentials.
 
+- Improved performance of API response parsing and pagination. Data frames
+  returned by getter functions now include all columns from the server, so new
+  fields added in future Connect releases will appear automatically. Column
+  names and types are now determined by the server response rather than a
+  hardcoded schema, so they may vary across Connect versions.
+
+
 # connectapi 0.11.1
 
 - `get_usage()` now returns the id column as a character to match other parts of the API (#512).

R/connect.R

@@ -138,8 +138,11 @@ Connect <- R6::R6Class(
     #' @param parser How the response is parsed. If `NULL`, the `httr_response`
     #' will be returned. Otherwise, the argument is forwarded to
     #' `httr::content(res, as = parser)`.
+    #' @param simplify Logical; if `TRUE`, JSON arrays of objects are
+    #' simplified to data frames by jsonlite. Default `FALSE` preserves
+    #' list-of-lists for compatibility with pagination helpers.
     #' @param ... Additional arguments passed to the request function
-    request = function(method, url, ..., parser = "parsed") {
+    request = function(method, url, ..., parser = "parsed", simplify = FALSE) {
       old_opt <- options(scipen = 999)
       on.exit(options(old_opt), add = TRUE)
 
@@ -161,7 +164,21 @@ Connect <- R6::R6Class(
         res
       } else {
         self$raise_error(res)
-        httr::content(res, as = parser)
+        if (parser != "parsed") {
+          return(httr::content(res, as = parser))
+        }
+        if (is.null(res$content) || length(res$content) == 0) {
+          return(NULL)
+        }
+        content_text <- httr::content(res, as = "text", encoding = "UTF-8")
+        if (is.null(content_text) || nchar(content_text) == 0) {
+          return(NULL)
+        }
+        jsonlite::fromJSON(
+          content_text,
+          simplifyVector = simplify,
+          simplifyDataFrame = simplify
+        )
       }
     },
 
@@ -173,8 +190,11 @@ Connect <- R6::R6Class(
     #' @param parser How the response is parsed. If `NULL`, the `httr_response`
     #' will be returned. Otherwise, the argument is forwarded to
     #' `httr::content(res, as = parser)`.
-    GET = function(path, ..., url = self$api_url(path), parser = "parsed") {
-      self$request("GET", url, parser = parser, ...)
+    #' @param simplify Logical; if `TRUE`, JSON arrays of objects are
+    #' simplified to data frames by jsonlite. Default `FALSE` preserves
+    #' list-of-lists for compatibility with pagination helpers.
+    GET = function(path, ..., url = self$api_url(path), parser = "parsed", simplify = FALSE) {
+      self$request("GET", url, parser = parser, simplify = simplify, ...)
     },
 
     #' @description Perform an HTTP PUT request of the named API path.

R/connectapi.R

@@ -10,7 +10,6 @@ utils::globalVariables(
   c(
     ".",
     "access_type",
-    "connectapi_ptypes",
     "guid",
     "last_deployed_time",
     "owner_guid",
@@ -27,6 +26,5 @@ current_connect_version <- "2024.03.0"
 
 .onLoad <- function(...) {
   vctrs::s3_register("dplyr::collect", "tbl_connect")
-  vctrs::s3_register("vctrs::vec_cast", "character.integer")
   invisible()
 }

R/content.R

@@ -788,7 +788,23 @@ get_jobs <- function(content) {
   validate_R6_class(content, "Content")
 
   jobs <- content$jobs()
-  parse_connectapi_typed(jobs, connectapi_ptypes$jobs, strict = TRUE)
+  out <- parse_connectapi_typed(
+    jobs,
+    datetime_cols = c("start_time", "end_time", "last_heartbeat_time", "queued_time")
+  )
+
+  # The older /applications/ endpoint returns timestamps as Unix epoch integers
+  # and ID fields as integers. Normalize to match the v1 endpoint's types.
+  # For the v1 endpoint these are already character/POSIXct, so the coercions
+  # are no-ops.
+  out <- coerce_epoch_to_posixct(
+    out,
+    c("start_time", "end_time", "last_heartbeat_time", "queued_time")
+  )
+  coerce_to_character(
+    out,
+    c("id", "ppid", "pid", "app_id", "content_id", "variant_id", "bundle_id")
+  )
 }
 
 #' Terminate Jobs
@@ -832,22 +848,19 @@ terminate_jobs <- function(content, keys = NULL) {
     keys <- all_jobs[all_jobs$status == 0, ]$key
     if (length(keys) == 0) {
       message("No active jobs found.")
-      return(vctrs::vec_ptype(connectapi_ptypes$job_termination))
+      return(tibble::tibble())
     }
   }
 
   res <- purrr::map(keys, content$register_job_kill_order)
   res_content <- purrr::map(res, httr::content)
-  res_df <- tibble::tibble(
-    parse_connectapi_typed(
-      res_content,
-      connectapi_ptypes$job_termination,
-      strict = TRUE
-    )
-  )
+  res_df <- parse_connectapi_typed(res_content)
   # Errors will not have the job_key.
   res_df$job_key <- keys
-  res_df
+  # Keep only the columns relevant to job termination; the API response
+  # includes extra fields (e.g. payload, guid) on error that vary by outcome.
+  keep <- c("app_id", "app_guid", "job_key", "job_id", "result", "code", "error")
+  res_df[, keep, drop = FALSE]
 }
 
 #' @rdname get_jobs
@@ -896,7 +909,7 @@ get_log <- function(job, max_log_lines = NULL) {
     v1_url("content", job$app_guid, "jobs", job$key, "log"),
     query = query
   )
-  parse_connectapi_typed(res$entries, connectapi_ptypes$job_log)
+  parse_connectapi_typed(res$entries, datetime_cols = "timestamp")
 }
 
 #' Set RunAs User
@@ -1141,7 +1154,8 @@ get_bundles <- function(content) {
   validate_R6_class(content, "Content")
   bundles <- content$get_bundles()
 
-  parse_connectapi_typed(bundles, connectapi_ptypes$bundles)
+  out <- parse_connectapi_typed(bundles, datetime_cols = "created_time")
+  coerce_fs_bytes(out, "size")
 }
 
 #' @rdname get_bundles
@@ -1347,7 +1361,7 @@ get_group_permission <- function(content, guid) {
 get_content_permissions <- function(content, add_owner = TRUE) {
   validate_R6_class(content, "Content")
   res <- content$permissions(add_owner = add_owner)
-  parse_connectapi_typed(res, connectapi_ptypes$permissions)
+  parse_connectapi_typed(res)
 }
 
 #' Render a content item.
@@ -1495,7 +1509,7 @@ content_restart <- function(content) {
 get_content_packages <- function(content) {
   error_if_less_than(content$connect$version, "2025.01.0")
   res <- content$packages()
-  parse_connectapi_typed(res, connectapi_ptypes$content_packages)
+  parse_connectapi_typed(res)
 }
 
 #' Search for content on the Connect server
@@ -1627,5 +1641,8 @@ as.data.frame.connect_content_list <- function(
 #' @export
 as_tibble.connect_content_list <- function(x, ...) {
   content_data <- purrr::map(x, "content")
-  parse_connectapi_typed(content_data, connectapi_ptypes$content)
+  parse_connectapi_typed(
+    content_data,
+    datetime_cols = datetime_columns$content
+  )
 }

R/get.R

@@ -73,7 +73,10 @@ get_users <- function(
     limit = limit
   )
 
-  out <- parse_connectapi_typed(res, connectapi_ptypes$users)
+  out <- parse_connectapi_typed(
+    res,
+    datetime_cols = datetime_columns$users
+  )
 
   return(out)
 }
@@ -229,12 +232,8 @@ get_content <- function(
   # v2024.06.0.
   if (compare_connect_version(src$version, "2024.06.0") < 0) {
     include <- "tags,owner"
-    content_ptype <- connectapi_ptypes$content[,
-      names(connectapi_ptypes$content) != "vanity_url"
-    ]
   } else {
     include <- "tags,owner,vanity_url"
-    content_ptype <- connectapi_ptypes$content
   }
 
   res <- src$content(
@@ -253,7 +252,10 @@ get_content <- function(
     res <- res %>% purrr::keep(.p = .p)
   }
 
-  out <- parse_connectapi_typed(res, content_ptype)
+  out <- parse_connectapi_typed(
+    res,
+    datetime_cols = datetime_columns$content
+  )
 
   return(out)
 }
@@ -327,7 +329,10 @@ content_list_by_tag <- function(src, tag) {
 
   res <- src$GET(v1_url("tags", tag_id, "content"))
 
-  out <- parse_connectapi_typed(res, connectapi_ptypes$content)
+  out <- parse_connectapi_typed(
+    res,
+    datetime_cols = datetime_columns$content
+  )
   return(out)
 }
 
@@ -425,7 +430,7 @@ get_usage_shiny <- function(
 
   res <- page_cursor(src, res, limit = limit)
 
-  out <- parse_connectapi_typed(res, connectapi_ptypes$usage_shiny)
+  out <- parse_connectapi_typed(res, datetime_cols = datetime_columns$usage_shiny)
 
   return(out)
 }
@@ -521,7 +526,7 @@ get_usage_static <- function(
 
   res <- page_cursor(src, res, limit = limit)
 
-  out <- parse_connectapi_typed(res, connectapi_ptypes$usage_static)
+  out <- parse_connectapi_typed(res, datetime_cols = datetime_columns$usage_static)
 
   return(out)
 }
@@ -658,7 +663,7 @@ as.data.frame.connect_list_hits <- function(
   ...,
   unnest = TRUE
 ) {
-  usage_df <- parse_connectapi_typed(x, connectapi_ptypes$usage)
+  usage_df <- parse_connectapi_typed(x, datetime_cols = "timestamp")
   if (unnest) {
     if (!requireNamespace("tidyr", quietly = TRUE)) {
       stop(
@@ -750,7 +755,7 @@ get_audit_logs <- function(
 
   res <- page_cursor(src, res, limit = limit)
 
-  out <- parse_connectapi_typed(res, connectapi_ptypes$audit_logs)
+  out <- parse_connectapi_typed(res, datetime_cols = datetime_columns$audit_logs)
 
   return(out)
 }
@@ -792,7 +797,8 @@ get_procs <- function(src) {
       c(list(pid = y), x)
     }
   )
-  tbl_data <- parse_connectapi_typed(proc_prep, connectapi_ptypes$procs)
+  tbl_data <- parse_connectapi_typed(proc_prep)
+  tbl_data <- coerce_fs_bytes(tbl_data, "ram")
 
   return(tbl_data)
 }
@@ -1217,7 +1223,7 @@ get_packages <- function(src, name = NULL, page_size = 100000, limit = Inf) {
       page_size = page_size
     )
   )
-  out <- parse_connectapi_typed(res, connectapi_ptypes$packages)
+  out <- parse_connectapi_typed(res)
 
   # Connect is standardizing on using `content_id` and `content_guid`.
   # Handle that name change now in a forward-compatible way.
@@ -1253,5 +1259,5 @@ get_packages <- function(src, name = NULL, page_size = 100000, limit = Inf) {
 #' @export
 get_vanity_urls <- function(client) {
   res <- client$vanities()
-  parse_connectapi_typed(res, connectapi_ptypes$vanities)
+  parse_connectapi_typed(res, datetime_cols = "created_time")
 }

R/groups.R

@@ -53,7 +53,7 @@ get_groups <- function(src, page_size = 500, prefix = NULL, limit = Inf) {
     limit = limit
   )
 
-  parse_connectapi_typed(res, connectapi_ptypes$groups)
+  parse_connectapi_typed(res, datetime_cols = datetime_columns$groups)
 }
 
 #' Get users within a specific group
@@ -106,7 +106,10 @@ get_group_members <- function(src, guid) {
 
   res <- src$group_members(guid)
 
-  parse_connectapi(res$results)
+  parse_connectapi_typed(
+    res$results,
+    datetime_cols = c("created_time", "updated_time", "active_time")
+  )
 }
 
 #' Get content access permissions for a group or groups
@@ -172,7 +175,7 @@ get_one_groups_content <- function(src, guid) {
       role = NA_character_
     ))
   }
-  parsed <- parse_connectapi_typed(res, connectapi_ptypes$group_content)
+  parsed <- parse_connectapi_typed(res)
 
   permissions_df <- purrr::list_rbind(
     purrr::map(

R/integrations.R

@@ -136,7 +136,10 @@ as.data.frame.connect_integration_list <- function(
 #' @return A tibble with one row per integration.
 #' @export
 as_tibble.connect_integration_list <- function(x, ...) {
-  parse_connectapi_typed(x, connectapi_ptypes$integrations)
+  parse_connectapi_typed(
+    x,
+    datetime_cols = c("created_time", "updated_time")
+  )
 }
 
 # Integration class ----