Format refactor

README.md

@@ -103,7 +103,7 @@ table in.csv --columns 'name,last_name'
 * Append a new column that has result of multiplication of two other columns. To substitute column value in the command `${column name}` format should be used. New column gets name 'newColumn1':
 
 ```bash
-table in.csv --add 'echo "${cost} * ${amount}" | bc'
+table in.csv --add sum='#{echo "${cost} + ${amount}" | bc}'
 ```
 
 * Joining two CSV files by a common column. Joins on the column named 'id' in the first file that should match 'product_id' in the second file:

Sources/table/Extensions.swift

@@ -16,7 +16,6 @@ extension String {
         return self.range(of: regex, options: .regularExpression, range: nil, locale: nil) != nil
     }
 
-    // TODO: remove me
     var isNumber: Bool {
         if let _ = Double(self) {
             return true

Sources/table/Format.swift

@@ -1,91 +1,79 @@
 import Foundation
 
-class Format {
-    static let regex = try! NSRegularExpression(pattern: "\\$\\{([%A-Za-z0-9_\\s]+)\\}")
-    static let internalVars = ["%header", "%values", "%quoted_values"]
-    let format: String
-    let matches: [NSTextCheckingResult]
-    let parts: [String]
-    let vars: [String]
+// Structure representing a format tree
+protocol FormatExpr: CustomStringConvertible, Equatable {
+    func fill(row: Row) throws -> String 
+    func validate(header: Header?) throws -> Void
+}
 
-    init(format: String) {
-        self.format = format
-        let range = NSRange(format.startIndex..., in: format)
-        matches = Format.regex.matches(in: format, range: range)
-
-        var variables: [String] = []
-        var strParts: [String] = []
-
-        var lastIndex = format.startIndex
-
-        // Break matches into 2 arrays text parts and variable names
-        for match in matches {        
-            let range = lastIndex..<format.index(format.startIndex, offsetBy: match.range.lowerBound)
-            variables.append(String(format[Range(match.range(at: 1), in: format)!]))
-            strParts.append(String(format[range]))
-            lastIndex = format.index(format.startIndex, offsetBy: match.range.upperBound)
-        }
+struct VarPart: FormatExpr {
+    let name: String
 
-        strParts.append(String(format[lastIndex...]))
+    init(_ name: String) {
+        self.name = name
+    }
 
-        parts = strParts
-        vars = variables
+    func fill(row: Row) -> String {
+        return row[name] ?? ""
     }
 
-    func validated(header: Header?) throws -> Format {
+    func validate(header: Header?) throws {
         if let h = header {
-            for v in vars {
-                if h.index(ofColumn: v) == nil && !Format.internalVars.contains(v) {
-                    throw RuntimeError("Unknown column in print format: \(v). Supported columns: \(h.columnsStr())")
-                }
-            }
+            if h.index(ofColumn: name) == nil {
+                throw RuntimeError("Unknown column in format: \(name). Supported columns: \(h.columnsStr())")
+            }        
         }
-
-        return self
     }
 
-    // Format allows to specify initial column values as well as 
-    // dynamically formed columns
-    func fill(row: Row) -> String {
-        var idx = 0
-        var newStr = ""
-
-        for v in vars {
-            newStr += parts[idx] + columnValue(row: row, name: v)
-            idx += 1
-        }
+    var description: String {
+        return "Var(name: \(name))"
+    }
+}
 
-        newStr += parts[idx]
+struct TextPart: FormatExpr {
+    let text: String
 
-        return newStr
+    init(_ text: String) {
+        self.text = text
     }
 
-    func fillData(row: Row) -> Data {
-        fill(row: row).data(using: .utf8)!
+    func fill(row: Row) -> String {
+        return text
     }
 
-    func columnValue(row: Row, name: String) -> String {
-        if let v = resolveInternalVariable(row, name) {
-            return v
-        }
+    func validate(header: Header?) throws {}
 
-        if let v = row[name] {
-            return v
-        }
+    var description: String {
+        return "Text(\(text))"
+    }
+}
+
+struct FunctionPart: FormatExpr {
+    let name: String
+
+    static let internalFunctions = ["header", "values", "quoted_values", "uuid"]
 
-        return ""
+    init(fnName: String) {
+        self.name = fnName
     }
 
-    func resolveInternalVariable(_ row: Row, _ name: String) -> String? {
-        if name == "%header" {
-            return row.header?.columnsStr()
+    func fill(row: Row) throws -> String {
+        if name == "header" {
+            guard let header = row.header else {
+                throw RuntimeError("Header is not defined")
+            }
+            return header.columnsStr()
         }
 
-        if name == "%values" {
+        if name == "values" {
             return row.components.map({ $0.value }).joined(separator: ",")
         }
 
-        if name == "%quoted_values" {
+        if name == "uuid" {
+            return UUID().uuidString
+        }
+
+        if name == "quoted_values" {
             return row.components.enumerated().map { (index, cell) in
                 let v = cell.value
                 let type = row.header?.type(ofIndex: index) ?? .string
@@ -98,6 +86,173 @@ class Format {
             }.joined(separator: ",")
         }
 
-        return nil
+        throw RuntimeError("Unknown function: \(name). Supported functions: \(FunctionPart.internalFunctions.joined(separator: ", "))")
+    }
+
+    func validate(header: Header?) throws {
+        if let h = header {
+            if !FunctionPart.internalFunctions.contains(name) && h.index(ofColumn: name) == nil {
+                throw RuntimeError("Unknown function in format: \(name). Supported columns: \(FunctionPart.internalFunctions.joined(separator: ", "))")
+            }
+        }
+    }
+
+    var description: String {
+        return "Function(name: \(name))"
+    }
+}
+
+struct FormatGroup: FormatExpr {
+    let parts: [any FormatExpr]
+
+    init(_ parts: [any FormatExpr]) {
+        self.parts = parts
+    }
+
+    func fill(row: Row) throws -> String {
+        var result = ""
+        for part in parts {
+            result += try part.fill(row: row)
+        }
+        return result
+    }
+
+    func validate(header: Header?) throws {
+        for part in parts {
+            try part.validate(header: header)
+        }
+    }
+
+    var description: String {
+        return "Group(parts: \(parts))"
+    }
+
+    static func == (lhs: FormatGroup, rhs: FormatGroup) -> Bool {
+        return lhs.parts.map { $0.description } == rhs.parts.map { $0.description }
+    }
+}
+
+struct ExecPart: FormatExpr {
+    let command: any FormatExpr
+
+    init(command: any FormatExpr) {
+        self.command = command
+    }
+
+    func fill(row: Row) throws -> String {
+        return try shell(String(describing: command.fill(row: row)))
+    }
+
+    func validate(header: Header?) throws {
+        try command.validate(header: header)
+    }
+
+    var description: String {
+        return "Exec(command: \(command))"
+    }
+
+    static func == (lhs: ExecPart, rhs: ExecPart) -> Bool {
+        return lhs.command.description == rhs.command.description
+    }
+}
+
+class Format {
+    static let regex: NSRegularExpression = try! NSRegularExpression(pattern: "\\$\\{([%A-Za-z0-9_\\s]+)\\}")
+
+    let original: String
+    let format: any FormatExpr
+
+    init(format: String) {
+        self.original = format
+        self.format = Format.parse(original).0
+    }
+
+    func validated(header: Header?) throws -> Format {
+        try format.validate(header: header)
+        return self
+    }
+
+    func fill(row: Row) -> String {
+        // we rely on the fact that `fill` is called only after validation
+        try! format.fill(row: row)
+    }
+
+    func fillData(row: Row) -> Data {
+        fill(row: row).data(using: .utf8)!
+    }
+
+    static func parse(_ input: String, from start: String.Index? = nil, until closing: Character? = nil) -> (any FormatExpr, String.Index) {
+        var index = start ?? input.startIndex
+        var nodes: [any FormatExpr] = []
+        var buffer = ""
+
+        while index < input.endIndex {
+            // Handle closing delimiter if needed
+            if let closing = closing, input[index] == closing {
+                if !buffer.isEmpty {
+                    nodes.append(TextPart(buffer))
+                }
+                return (FormatGroup(nodes), input.index(after: index))
+            }
+
+            if input[index...].hasPrefix("${") {
+                if !buffer.isEmpty {
+                    nodes.append(TextPart(buffer))
+                    buffer = ""
+                }
+                index = input.index(index, offsetBy: 2)
+                let (name, newIndex) = readUntil(input, delimiter: "}", from: index)
+                if let name = name {
+                    nodes.append(VarPart(name))
+                }
+                index = newIndex
+
+            } else if input[index...].hasPrefix("%{") {
+                if !buffer.isEmpty {
+                    nodes.append(TextPart(buffer))
+                    buffer = ""
+                }
+                index = input.index(index, offsetBy: 2)
+                let (name, newIndex) = readUntil(input, delimiter: "}", from: index)
+                if let name = name {
+                    nodes.append(FunctionPart(fnName: name))
+                }
+                index = newIndex
+            } else if input[index...].hasPrefix("#{") {
+                if !buffer.isEmpty {
+                    nodes.append(TextPart(buffer))
+                    buffer = ""
+                }
+                index = input.index(index, offsetBy: 2)
+                let (inner, newIndex) = parse(input, from: index, until: "}")
+                nodes.append(ExecPart(command: inner))
+                index = newIndex
+
+            } else {
+                buffer.append(input[index])
+                index = input.index(after: index)
+            }
+        }
+
+        if !buffer.isEmpty {
+            nodes.append(TextPart(buffer))
+        }
+
+        return (FormatGroup(nodes), index)
+    }
+
+    private static func readUntil(_ input: String, delimiter: Character, from start: String.Index) -> (String?, String.Index) {
+        var index = start
+        var result = ""
+
+        while index < input.endIndex {
+            if input[index] == delimiter {
+                return (result, input.index(after: index))
+            }
+            result.append(input[index])
+            index = input.index(after: index)
+        }
+
+        return (nil, index)
     }
 }

Sources/table/MainApp.swift

@@ -79,7 +79,7 @@ struct MainApp: ParsableCommand {
     @Option(name: .customLong("limit"), help: "Process only up to specified number of lines.")
     var limitLines: Int?
 
-    @Option(name: [.customLong("print")], help: "Format output accorindg to format string. Use ${column name} to print column value. Example: Column1 value is ${column1}.")
+    @Option(name: [.customLong("print")], help: "Format output accorindg to format string. Use ${column name} to print column value. Expression #{cmd} can be used to execute command. Example: Column1 value is ${column1} and execution result #{curl service/${column2}}.")
     var printFormat: String?
 
     @Option(name: [.customLong("as")], help: "Prints output in the specified format. Supported formats: table (default) or csv.")
@@ -89,8 +89,7 @@ struct MainApp: ParsableCommand {
     @Option(name: .shortAndLong, help: "Filter rows by a single value criteria. Example: country=UA or size>10. Supported comparison operations: '=' - equal,'!=' - not equal, < - smaller, <= - smaller or equal, > - bigger, >= - bigger or equal, '^=' - starts with, '$=' - ends with, '~=' - contains.")
     var filter: String?
 
-    // TODO: Support adding more than one column?
-    @Option(name: .customLong("add"), help: "Adds a new column from a shell command output allowing to substitute other column values into it. Example: --add 'curl http://email-db.com/${email}'.")
+    @Option(name: .customLong("add"), help: "Adds a new column from a shell command output allowing to substitute other column values into it. Expressions ${name} and #{cmd} are substituted by column value and command result respectively. Example: --add 'col_name=#{curl http://email-db.com/${email}}'.")
     var addColumns: [String] = []
 
     @Option(name: .customLong("distinct"), help: "Returns only distinct values for the specified column set. Example: --distinct name,city_id.")
@@ -125,7 +124,21 @@ struct MainApp: ParsableCommand {
 
         if !addColumns.isEmpty {
             // TODO: add support of Dynamic Row values and move validation right before rendering
-            let columns = try addColumns.enumerated().map { (index, element) in ("newColumn\(index + 1)", try Format(format: element).validated(header: table.header)) }
+            let columns = try addColumns.enumerated().map { (index, colDefinition) in
+                let parts = colDefinition.split(separator: "=", maxSplits: 1)
+                if (parts.count != 2) {
+                    throw RuntimeError("Invalid add column format: '--add \(colDefinition)'. Expected format: col_name=format")
+                }
+
+                let colName = String(parts[0]).trimmingCharacters(in: CharacterSet.whitespaces)
+                let formatStr = String(parts[1])
+
+                if (Global.debug) { 
+                    print("Adding a column: \(colName) with format: '\(formatStr)'") 
+                }                
+                return (colName, try Format(format: formatStr).validated(header: table.header)) 
+            }
+
             table = NewColumnsTableView(table: table, additionalColumns: columns)
         }
 

Sources/table/Row.swift

@@ -5,6 +5,22 @@ class Row {
     let components: [Cell]
     let header: Header?
 
+    var dictionary: [String: Cell] {
+        var dict = [String: Cell]()
+
+        if let header = header {
+            for (index, name) in header.components().enumerated() {                
+                dict[name] = components[index]
+            }
+        } else {
+            for (index, cell) in components.enumerated() {
+                dict[index.description] = cell
+            }
+        }
+
+        return dict
+    }
+
     convenience init(header: Header, index: Int, components: [String]) {
         let components = zip(components, header.types).map { Cell(value: $0.0, type: $0.1) }
         self.init(header: header, index: index, cells: components)