diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Convert.java b/core/src/main/java/org/opensearch/sql/ast/tree/Convert.java index 74406b0daf2..259330b2dba 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Convert.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Convert.java @@ -23,6 +23,7 @@ @RequiredArgsConstructor public class Convert extends UnresolvedPlan { private final List conversions; + private final String timeFormat; private UnresolvedPlan child; @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index edfe0b22e85..c7d3831a947 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -1008,7 +1008,7 @@ public RelNode visitConvert(Convert node, CalcitePlanContext context) { ConversionState state = new ConversionState(); for (Let conversion : node.getConversions()) { - processConversion(conversion, state, context); + processConversion(conversion, node.getTimeFormat(), state, context); } return buildConversionProjection(state, context); @@ -1021,14 +1021,14 @@ private static class ConversionState { } private void processConversion( - Let conversion, ConversionState state, CalcitePlanContext context) { + Let conversion, String timeFormat, ConversionState state, CalcitePlanContext context) { String target = conversion.getVar().getField().toString(); UnresolvedExpression expression = conversion.getExpression(); if (expression instanceof Field) { processFieldCopyConversion(target, (Field) expression, state, context); } else if (expression instanceof Function) { - processFunctionConversion(target, (Function) expression, state, context); + processFunctionConversion(target, (Function) expression, timeFormat, state, context); } else { throw new SemanticCheckException("Convert command requires function call expressions"); } @@ -1051,7 +1051,11 @@ private void processFieldCopyConversion( } private void processFunctionConversion( - String target, Function function, ConversionState state, CalcitePlanContext context) { + String target, + Function function, + String timeFormat, + ConversionState state, + CalcitePlanContext context) { String functionName = function.getFuncName(); List args = function.getFuncArgs(); @@ -1068,8 +1072,7 @@ private void processFunctionConversion( state.seenFields.add(source); RexNode sourceField = context.relBuilder.field(source); - RexNode convertCall = - PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, functionName, sourceField); + RexNode convertCall = resolveConvertFunction(functionName, sourceField, timeFormat, context); if (!target.equals(source)) { state.additions.add(Pair.of(target, context.relBuilder.alias(convertCall, target))); @@ -1078,6 +1081,23 @@ private void processFunctionConversion( } } + private RexNode resolveConvertFunction( + String functionName, RexNode sourceField, String timeFormat, CalcitePlanContext context) { + + // Time functions that support timeformat parameter + Set timeFunctions = Set.of("ctime", "mktime"); + + if (timeFunctions.contains(functionName.toLowerCase()) && timeFormat != null) { + // For time functions with custom timeformat, pass the format as a second parameter + RexNode timeFormatLiteral = context.rexBuilder.makeLiteral(timeFormat); + return PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, functionName, sourceField, timeFormatLiteral); + } else { + // Regular conversion functions or time functions without custom format + return PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, functionName, sourceField); + } + } + private RelNode buildConversionProjection(ConversionState state, CalcitePlanContext context) { List originalFields = context.relBuilder.peek().getRowType().getFieldNames(); List projectList = new ArrayList<>(); diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index abf37e68392..fcd361ba229 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -84,6 +84,10 @@ private PPLOperandTypes() {} UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.INTEGER))); + public static final UDFOperandMetadata ANY_OPTIONAL_STRING = + UDFOperandMetadata.wrap( + (CompositeOperandTypeChecker) + OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.CHARACTER))); public static final UDFOperandMetadata ANY_OPTIONAL_TIMESTAMP = UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java b/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java index f42d376f649..bd0796b05af 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java @@ -249,4 +249,40 @@ private static long extractFirstNDigits(double value, int digits) { return isNegative ? -result : result; } + + /** Mapping from strftime specifiers to Java DateTimeFormatter patterns for parsing. */ + private static final Map STRFTIME_TO_JAVA_PARSE = + ImmutableMap.builder() + .put("%Y", "yyyy") + .put("%y", "yy") + .put("%m", "MM") + .put("%B", "MMMM") + .put("%b", "MMM") + .put("%d", "dd") + .put("%H", "HH") + .put("%I", "hh") + .put("%M", "mm") + .put("%S", "ss") + .put("%p", "a") + .put("%T", "HH:mm:ss") + .put("%F", "yyyy-MM-dd") + .put("%%", "'%'") + .build(); + + /** + * Convert a strftime format string to a Java DateTimeFormatter pattern suitable for parsing. + * + * @param strftimeFormat the strftime-style format string (e.g. {@code %Y-%m-%d %H:%M:%S}) + * @return a Java DateTimeFormatter pattern (e.g. {@code yyyy-MM-dd HH:mm:ss}) + */ + public static String toJavaPattern(String strftimeFormat) { + Matcher m = Pattern.compile("%[A-Za-z%]").matcher(strftimeFormat); + StringBuilder sb = new StringBuilder(); + while (m.find()) { + String replacement = STRFTIME_TO_JAVA_PARSE.getOrDefault(m.group(), m.group()); + m.appendReplacement(sb, Matcher.quoteReplacement(replacement)); + } + m.appendTail(sb); + return sb.toString(); + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 2aebf7efe34..0a5b0fe0e03 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -63,8 +63,12 @@ import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl; import org.opensearch.sql.expression.function.udf.AutoConvertFunction; +import org.opensearch.sql.expression.function.udf.CTimeConvertFunction; import org.opensearch.sql.expression.function.udf.CryptographicFunction; +import org.opensearch.sql.expression.function.udf.Dur2SecConvertFunction; import org.opensearch.sql.expression.function.udf.MemkConvertFunction; +import org.opensearch.sql.expression.function.udf.MkTimeConvertFunction; +import org.opensearch.sql.expression.function.udf.MsTimeConvertFunction; import org.opensearch.sql.expression.function.udf.NumConvertFunction; import org.opensearch.sql.expression.function.udf.ParseFunction; import org.opensearch.sql.expression.function.udf.RelevanceQueryFunction; @@ -431,6 +435,10 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator RMCOMMA = new RmcommaConvertFunction().toUDF("RMCOMMA"); public static final SqlOperator RMUNIT = new RmunitConvertFunction().toUDF("RMUNIT"); public static final SqlOperator MEMK = new MemkConvertFunction().toUDF("MEMK"); + public static final SqlOperator CTIME = new CTimeConvertFunction().toUDF("CTIME"); + public static final SqlOperator MKTIME = new MkTimeConvertFunction().toUDF("MKTIME"); + public static final SqlOperator MSTIME = new MsTimeConvertFunction().toUDF("MSTIME"); + public static final SqlOperator DUR2SEC = new Dur2SecConvertFunction().toUDF("DUR2SEC"); public static final SqlOperator WIDTH_BUCKET = new org.opensearch.sql.expression.function.udf.binning.WidthBucketFunction() diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index fe364c7b7c4..66713b18b80 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -39,6 +39,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.COT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.COUNT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CRC32; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CTIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURDATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_DATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_TIME; @@ -61,6 +62,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.DEGREES; import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDEFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DUR2SEC; import static org.opensearch.sql.expression.function.BuiltinFunctionName.E; import static org.opensearch.sql.expression.function.BuiltinFunctionName.EARLIEST; import static org.opensearch.sql.expression.function.BuiltinFunctionName.EQUAL; @@ -144,12 +146,14 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE_OF_DAY; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE_OF_HOUR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MKTIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MOD; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MODULUS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MODULUSFUNCTION; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTHNAME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH_OF_YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MSTIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLYFUNCTION; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTI_MATCH; @@ -995,6 +999,10 @@ void populate() { registerOperator(RMCOMMA, PPLBuiltinOperators.RMCOMMA); registerOperator(RMUNIT, PPLBuiltinOperators.RMUNIT); registerOperator(MEMK, PPLBuiltinOperators.MEMK); + registerOperator(CTIME, PPLBuiltinOperators.CTIME); + registerOperator(MKTIME, PPLBuiltinOperators.MKTIME); + registerOperator(MSTIME, PPLBuiltinOperators.MSTIME); + registerOperator(DUR2SEC, PPLBuiltinOperators.DUR2SEC); register( TOSTRING, diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java new file mode 100644 index 00000000000..6b507936348 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; +import org.opensearch.sql.calcite.utils.PPLReturnTypes; +import org.opensearch.sql.expression.datetime.StrftimeFormatterUtil; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * PPL ctime() conversion function. Converts UNIX epoch timestamps to human-readable time strings + * using strftime format specifiers. Default format: {@code %m/%d/%Y %H:%M:%S}. + */ +public class CTimeConvertFunction extends ImplementorUDF { + + private static final String DEFAULT_FORMAT = "%m/%d/%Y %H:%M:%S"; + + public CTimeConvertFunction() { + super(new CTimeImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return PPLReturnTypes.STRING_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return PPLOperandTypes.ANY_OPTIONAL_STRING; + } + + public static class CTimeImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + if (translatedOperands.isEmpty()) { + return Expressions.constant(null, String.class); + } + Expression fieldValue = Expressions.box(translatedOperands.get(0)); + if (translatedOperands.size() == 1) { + return Expressions.call(CTimeConvertFunction.class, "convert", fieldValue); + } + Expression timeFormat = Expressions.box(translatedOperands.get(1)); + return Expressions.call( + CTimeConvertFunction.class, "convertWithFormat", fieldValue, timeFormat); + } + } + + public static String convert(Object value) { + return convertWithFormat(value, null); + } + + public static String convertWithFormat(Object value, Object timeFormatObj) { + Double timestamp = toEpochSeconds(value); + if (timestamp == null) { + return null; + } + String format = (timeFormatObj != null) ? timeFormatObj.toString().trim() : DEFAULT_FORMAT; + if (format.isEmpty()) { + return null; + } + try { + long seconds = timestamp.longValue(); + int nanos = (int) ((timestamp - seconds) * 1_000_000_000); + Instant instant = Instant.ofEpochSecond(seconds, nanos); + ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); + return StrftimeFormatterUtil.formatZonedDateTime(zdt, format).stringValue(); + } catch (Exception e) { + return null; + } + } + + public static Double toEpochSeconds(Object value) { + if (value == null) { + return null; + } + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } + String str = value.toString().trim(); + if (str.isEmpty()) { + return null; + } + try { + return Double.parseDouble(str); + } catch (NumberFormatException e) { + return null; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java new file mode 100644 index 00000000000..78facf743be --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** PPL dur2sec() conversion function. Converts duration format {@code [D+]HH:MM:SS} to seconds */ +public class Dur2SecConvertFunction extends BaseConversionUDF { + + public static final Dur2SecConvertFunction INSTANCE = new Dur2SecConvertFunction(); + + // Matches [D+]HH:MM:SS — optional days prefix with + separator + private static final Pattern DURATION_PATTERN = + Pattern.compile("^(?:(\\d+)\\+)?(\\d{1,2}):(\\d{1,2}):(\\d{1,2})$"); + + public Dur2SecConvertFunction() { + super(Dur2SecConvertFunction.class); + } + + public static Object convert(Object value) { + return INSTANCE.convertValue(value); + } + + @Override + protected Object applyConversion(String preprocessedValue) { + Double existingSeconds = tryParseDouble(preprocessedValue); + if (existingSeconds != null) { + return existingSeconds; + } + + Matcher matcher = DURATION_PATTERN.matcher(preprocessedValue); + if (!matcher.matches()) { + return null; + } + + try { + int days = matcher.group(1) != null ? Integer.parseInt(matcher.group(1)) : 0; + int hours = Integer.parseInt(matcher.group(2)); + int minutes = Integer.parseInt(matcher.group(3)); + int seconds = Integer.parseInt(matcher.group(4)); + + if (hours >= 24 || minutes >= 60 || seconds >= 60) { + return null; + } + + return (double) (days * 86400 + hours * 3600 + minutes * 60 + seconds); + } catch (NumberFormatException e) { + return null; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java new file mode 100644 index 00000000000..0127d63e9cd --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MkTimeConvertFunction.java @@ -0,0 +1,106 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.List; +import java.util.Locale; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; +import org.opensearch.sql.expression.datetime.StrftimeFormatterUtil; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * PPL mktime() conversion function. Parses a human-readable time string into UNIX epoch seconds + * using strftime format specifiers. Default format: {@code %m/%d/%Y %H:%M:%S}. + */ +public class MkTimeConvertFunction extends ImplementorUDF { + + public static final MkTimeConvertFunction INSTANCE = new MkTimeConvertFunction(); + + private static final String DEFAULT_FORMAT = "%m/%d/%Y %H:%M:%S"; + + public MkTimeConvertFunction() { + super(new MkTimeImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return ReturnTypes.explicit( + factory -> + factory.createTypeWithNullability(factory.createSqlType(SqlTypeName.DOUBLE), true)); + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return PPLOperandTypes.ANY_OPTIONAL_STRING; + } + + public static class MkTimeImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + if (translatedOperands.isEmpty()) { + return Expressions.constant(null, Double.class); + } + Expression fieldValue = Expressions.box(translatedOperands.get(0)); + if (translatedOperands.size() == 1) { + return Expressions.call(MkTimeConvertFunction.class, "convert", fieldValue); + } + Expression timeFormat = Expressions.box(translatedOperands.get(1)); + return Expressions.call( + MkTimeConvertFunction.class, "convertWithFormat", fieldValue, timeFormat); + } + } + + public static Object convert(Object value) { + return convertWithFormat(value, null); + } + + public static Object convertWithFormat(Object value, Object timeFormatObj) { + Double numeric = CTimeConvertFunction.toEpochSeconds(value); + if (numeric != null) { + return numeric; + } + if (value == null) { + return null; + } + String str = value instanceof String ? ((String) value).trim() : value.toString().trim(); + if (str.isEmpty()) { + return null; + } + + String strftimeFormat = + (timeFormatObj != null) ? timeFormatObj.toString().trim() : DEFAULT_FORMAT; + if (strftimeFormat.isEmpty()) { + return null; + } + return parseWithFormat(str, strftimeFormat); + } + + private static Object parseWithFormat(String dateStr, String strftimeFormat) { + try { + String javaPattern = StrftimeFormatterUtil.toJavaPattern(strftimeFormat); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(javaPattern, Locale.ROOT); + LocalDateTime dateTime = LocalDateTime.parse(dateStr, formatter); + return (double) dateTime.toEpochSecond(ZoneOffset.UTC); + } catch (DateTimeParseException | IllegalArgumentException e) { + return null; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java new file mode 100644 index 00000000000..362896b06b9 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/MsTimeConvertFunction.java @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * PPL mstime() conversion function. Converts {@code [MM:]SS.SSS} format to seconds The minutes + * portion is optional. + */ +public class MsTimeConvertFunction extends BaseConversionUDF { + + public static final MsTimeConvertFunction INSTANCE = new MsTimeConvertFunction(); + + // Matches optional MM: prefix, required SS, optional .SSS + private static final Pattern MSTIME_PATTERN = + Pattern.compile("^(?:(\\d{1,2}):)?(\\d{1,2})(?:\\.(\\d{1,3}))?$"); + + public MsTimeConvertFunction() { + super(MsTimeConvertFunction.class); + } + + public static Object convert(Object value) { + return INSTANCE.convertValue(value); + } + + @Override + protected Object applyConversion(String preprocessedValue) { + Double existingSeconds = tryParseDouble(preprocessedValue); + if (existingSeconds != null) { + return existingSeconds; + } + + Matcher matcher = MSTIME_PATTERN.matcher(preprocessedValue); + if (!matcher.matches()) { + return null; + } + + try { + int minutes = matcher.group(1) != null ? Integer.parseInt(matcher.group(1)) : 0; + int seconds = Integer.parseInt(matcher.group(2)); + + if (seconds >= 60) { + return null; + } + + double millis = 0.0; + if (matcher.group(3) != null) { + String milliStr = matcher.group(3); + // Pad to 3 digits + while (milliStr.length() < 3) { + milliStr += "0"; + } + millis = Double.parseDouble(milliStr.substring(0, 3)) / 1000.0; + } + + return (double) (minutes * 60 + seconds) + millis; + } catch (NumberFormatException e) { + return null; + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java index 163d6508445..490f72ba346 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ConversionFunctionsTest.java @@ -336,4 +336,132 @@ public void testRmunitConvertNumericExtremes() { assertEquals(1.7e308, RmunitConvertFunction.convert("1.7e308")); assertEquals(-1.7e308, RmunitConvertFunction.convert("-1.7e308")); } + + // ctime() Function Tests + @Test + public void testCtimeConvertBasic() { + // Default format is %m/%d/%Y %H:%M:%S + assertEquals("10/18/2003 20:07:13", CTimeConvertFunction.convert(1066507633)); + assertEquals("01/01/1970 00:00:00", CTimeConvertFunction.convert(0)); + assertEquals("10/18/2003 20:07:13", CTimeConvertFunction.convert("1066507633")); + } + + @Test + public void testCtimeConvertInvalid() { + assertNull(CTimeConvertFunction.convert("invalid")); + assertNull(CTimeConvertFunction.convert(null)); + assertNull(CTimeConvertFunction.convert("")); + assertNull(CTimeConvertFunction.convert("abc123")); + } + + // mktime() Function Tests + @Test + public void testMktimeConvertBasic() { + // Default format is %m/%d/%Y %H:%M:%S + assertEquals(1066507633.0, MkTimeConvertFunction.convert("10/18/2003 20:07:13")); + assertEquals(946684800.0, MkTimeConvertFunction.convert("01/01/2000 00:00:00")); + assertEquals(1066473433.0, MkTimeConvertFunction.convert(1066473433)); + assertEquals(1066473433.0, MkTimeConvertFunction.convert("1066473433")); + } + + @Test + public void testMktimeConvertInvalid() { + assertNull(MkTimeConvertFunction.convert("invalid")); + assertNull(MkTimeConvertFunction.convert(null)); + assertNull(MkTimeConvertFunction.convert("")); + assertNull(MkTimeConvertFunction.convert("not-a-date")); + } + + // mstime() Function Tests + @Test + public void testMstimeConvertBasic() { + assertEquals(225.0, MsTimeConvertFunction.convert("03:45")); + assertEquals(225.123, MsTimeConvertFunction.convert("03:45.123")); + assertEquals(90.5, MsTimeConvertFunction.convert("01:30.5")); + assertEquals(3661.0, MsTimeConvertFunction.convert("61:01")); + + // SS.SSS without MM: prefix + assertEquals(45.123, MsTimeConvertFunction.convert("45.123")); + assertEquals(30.0, MsTimeConvertFunction.convert("30")); + + // Test already numeric + assertEquals(225.0, MsTimeConvertFunction.convert(225)); + assertEquals(225.0, MsTimeConvertFunction.convert("225")); + } + + @Test + public void testMstimeConvertEdgeCases() { + assertEquals(0.0, MsTimeConvertFunction.convert("00:00")); + assertEquals(0.001, MsTimeConvertFunction.convert("00:00.001")); + assertEquals(59.999, MsTimeConvertFunction.convert("00:59.999")); + } + + @Test + public void testMstimeConvertInvalid() { + assertNull(MsTimeConvertFunction.convert("invalid")); + assertNull(MsTimeConvertFunction.convert(null)); + assertNull(MsTimeConvertFunction.convert("")); + assertNull(MsTimeConvertFunction.convert("25:70")); + assertNull(MsTimeConvertFunction.convert("1:2:3")); + } + + // dur2sec() Function Tests + @Test + public void testDur2secConvertBasic() { + assertEquals(5025.0, Dur2SecConvertFunction.convert("01:23:45")); + assertEquals(3661.0, Dur2SecConvertFunction.convert("01:01:01")); + assertEquals(217815.0, Dur2SecConvertFunction.convert("2+12:30:15")); + assertEquals(90061.0, Dur2SecConvertFunction.convert("1+01:01:01")); + assertEquals(5025.0, Dur2SecConvertFunction.convert(5025)); + assertEquals(5025.0, Dur2SecConvertFunction.convert("5025")); + } + + @Test + public void testDur2secConvertEdgeCases() { + assertEquals(0.0, Dur2SecConvertFunction.convert("00:00:00")); + assertEquals(86400.0, Dur2SecConvertFunction.convert("1+00:00:00")); + assertEquals(3599.0, Dur2SecConvertFunction.convert("00:59:59")); + } + + @Test + public void testDur2secConvertInvalid() { + assertNull(Dur2SecConvertFunction.convert("invalid")); + assertNull(Dur2SecConvertFunction.convert(null)); + assertNull(Dur2SecConvertFunction.convert("")); + assertNull(Dur2SecConvertFunction.convert("25:70:80")); + assertNull(Dur2SecConvertFunction.convert("1:2")); + assertNull(Dur2SecConvertFunction.convert("1+2")); + } + + // timeformat tests for mktime() and ctime() + @Test + public void testMktimeWithCustomTimeformat() { + // Strftime format specifiers + assertEquals( + 1066507633.0, + MkTimeConvertFunction.convertWithFormat("18/10/2003 20:07:13", "%d/%m/%Y %H:%M:%S")); + assertEquals( + 1066507633.0, + MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "%Y-%m-%d %H:%M:%S")); + assertEquals( + 946684800.0, + MkTimeConvertFunction.convertWithFormat("01/01/2000 00:00:00", "%d/%m/%Y %H:%M:%S")); + + // Invalid format returns null + assertNull(MkTimeConvertFunction.convertWithFormat("2003-10-18 20:07:13", "invalid format")); + + assertNull(MkTimeConvertFunction.convertWithFormat("10/18/2003 20:07:13", "")); + } + + @Test + public void testCtimeWithCustomTimeformat() { + // Strftime format specifiers + assertEquals( + "2003-10-18 20:07:13", + CTimeConvertFunction.convertWithFormat(1066507633, "%Y-%m-%d %H:%M:%S")); + assertEquals("18/10/2003", CTimeConvertFunction.convertWithFormat(1066507633, "%d/%m/%Y")); + assertEquals("1970", CTimeConvertFunction.convertWithFormat(0, "%Y")); + + assertNull(CTimeConvertFunction.convertWithFormat(1066507633, "")); + } } diff --git a/docs/user/ppl/cmd/convert.md b/docs/user/ppl/cmd/convert.md index b3fbb7d3577..457ec2563e0 100644 --- a/docs/user/ppl/cmd/convert.md +++ b/docs/user/ppl/cmd/convert.md @@ -7,29 +7,34 @@ The `convert` command uses conversion functions to transform field values into n The `convert` command has the following syntax: ```syntax -convert () [AS ] [, () [AS ]]... +convert [timeformat=] () [AS ] [, () [AS ]]... ``` ## Parameters The `convert` command supports the following parameters. -| Parameter | Required/Optional | Description | -| --- | --- | --- | -| `` | Required | One of the conversion functions: `auto()`, `num()`, `rmcomma()`, `rmunit()`, `memk()`, or `none()`. | -| `` | Required | Single field name to convert. | -| `AS ` | Optional | Create new field with converted value, preserving original field. | +| Parameter | Required/Optional | Description | Default | +| --- | --- | --- | --- | +| `` | Required | One of the conversion functions: `auto()`, `ctime()`, `dur2sec()`, `memk()`, `mktime()`, `mstime()`, `none()`, `num()`, `rmcomma()`, or `rmunit()`. | N/A | +| `` | Required | Single field name to convert. | N/A | +| `AS ` | Optional | Create new field with converted value, preserving original field. | N/A | +| `timeformat=` | Optional | A strftime format string used by `ctime()` and `mktime()`. | `%m/%d/%Y %H:%M:%S`. | ## Conversion Functions | Function | Description | | --- | --- | | `auto(field)` | Automatically converts fields to numbers using intelligent conversion. Handles memory sizes (k/m/g), commas, units, and scientific notation. Returns `null` for non-convertible values. | +| `ctime(field)` | Converts a UNIX epoch timestamp to a human-readable time string. Uses the `timeformat` parameter if specified, otherwise defaults to `%m/%d/%Y %H:%M:%S`. All timestamps are interpreted in UTC timezone. | +| `dur2sec(field)` | Converts a duration string in `HH:MM:SS` format to total seconds. Hours must be less than 24. Returns `null` for invalid formats. | +| `memk(field)` | Converts memory size strings to kilobytes. Accepts numbers with optional k/m/g suffix (case-insensitive). Default unit is kilobytes. Returns `null` for invalid formats. | +| `mktime(field)` | Converts a human-readable time string to a UNIX epoch timestamp. Uses the `timeformat` parameter if specified, otherwise defaults to `%m/%d/%Y %H:%M:%S`. Input strings are interpreted as UTC timezone. | +| `mstime(field)` | Converts a time string in `[MM:]SS.SSS` format to total seconds. The minutes portion is optional. Returns `null` for invalid formats. | +| `none(field)` | No-op function that preserves the original field value. | | `num(field)` | Extracts leading numbers from strings. For strings without letters: removes commas as thousands separators. For strings with letters: extracts leading number, stops at letters or commas. Returns `null` for non-convertible values. | | `rmcomma(field)` | Removes commas from field values and converts to a number. Returns `null` if the value contains letters. | | `rmunit(field)` | Extracts leading numeric values from strings. Stops at the first non-numeric character (including commas). Returns `null` for non-convertible values. | -| `memk(field)` | Converts memory size strings to kilobytes. Accepts numbers with optional k/m/g suffix (case-insensitive). Default unit is kilobytes. Returns `null` for invalid formats. | -| `none(field)` | No-op function that preserves the original field value. Used for excluding specific fields from wildcard conversions. | ## Example 1: Basic auto() conversion @@ -241,6 +246,128 @@ fetched rows / total rows = 3/3 **Note:** The `none()` function is particularly useful when wildcard support is implemented, allowing you to exclude specific fields from bulk conversions. +## Example 9: Convert epoch timestamp to time string with ctime() + +```ppl +source=accounts +| eval timestamp = 1066507633 +| convert ctime(timestamp) +| fields timestamp +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++---------------------+ +| timestamp | +|---------------------| +| 10/18/2003 20:07:13 | ++---------------------+ +``` + +## Example 10: Convert time string to epoch with mktime() + +```ppl +source=accounts +| eval date_str = '10/18/2003 20:07:13' +| convert mktime(date_str) +| fields date_str +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++--------------+ +| date_str | +|--------------| +| 1.066507633E9| ++--------------+ +``` + +## Example 11: Using timeformat with ctime() and mktime() + +The `timeformat` parameter specifies a strftime format string for `ctime()` and `mktime()`: + +```ppl +source=accounts +| eval timestamp = 1066507633 +| convert timeformat="%Y-%m-%d %H:%M:%S" ctime(timestamp) +| fields timestamp +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++---------------------+ +| timestamp | +|---------------------| +| 2003-10-18 20:07:13 | ++---------------------+ +``` + +Similarly, you can use `timeformat` with `mktime()` to parse dates in custom formats: + +```ppl +source=accounts +| eval date_str = '2000-01-01 00:00:00' +| convert timeformat="%Y-%m-%d %H:%M:%S" mktime(date_str) +| fields date_str +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++------------+ +| date_str | +|------------| +| 9.466848E8 | ++------------+ +``` + +## Example 12: Convert duration to seconds with dur2sec() + +```ppl +source=accounts +| eval duration = '01:23:45' +| convert dur2sec(duration) +| fields duration +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++----------+ +| duration | +|----------| +| 5025.0 | ++----------+ +``` + +## Example 13: Convert minutes and seconds with mstime() + +```ppl +source=accounts +| eval time_str = '03:45.5' +| convert mstime(time_str) +| fields time_str +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++----------+ +| time_str | +|----------| +| 225.5 | ++----------+ +``` + ## Notes - All conversion functions return `null` for values that cannot be converted to a number diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java index 1c9b6de3454..7d666951b1a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteConvertCommandIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.remote; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -260,4 +261,118 @@ public void testConvertAutoWithMemorySizesGigabytes() throws IOException { verifySchema(result, schema("memory", null, "double")); verifyDataRows(result, rows(2097152.0)); } + + @Test + public void testConvertMktimeWithDefaultFormat() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval date_str = '10/18/2003 20:07:13' | convert" + + " mktime(date_str) | fields date_str | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("date_str", null, "double")); + verifyDataRows(result, rows(1066507633.0)); + } + + @Test + public void testConvertMktimeWithCustomTimeformat() throws IOException { + JSONObject result = + executeQuery( + "search source=" + + TEST_INDEX_BANK + + " | eval date_str = '18/10/2003 20:07:13' | convert timeformat=\\\"%d/%m/%Y" + + " %H:%M:%S\\\" mktime(date_str) | fields date_str | head 1"); + verifySchema(result, schema("date_str", null, "double")); + verifyDataRows(result, rows(1066507633.0)); + } + + @Test + public void testConvertCtimeWithDefaultFormat() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval timestamp = 1066507633 | convert ctime(timestamp) |" + + " fields timestamp | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("timestamp", null, "string")); + verifyDataRows(result, rows("10/18/2003 20:07:13")); + } + + @Test + public void testConvertCtimeWithCustomTimeformat() throws IOException { + JSONObject result = + executeQuery( + "search source=" + + TEST_INDEX_BANK + + " | eval timestamp = 1066507633 | convert timeformat=\\\"%Y-%m-%d %H:%M:%S\\\"" + + " ctime(timestamp) | fields timestamp | head 1"); + verifySchema(result, schema("timestamp", null, "string")); + verifyDataRows(result, rows("2003-10-18 20:07:13")); + } + + @Test + public void testConvertDur2secFunction() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval duration = '01:23:45' | convert dur2sec(duration) |" + + " fields duration | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("duration", null, "double")); + verifyDataRows(result, rows(5025.0)); + } + + @Test + public void testConvertMstimeFunction() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval time_str = '03:45' | convert mstime(time_str) |" + + " fields time_str | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("time_str", null, "double")); + verifyDataRows(result, rows(225.0)); + } + + @Test + public void testConvertTimeformatWithMultipleFunctions() throws IOException { + JSONObject result = + executeQuery( + "search source=" + + TEST_INDEX_BANK + + " | eval date_str = '18/10/2003 20:07:13', timestamp = 1066507633 | convert" + + " timeformat=\\\"%d/%m/%Y %H:%M:%S\\\" mktime(date_str), ctime(timestamp) |" + + " fields date_str, timestamp | head 1"); + verifySchema(result, schema("date_str", null, "double"), schema("timestamp", null, "string")); + verifyNumOfRows(result, 1); + assertEquals(1066507633.0, result.getJSONArray("datarows").getJSONArray(0).getDouble(0), 0.001); + assertEquals( + "18/10/2003 20:07:13", result.getJSONArray("datarows").getJSONArray(0).getString(1)); + } + + @Test + public void testConvertTimeformatWithWhere() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s | eval date_str = '10/18/2003 20:07:13' |" + + " convert mktime(date_str) | where date_str > 1000000000 |" + + " fields date_str | head 1", + TEST_INDEX_BANK)); + verifySchema(result, schema("date_str", null, "double")); + verifyDataRows(result, rows(1066507633.0)); + } + + @Test + public void testConvertTimeformatWithStats() throws IOException { + JSONObject result = + executeQuery( + "search source=" + + TEST_INDEX_BANK + + " | eval timestamp = 1066507633 |" + + " convert timeformat=\\\"%Y\\\" ctime(timestamp) |" + + " stats count() by timestamp"); + verifySchema(result, schema("count()", null, "bigint"), schema("timestamp", "string")); + verifyDataRows(result, rows(7, "2003")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 73531a8895c..f6730e53d69 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2420,6 +2420,46 @@ public void testConvertMultipleFunctionsExplain() throws IOException { + " balance, age")); } + @Test + public void testConvertCtimeExplain() throws IOException { + String expected = loadExpectedPlan("explain_convert_ctime.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_bank | eval ts=1066507633 | convert ctime(ts) |" + + " fields ts")); + } + + @Test + public void testConvertMktimeExplain() throws IOException { + String expected = loadExpectedPlan("explain_convert_mktime.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_bank | eval d='10/18/2003 20:07:13' | convert" + + " mktime(d) | fields d")); + } + + @Test + public void testConvertDur2secExplain() throws IOException { + String expected = loadExpectedPlan("explain_convert_dur2sec.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_bank | eval d='01:23:45' | convert dur2sec(d) |" + + " fields d")); + } + + @Test + public void testConvertMstimeExplain() throws IOException { + String expected = loadExpectedPlan("explain_convert_mstime.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_bank | eval t='03:45.5' | convert mstime(t) |" + + " fields t")); + } + @Test public void testNotBetweenPushDownExplain() throws Exception { // test for issue https://github.com/opensearch-project/sql/issues/4903 diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java index 099992c9298..b1c794130b0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ConvertCommandIT.java @@ -69,6 +69,38 @@ public void testConvertWithStats() { "source=%s | convert auto(balance) | stats avg(balance) by gender"); } + @Test + public void testConvertMktimeFunction() { + verifyQueryThrowsCalciteError( + "source=%s | eval date_str = '2003-10-18 20:07:13' | convert mktime(date_str) | fields" + + " date_str"); + } + + @Test + public void testConvertCtimeFunction() { + verifyQueryThrowsCalciteError( + "source=%s | eval timestamp = 1066507633 | convert ctime(timestamp) | fields timestamp"); + } + + @Test + public void testConvertDur2secFunction() { + verifyQueryThrowsCalciteError( + "source=%s | eval duration = '01:23:45' | convert dur2sec(duration) | fields duration"); + } + + @Test + public void testConvertMstimeFunction() { + verifyQueryThrowsCalciteError( + "source=%s | eval time_str = '03:45' | convert mstime(time_str) | fields time_str"); + } + + @Test + public void testConvertWithTimeformat() { + verifyQueryThrowsCalciteError( + "source=%s | eval date_str = '18/10/2003 20:07:13' | convert" + + " timeformat=\\\"%%d/%%m/%%Y %%H:%%M:%%S\\\" mktime(date_str) | fields date_str"); + } + private void verifyQueryThrowsCalciteError(String query) { Exception e = assertThrows(Exception.class, () -> executeQuery(String.format(query, TEST_INDEX_BANK))); diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_ctime.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_ctime.yaml new file mode 100644 index 00000000000..dd3c53dc0da --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_ctime.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(ts=[CTIME(1066507633)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1066507633], expr#20=[CTIME($t19)], ts=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m"}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_dur2sec.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_dur2sec.yaml new file mode 100644 index 00000000000..fdbe6f1e8b7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_dur2sec.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(d=[DUR2SEC('01:23:45':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['01:23:45':VARCHAR], expr#20=[DUR2SEC($t19)], d=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m"}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mktime.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mktime.yaml new file mode 100644 index 00000000000..a817226a708 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mktime.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(d=[MKTIME('10/18/2003 20:07:13':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['10/18/2003 20:07:13':VARCHAR], expr#20=[MKTIME($t19)], d=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m"}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mstime.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mstime.yaml new file mode 100644 index 00000000000..43cc390ac77 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_convert_mstime.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(t=[MSTIME('03:45.5':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['03:45.5':VARCHAR], expr#20=[MSTIME($t19)], t=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m"}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_ctime.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_ctime.yaml new file mode 100644 index 00000000000..e93685e279d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_ctime.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(ts=[CTIME(1066507633)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1066507633], expr#20=[CTIME($t19)], ts=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_dur2sec.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_dur2sec.yaml new file mode 100644 index 00000000000..3f01b02a624 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_dur2sec.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(d=[DUR2SEC('01:23:45':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['01:23:45':VARCHAR], expr#20=[DUR2SEC($t19)], d=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mktime.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mktime.yaml new file mode 100644 index 00000000000..2367ea48feb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mktime.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(d=[MKTIME('10/18/2003 20:07:13':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['10/18/2003 20:07:13':VARCHAR], expr#20=[MKTIME($t19)], d=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mstime.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mstime.yaml new file mode 100644 index 00000000000..9bd873d1b3e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_convert_mstime.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(t=[MSTIME('03:45.5':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['03:45.5':VARCHAR], expr#20=[MSTIME($t19)], t=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 53cb4eda36c..ddd945572e0 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -543,7 +543,7 @@ replacementPair ; convertCommand - : CONVERT convertFunction (COMMA? convertFunction)* + : CONVERT (TIMEFORMAT EQUAL timeFormat=stringLiteral)? convertFunction (COMMA? convertFunction)* ; convertFunction diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index c422470bd39..c87cc239399 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -1212,12 +1212,20 @@ public UnresolvedPlan visitConvertCommand(OpenSearchPPLParser.ConvertCommandCont .map(this::buildConversion) .filter(conversion -> conversion != null) .collect(Collectors.toList()); - return new Convert(conversions); + + String timeFormat = null; + if (ctx.timeFormat != null) { + timeFormat = StringUtils.unquoteText(ctx.timeFormat.getText()); + } + + return new Convert(conversions, timeFormat); } /** Supported PPL convert function names (case-insensitive). */ private static final Set SUPPORTED_CONVERSION_FUNCTIONS = - Set.of("auto", "num", "rmcomma", "rmunit", "memk", "none"); + Set.of( + "auto", "num", "rmcomma", "rmunit", "memk", "none", "ctime", "mktime", "dur2sec", + "mstime"); private Let buildConversion(OpenSearchPPLParser.ConvertFunctionContext funcCtx) { if (funcCtx.fieldExpression().isEmpty()) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 53c1ab71e63..611b36956ca 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -528,7 +528,11 @@ public String visitConvert(Convert node, String context) { return StringUtils.format("%s(%s)%s", functionName, fields, asClause); }) .collect(Collectors.joining(",")); - return StringUtils.format("%s | convert %s", child, conversions); + String timeformatClause = + node.getTimeFormat() != null + ? StringUtils.format("timeformat=\"%s\" ", node.getTimeFormat()) + : ""; + return StringUtils.format("%s | convert %s%s", child, timeformatClause, conversions); } @Override diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLConvertTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLConvertTest.java index 936b4212f4f..f49a967aa86 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLConvertTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLConvertTest.java @@ -269,4 +269,139 @@ public void testConvertAutoWithMemoryField() { + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testConvertMktimeFunction() { + String ppl = "source=EMP | convert mktime(ENAME)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1)], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, MKTIME(`ENAME`) `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`," + + " `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertCtimeFunction() { + String ppl = "source=EMP | convert ctime(SAL)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[CTIME($5)]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, CTIME(`SAL`) `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertDur2secFunction() { + String ppl = "source=EMP | convert dur2sec(ENAME)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[DUR2SEC($1)], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, DUR2SEC(`ENAME`) `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`," + + " `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertMstimeFunction() { + String ppl = "source=EMP | convert mstime(ENAME)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[MSTIME($1)], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, MSTIME(`ENAME`) `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`," + + " `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertWithTimeformatMktime() { + String ppl = "source=EMP | convert timeformat=\"%Y-%m-%d\" mktime(ENAME)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1, '%Y-%m-%d')], JOB=[$2], MGR=[$3]," + + " HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, MKTIME(`ENAME`, '%Y-%m-%d') `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`," + + " `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertWithTimeformatCtime() { + String ppl = "source=EMP | convert timeformat=\"%Y-%m-%d %H:%M:%S\" ctime(SAL)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[CTIME($5," + + " '%Y-%m-%d %H:%M:%S')], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, CTIME(`SAL`, '%Y-%m-%d %H:%M:%S')" + + " `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertTimeformatWithMultipleFunctions() { + String ppl = "source=EMP | convert timeformat=\"%Y-%m-%d\" mktime(ENAME), ctime(SAL)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1, '%Y-%m-%d')], JOB=[$2], MGR=[$3]," + + " HIREDATE=[$4], SAL=[CTIME($5, '%Y-%m-%d')], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, MKTIME(`ENAME`, '%Y-%m-%d') `ENAME`, `JOB`, `MGR`, `HIREDATE`," + + " CTIME(`SAL`, '%Y-%m-%d') `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testConvertTimeformatMixedWithNonTimeFunctions() { + String ppl = "source=EMP | convert timeformat=\"%Y-%m-%d\" mktime(ENAME), auto(SAL)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[MKTIME($1, '%Y-%m-%d')], JOB=[$2], MGR=[$3]," + + " HIREDATE=[$4], SAL=[AUTO($5)], COMM=[$6], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, MKTIME(`ENAME`, '%Y-%m-%d') `ENAME`, `JOB`, `MGR`, `HIREDATE`, AUTO(`SAL`)" + + " `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 7aeba3d6c98..866c45a26e0 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -1127,6 +1127,26 @@ public void testConvertCommand() { assertEquals( "source=table | convert (identifier) AS identifier", anonymize("source=t | convert none(empno) AS empno_same")); + assertEquals( + "source=table | convert dur2sec(identifier)", + anonymize("source=t | convert dur2sec(duration)")); + assertEquals( + "source=table | convert mstime(identifier)", + anonymize("source=t | convert mstime(elapsed)")); + assertEquals( + "source=table | convert memk(identifier) AS identifier", + anonymize("source=t | convert memk(virt) AS virt_kb")); + } + + @Test + public void testConvertCommandWithTimeformat() { + assertEquals( + "source=table | convert timeformat=\"%Y-%m-%d\" mktime(identifier)", + anonymize("source=t | convert timeformat=\"%Y-%m-%d\" mktime(date_str)")); + assertEquals( + "source=table | convert timeformat=\"%m/%d/%Y %H:%M:%S\" ctime(identifier) AS identifier", + anonymize( + "source=t | convert timeformat=\"%m/%d/%Y %H:%M:%S\" ctime(ts) AS formatted_time")); } @Test