diff --git a/components/api/api-database/pom.xml b/components/api/api-database/pom.xml index 0988d6db412..8d472b6e6b0 100644 --- a/components/api/api-database/pom.xml +++ b/components/api/api-database/pom.xml @@ -52,6 +52,10 @@ org.eclipse.dirigible dirigible-components-data-store + + org.eclipse.dirigible + dirigible-components-data-export + diff --git a/components/api/api-database/src/main/java/org/eclipse/dirigible/components/api/db/DatabaseFacade.java b/components/api/api-database/src/main/java/org/eclipse/dirigible/components/api/db/DatabaseFacade.java index b600a046437..4afffd0f2aa 100644 --- a/components/api/api-database/src/main/java/org/eclipse/dirigible/components/api/db/DatabaseFacade.java +++ b/components/api/api-database/src/main/java/org/eclipse/dirigible/components/api/db/DatabaseFacade.java @@ -34,6 +34,7 @@ import org.apache.commons.io.output.WriterOutputStream; import org.eclipse.dirigible.commons.api.helpers.GsonHelper; import org.eclipse.dirigible.components.base.logging.LoggingExecutor; +import org.eclipse.dirigible.components.data.export.service.DataAsyncExportService; import org.eclipse.dirigible.components.data.management.service.DatabaseDefinitionService; import org.eclipse.dirigible.components.data.sources.manager.DataSourcesManager; import org.eclipse.dirigible.components.database.DatabaseParameters; @@ -62,6 +63,8 @@ @Component public class DatabaseFacade implements InitializingBean { + private static final String DEFAULT_DB = "DefaultDB"; + /** The Constant logger. */ private static final Logger logger = LoggerFactory.getLogger(DatabaseFacade.class); @@ -78,6 +81,8 @@ public class DatabaseFacade implements InitializingBean { /** The data sources manager. */ private final DataSourcesManager dataSourcesManager; + private final DataAsyncExportService dataAsyncExportService; + /** * Instantiates a new database facade. * @@ -85,9 +90,11 @@ public class DatabaseFacade implements InitializingBean { * @param dataSourcesManager the data sources manager */ @Autowired - private DatabaseFacade(DatabaseDefinitionService databaseDefinitionService, DataSourcesManager dataSourcesManager) { + private DatabaseFacade(DatabaseDefinitionService databaseDefinitionService, DataSourcesManager dataSourcesManager, + DataAsyncExportService dataAsyncExportService) { this.databaseDefinitionService = databaseDefinitionService; this.dataSourcesManager = dataSourcesManager; + this.dataAsyncExportService = dataAsyncExportService; } /** @@ -147,6 +154,10 @@ public DataSourcesManager getDataSourcesManager() { return dataSourcesManager; } + public DataAsyncExportService getDataAsyncExportService() { + return dataAsyncExportService; + } + /** * Gets the metadata. * @@ -169,7 +180,7 @@ public static DirigibleDataSource getDataSource(String datasourceName) { try { boolean defaultDB = datasourceName == null || datasourceName.trim() .isEmpty() - || "DefaultDB".equals(datasourceName); + || DEFAULT_DB.equals(datasourceName); DirigibleDataSource dataSource = defaultDB ? DatabaseFacade.get() .getDataSourcesManager() .getDefaultDataSource() @@ -1040,4 +1051,15 @@ public static void toJson(ResultSet resultSet, boolean limited, boolean stringif DatabaseResultSetHelper.toJson(resultSet, limited, stringify, output); } + public static void exportToCsv(String sql, String parametersJson, String datasourceName, String fileName) throws SQLException { + if (datasourceName == null || datasourceName.trim() + .isEmpty()) { + datasourceName = DEFAULT_DB; + } + Optional parameters = parseOptionalJson(parametersJson); + DatabaseFacade.get() + .getDataAsyncExportService() + .exportStatement(datasourceName, sql, parameters, Optional.of(fileName)); + } + } diff --git a/components/api/api-modules-javascript/src/main/resources/META-INF/dirigible/modules/src/db/query.sample b/components/api/api-modules-javascript/src/main/resources/META-INF/dirigible/modules/src/db/query.sample index a43dfcf2213..9fe71661407 100644 --- a/components/api/api-modules-javascript/src/main/resources/META-INF/dirigible/modules/src/db/query.sample +++ b/components/api/api-modules-javascript/src/main/resources/META-INF/dirigible/modules/src/db/query.sample @@ -4,20 +4,25 @@ import { response } from "@aerokit/sdk/http"; // Regular const sql = "SELECT * FROM DIRIGIBLE_EXTENSIONS WHERE EXTENSION_EXTENSIONPOINT_NAME = ?"; -let resultset = query.execute(sql, ["ide-editor"], "SystemDB"); +let resultset = query.execute(sql, ["platform-editors"], "SystemDB"); response.println(JSON.stringify(resultset)); // Typed Parameters const sql = "SELECT * FROM DIRIGIBLE_EXTENSIONS WHERE EXTENSION_EXTENSIONPOINT_NAME = ?"; -let resultset = query.execute(sql, [{ "type": "VARCHAR", "value": "ide-editor" }], "SystemDB"); +let resultset = query.execute(sql, [{ "type": "VARCHAR", "value": "platform-editors" }], "SystemDB"); response.println(JSON.stringify(resultset)); // Named Parameters const sql = "SELECT * FROM DIRIGIBLE_EXTENSIONS WHERE EXTENSION_EXTENSIONPOINT_NAME = :editor"; -let resultset = query.executeNamed(sql, [{ "name": "editor", "type": "VARCHAR", "value": "ide-editor" }], "SystemDB"); +let resultset = query.executeNamed(sql, [{ "name": "editor", "type": "VARCHAR", "value": "platform-editors" }], "SystemDB"); response.println(JSON.stringify(resultset)); + +// Export CSV + +const sql = "SELECT * FROM DIRIGIBLE_EXTENSIONS WHERE EXTENSION_EXTENSIONPOINT_NAME = :editor"; +query.exportCsv(sql, [{ "name": "editor", "type": "VARCHAR", "value": "platform-editors" }], "SystemDB", "my-export-file"); diff --git a/components/api/api-modules-javascript/src/main/resources/META-INF/dirigible/modules/src/db/query.ts b/components/api/api-modules-javascript/src/main/resources/META-INF/dirigible/modules/src/db/query.ts index 9eb523fbead..eb499247aa5 100644 --- a/components/api/api-modules-javascript/src/main/resources/META-INF/dirigible/modules/src/db/query.ts +++ b/components/api/api-modules-javascript/src/main/resources/META-INF/dirigible/modules/src/db/query.ts @@ -150,6 +150,32 @@ export class Query { // Parse the JSON string back into a JavaScript array of objects return JSON.parse(resultset); } + + /** + * Exports a SQL query with named parameters (e.g., ":name", ":id"). + * + * @param sql The SQL query to execute. + * @param parameters An optional array of NamedQueryParameter objects. + * @param datasourceName The name of the database connection to use (optional). + * @param fileName The file name pattern. + * @returns An array of records representing the query results. + */ + public static exportCsv( + sql: string, + parameters?: NamedQueryParameter[], + datasourceName?: string, + fileName?: string + ) { + // Serialize the array of named parameters for the Java facade + const paramsJson = parameters ? JSON.stringify(parameters) : undefined; + + DatabaseFacade.exportToCsv( + sql, + paramsJson, + datasourceName, + fileName + ); + } } // @ts-ignore diff --git a/components/data/data-core/src/main/java/org/eclipse/dirigible/components/database/helpers/DatabaseQueryHelper.java b/components/data/data-core/src/main/java/org/eclipse/dirigible/components/database/helpers/DatabaseQueryHelper.java index 90f4795ba72..99f357d3b63 100644 --- a/components/data/data-core/src/main/java/org/eclipse/dirigible/components/database/helpers/DatabaseQueryHelper.java +++ b/components/data/data-core/src/main/java/org/eclipse/dirigible/components/database/helpers/DatabaseQueryHelper.java @@ -17,11 +17,16 @@ import java.util.ArrayList; import java.util.List; import java.util.NavigableMap; +import java.util.Optional; import java.util.TreeMap; +import org.eclipse.dirigible.components.database.NamedParameterStatement; +import org.eclipse.dirigible.components.database.params.ParametersSetter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.gson.JsonElement; + /** * Convenience class for common DataSource operations. An instance represents a single DataSource. */ @@ -119,6 +124,29 @@ public static void executeSingleStatement(Connection connection, String sql, boo } } + public static void executeSingleStatement(Connection connection, String sql, boolean isQuery, Optional parameters, + RequestExecutionCallback callback) { + try { + try (NamedParameterStatement preparedStatement = new NamedParameterStatement(connection, sql)) { + if (parameters.isPresent()) { + ParametersSetter.setNamedParameters(parameters.get(), preparedStatement); + } + if (isQuery) { + try (ResultSet resultSet = preparedStatement.executeQuery()) { + callback.queryDone(resultSet); + } + } else { + preparedStatement.executeUpdate(); + callback.updateDone(preparedStatement.getStatement() + .getUpdateCount()); + } + } + } catch (Exception e) { + logger.error("Failed to execute SQL [{}]", sql, e); + callback.error(e); + } + } + /** * Executes a single SQL procedure. The callbacks are on queryDone in case of query or updateDone in * case of update, and on error. The method does not iterate on the result set and its pointer is in diff --git a/components/data/data-export/src/main/java/org/eclipse/dirigible/components/data/export/endpoint/DataAsyncExportEndpoint.java b/components/data/data-export/src/main/java/org/eclipse/dirigible/components/data/export/endpoint/DataAsyncExportEndpoint.java index 76e6420c5aa..a79c91f20a9 100644 --- a/components/data/data-export/src/main/java/org/eclipse/dirigible/components/data/export/endpoint/DataAsyncExportEndpoint.java +++ b/components/data/data-export/src/main/java/org/eclipse/dirigible/components/data/export/endpoint/DataAsyncExportEndpoint.java @@ -76,7 +76,7 @@ public DataAsyncExportService getDataAsyncExportService() { public ResponseEntity exportStatement(@PathVariable("datasource") String datasource, @Valid @RequestBody String statement, @RequestParam("name") Optional name) throws SQLException { - getDataAsyncExportService().exportStatement(datasource, statement, name); + getDataAsyncExportService().exportStatement(datasource, statement, Optional.empty(), name); return ResponseEntity.ok() .build(); diff --git a/components/data/data-export/src/main/java/org/eclipse/dirigible/components/data/export/service/DataAsyncExportService.java b/components/data/data-export/src/main/java/org/eclipse/dirigible/components/data/export/service/DataAsyncExportService.java index 8369f16ef04..2c1ed45d2a9 100644 --- a/components/data/data-export/src/main/java/org/eclipse/dirigible/components/data/export/service/DataAsyncExportService.java +++ b/components/data/data-export/src/main/java/org/eclipse/dirigible/components/data/export/service/DataAsyncExportService.java @@ -30,6 +30,8 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.server.ResponseStatusException; +import com.google.gson.JsonElement; + @Service public class DataAsyncExportService { @@ -83,14 +85,23 @@ public CmsService getCmsService() { return cmsService; } - public void exportStatement(String datasource, String statement, Optional name) throws SQLException { + public void exportStatement(String datasource, String statement) throws SQLException { + exportStatement(datasource, statement, Optional.empty(), Optional.empty()); + } + + public void exportStatement(String datasource, String statement, Optional parameters) throws SQLException { + exportStatement(datasource, statement, parameters, Optional.empty()); + } + + public void exportStatement(String datasource, String statement, Optional parameters, Optional name) + throws SQLException { if (!databaseMetadataService.existsDataSourceMetadata(datasource)) { String error = format("Datasource {0} does not exist.", datasource); throw new ResponseStatusException(HttpStatus.NOT_FOUND, error); } - launchExportJob(datasource, statement, name); + launchExportJob(datasource, statement, parameters, name); } public List get() throws IOException { @@ -151,7 +162,7 @@ public void deleteAll() throws IOException { private final ExecutorService executorService = Executors.newFixedThreadPool(2); - private void launchExportJob(String datasource, String statement, Optional pattern) { + private void launchExportJob(String datasource, String statement, Optional parameters, Optional pattern) { executorService.submit(() -> { CmisFolder root; try { @@ -165,7 +176,7 @@ private void launchExportJob(String datasource, String statement, Optional { try (PipedOutputStream producerPos = pos) { - databaseExportService.exportStatement(datasource, statement, producerPos); + databaseExportService.exportStatement(datasource, statement, parameters, producerPos); } catch (Exception e) { logger.error("Database export failed.", e); export.setStatus(ExportStatus.FAILED); diff --git a/components/data/data-export/src/main/java/org/eclipse/dirigible/components/data/export/service/DatabaseExportService.java b/components/data/data-export/src/main/java/org/eclipse/dirigible/components/data/export/service/DatabaseExportService.java index bc1943bb33a..beb4531e72e 100644 --- a/components/data/data-export/src/main/java/org/eclipse/dirigible/components/data/export/service/DatabaseExportService.java +++ b/components/data/data-export/src/main/java/org/eclipse/dirigible/components/data/export/service/DatabaseExportService.java @@ -11,10 +11,12 @@ import java.io.IOException; import java.io.OutputStream; +import java.io.PipedOutputStream; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Optional; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -129,6 +131,14 @@ public void exportStatement(String datasource, String statement, OutputStream ou } + public void exportStatement(String datasource, String statement, Optional parameters, PipedOutputStream output) { + DirigibleDataSource dataSource = datasourceManager.getDataSource(datasource); + if (dataSource != null) { + databaseExecutionService.executeStatement(dataSource, statement, parameters, true, false, true, false, output); + } + + } + /** * Get structure type by datasource. * @@ -246,4 +256,5 @@ public static String getIntegerPrimaryKey(Connection connection, String tableNam return integerPrimaryKey; } + } diff --git a/components/data/data-management/src/main/java/org/eclipse/dirigible/components/data/management/service/DatabaseExecutionService.java b/components/data/data-management/src/main/java/org/eclipse/dirigible/components/data/management/service/DatabaseExecutionService.java index 0cfac589a36..ada4a7ea42b 100644 --- a/components/data/data-management/src/main/java/org/eclipse/dirigible/components/data/management/service/DatabaseExecutionService.java +++ b/components/data/data-management/src/main/java/org/eclipse/dirigible/components/data/management/service/DatabaseExecutionService.java @@ -11,11 +11,13 @@ import java.io.OutputStream; +import java.io.PipedOutputStream; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.StringTokenizer; import java.util.stream.Collectors; @@ -25,6 +27,7 @@ import org.eclipse.dirigible.components.data.sources.domain.DataSource; import org.eclipse.dirigible.components.data.sources.manager.DataSourcesManager; import org.eclipse.dirigible.components.data.sources.service.DataSourceService; +import org.eclipse.dirigible.components.database.DirigibleDataSource; import org.eclipse.dirigible.components.database.helpers.DatabaseErrorHelper; import org.eclipse.dirigible.components.database.helpers.DatabaseQueryHelper; import org.eclipse.dirigible.components.database.helpers.DatabaseResultSetHelper; @@ -34,6 +37,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import com.google.gson.JsonElement; + /** * The Class DataSourceMetadataService. */ @@ -221,6 +226,65 @@ public void error(Throwable t) { } } + public void executeStatement(DirigibleDataSource dataSource, String sql, Optional parameters, boolean isQuery, + boolean isJson, boolean isCsv, boolean limited, OutputStream output) { + if ((sql == null) || (sql.length() == 0)) { + return; + } + + List errors = new ArrayList(); + + StringTokenizer tokenizer = new StringTokenizer(sql, getDelimiter(sql)); + while (tokenizer.hasMoreTokens()) { + String line = tokenizer.nextToken(); + if ("".equals(line.trim())) { + continue; + } + + try (Connection connection = dataSource.getConnection()) { + DatabaseQueryHelper.executeSingleStatement(connection, line, isQuery, parameters, new RequestExecutionCallback() { + @Override + public void updateDone(int recordsCount) {} + + @Override + public void queryDone(ResultSet rs) { + try { + if (isJson) { + DatabaseResultSetHelper.toJson(rs, limited, true, output); + } else if (isCsv) { + DatabaseResultSetHelper.toCsv(rs, limited, false, output); + } else { + DatabaseResultSetHelper.print(rs, limited, output); + } + } catch (Exception e) { + if (logger.isWarnEnabled()) { + logger.warn(e.getMessage(), e); + } + errors.add(e.getMessage()); + } + } + + @Override + public void error(Throwable t) { + if (logger.isWarnEnabled()) { + logger.warn(t.getMessage(), t); + } + errors.add(t.getMessage()); + } + }); + } catch (SQLException e) { + if (logger.isWarnEnabled()) { + logger.warn(e.getMessage(), e); + } + errors.add(e.getMessage()); + } + } + + if (!errors.isEmpty()) { + throw new RuntimeException(DatabaseErrorHelper.print(String.join("\n", errors))); + } + } + /** * Execute procedure. * @@ -302,4 +366,5 @@ private String getDelimiter(String sql) { } return SCRIPT_DELIMITER; } + }