diff --git a/cassandra-plugins/pom.xml b/cassandra-plugins/pom.xml index 53769b2e8..3a2ad9f53 100644 --- a/cassandra-plugins/pom.xml +++ b/cassandra-plugins/pom.xml @@ -21,7 +21,7 @@ io.cdap.plugin hydrator-plugins - 2.11.5 + 2.11.6-SNAPSHOT Cassandra Plugins diff --git a/core-plugins/pom.xml b/core-plugins/pom.xml index a98704758..c06d05c25 100644 --- a/core-plugins/pom.xml +++ b/core-plugins/pom.xml @@ -23,7 +23,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT Hydrator Core Plugins @@ -299,6 +299,11 @@ org.mockito mockito-core + + com.google.api-client + google-api-client + 1.32.1 + diff --git a/core-plugins/src/main/java/io/cdap/plugin/batch/action/FileMoveAction.java b/core-plugins/src/main/java/io/cdap/plugin/batch/action/FileMoveAction.java index acd8fb9e4..898bad6ca 100644 --- a/core-plugins/src/main/java/io/cdap/plugin/batch/action/FileMoveAction.java +++ b/core-plugins/src/main/java/io/cdap/plugin/batch/action/FileMoveAction.java @@ -16,6 +16,7 @@ package io.cdap.plugin.batch.action; +import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.common.annotations.VisibleForTesting; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; @@ -70,7 +71,23 @@ public void run(ActionContext context) throws Exception { Path dest = new Path(config.destPath); FileSystem fileSystem = source.getFileSystem(new Configuration()); - fileSystem.mkdirs(dest.getParent()); + try { + fileSystem.mkdirs(dest.getParent()); + LOG.debug("mkdirs(): ensured destination parent directory exists: {}", dest.getParent()); + } catch (IOException exception) { + if (exception instanceof GoogleJsonResponseException) { + GoogleJsonResponseException googleJsonResponseException = (GoogleJsonResponseException) exception; + if (googleJsonResponseException.getStatusCode() == 412 || (googleJsonResponseException.getStatusMessage() + != null && googleJsonResponseException.getStatusMessage().contains("412 Precondition Failed"))) { + // Expected GCS directory marker race condition — safe to ignore + LOG.info("Directory already exists in GCS, skipping creation: {}", dest.getParent()); + return; + } + throw googleJsonResponseException; + } else { + throw exception; + } + } if (fileSystem.getFileStatus(source).isFile()) { //moving single file @@ -119,7 +136,23 @@ public boolean accept(Path path) { throw new IllegalArgumentException(String.format("destPath %s needs to be a directory since sourcePath is a " + "directory", config.destPath)); } - fileSystem.mkdirs(dest); //create destination directory if necessary + try { + fileSystem.mkdirs(dest); //create destination directory if necessary + LOG.debug("mkdirs(): ensured destination parent directory exists: {}", dest); + } catch (IOException ex) { + if (ex instanceof GoogleJsonResponseException) { + GoogleJsonResponseException googleJsonResponseException = (GoogleJsonResponseException) ex; + if (googleJsonResponseException.getStatusCode() == 412 || (googleJsonResponseException.getStatusMessage() + != null && googleJsonResponseException.getStatusMessage().contains("412 Precondition Failed"))) { + // Expected GCS directory marker race condition — safe to ignore + LOG.info("Directory already exists in GCS, skipping creation: {}", dest); + return; + } + throw googleJsonResponseException; + } else { + throw ex; + } + } for (FileStatus file : listFiles) { source = file.getPath(); diff --git a/database-plugins/pom.xml b/database-plugins/pom.xml index 366558864..1dc8a3eda 100644 --- a/database-plugins/pom.xml +++ b/database-plugins/pom.xml @@ -20,7 +20,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT Database Plugins diff --git a/format-avro/pom.xml b/format-avro/pom.xml index 4feb6e7ac..f693d88a7 100644 --- a/format-avro/pom.xml +++ b/format-avro/pom.xml @@ -21,7 +21,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT 4.0.0 diff --git a/format-blob/pom.xml b/format-blob/pom.xml index e020c885a..a25299e0b 100644 --- a/format-blob/pom.xml +++ b/format-blob/pom.xml @@ -21,7 +21,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT 4.0.0 diff --git a/format-common/pom.xml b/format-common/pom.xml index 393a9b7f8..370279963 100644 --- a/format-common/pom.xml +++ b/format-common/pom.xml @@ -21,7 +21,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT 4.0.0 diff --git a/format-delimited/pom.xml b/format-delimited/pom.xml index 688be8a94..3988dd764 100644 --- a/format-delimited/pom.xml +++ b/format-delimited/pom.xml @@ -21,7 +21,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT 4.0.0 diff --git a/format-json/pom.xml b/format-json/pom.xml index bb4c67d29..232675134 100644 --- a/format-json/pom.xml +++ b/format-json/pom.xml @@ -21,7 +21,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT 4.0.0 diff --git a/format-orc/pom.xml b/format-orc/pom.xml index 5005fae53..b96c513bc 100644 --- a/format-orc/pom.xml +++ b/format-orc/pom.xml @@ -21,7 +21,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT 4.0.0 diff --git a/format-parquet/pom.xml b/format-parquet/pom.xml index 899082487..7c1e22578 100644 --- a/format-parquet/pom.xml +++ b/format-parquet/pom.xml @@ -21,7 +21,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT 4.0.0 diff --git a/format-text/pom.xml b/format-text/pom.xml index 2eb69a4cc..888c9227c 100644 --- a/format-text/pom.xml +++ b/format-text/pom.xml @@ -21,7 +21,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT 4.0.0 diff --git a/hbase-plugins/pom.xml b/hbase-plugins/pom.xml index a3b6e871e..2889aa30f 100644 --- a/hbase-plugins/pom.xml +++ b/hbase-plugins/pom.xml @@ -20,7 +20,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT HBase Plugins diff --git a/http-plugins/pom.xml b/http-plugins/pom.xml index 08c6d40fe..04fc8a818 100644 --- a/http-plugins/pom.xml +++ b/http-plugins/pom.xml @@ -20,7 +20,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT HTTP Plugins diff --git a/hydrator-common/pom.xml b/hydrator-common/pom.xml index 75a4a2471..f1b463b22 100644 --- a/hydrator-common/pom.xml +++ b/hydrator-common/pom.xml @@ -21,7 +21,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT 4.0.0 diff --git a/mongodb-plugins/pom.xml b/mongodb-plugins/pom.xml index 4af5da0aa..102a95557 100644 --- a/mongodb-plugins/pom.xml +++ b/mongodb-plugins/pom.xml @@ -20,7 +20,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT MongoDB Plugins diff --git a/pom.xml b/pom.xml index 93baca5d2..3105d4053 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin hydrator-plugins - 2.11.5 + 2.11.6-SNAPSHOT pom Hydrator Plugin Collection A collection of plugins for CDAP diff --git a/solrsearch-plugins/pom.xml b/solrsearch-plugins/pom.xml index 430d8566e..488e5cf8f 100644 --- a/solrsearch-plugins/pom.xml +++ b/solrsearch-plugins/pom.xml @@ -22,7 +22,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT Solr Search Plugins diff --git a/spark-plugins/pom.xml b/spark-plugins/pom.xml index bffbca1a3..09c07df27 100644 --- a/spark-plugins/pom.xml +++ b/spark-plugins/pom.xml @@ -20,7 +20,7 @@ hydrator-plugins io.cdap.plugin - 2.11.5 + 2.11.6-SNAPSHOT Spark Hydrator Plugins diff --git a/transform-plugins/pom.xml b/transform-plugins/pom.xml index 752f91284..609566708 100644 --- a/transform-plugins/pom.xml +++ b/transform-plugins/pom.xml @@ -21,7 +21,7 @@ io.cdap.plugin hydrator-plugins - 2.11.5 + 2.11.6-SNAPSHOT Transform Plugins