diff --git a/exhibitor-core/build.gradle b/exhibitor-core/build.gradle index a1470e6b..4368c06d 100644 --- a/exhibitor-core/build.gradle +++ b/exhibitor-core/build.gradle @@ -18,6 +18,7 @@ dependencies { compile "org.apache.lucene:lucene-core:${luceneVersion}" compile "com.sun.jersey:jersey-client:${jerseyVersion}" + compile "com.sun.jersey.contribs:jersey-multipart:${jerseyVersion}" // if you are using Java 7 you can remove this and switch to the JDK version compile 'org.codehaus.jsr166-mirror:jsr166y:1.7.0' diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/entities/ExportRequest.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/entities/ExportRequest.java new file mode 100644 index 00000000..dc3ab18d --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/entities/ExportRequest.java @@ -0,0 +1,24 @@ +package com.netflix.exhibitor.core.entities; + +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +public class ExportRequest { + private String startPath; + + public ExportRequest() { + this("/"); + } + + public ExportRequest(String startPath) { + this.startPath = startPath; + } + + public String getStartPath() { + return startPath; + } + + public void setStartPath(String startPath) { + this.startPath = startPath; + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/importandexport/Exporter.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/importandexport/Exporter.java new file mode 100644 index 00000000..9c8ed5fc --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/importandexport/Exporter.java @@ -0,0 +1,194 @@ +package com.netflix.exhibitor.core.importandexport; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; +import com.netflix.exhibitor.core.Exhibitor; +import com.netflix.exhibitor.core.rest.UIContext; +import com.sun.jersey.core.util.Base64; +import org.apache.curator.utils.ZKPaths; +import org.apache.zookeeper.data.ACL; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.node.ArrayNode; +import org.codehaus.jackson.node.JsonNodeFactory; +import org.codehaus.jackson.node.ObjectNode; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class Exporter { + private final String startPath; + private final Exhibitor exhibitor; + private final ObjectMapper objectMapper = new ObjectMapper(); + + public Exporter(UIContext context, String startPath) { + this.exhibitor = context.getExhibitor(); + + if (Strings.isNullOrEmpty(startPath)) { + this.startPath = "/"; + } else { + if (startPath.startsWith("/")) { + this.startPath = startPath; + } else { + this.startPath = "/" + startPath; + } + } + } + + public String generate() throws Exception { + ArrayNode jsonArray = JsonNodeFactory.instance.arrayNode(); + + return convertToExportFormat(getChildren(startPath, jsonArray)); + } + + private String convertToExportFormat(ArrayNode jsonArray) throws JsonProcessingException { + StringBuilder sb = new StringBuilder(); + sb.append("[\r\n"); + + Iterator iterator = jsonArray.iterator(); + while (iterator.hasNext()) { + JsonNode jsonNode = iterator.next(); + + sb.append(convertToFlatJsonAsString(jsonNode.get("path").getTextValue(), jsonNode.get("data").getTextValue(), + jsonNode.get("acls").getElements())); + + if (iterator.hasNext()) { + sb.append(",\r\n"); + } + } + + sb.append("\r\n]"); + + return sb.toString(); + } + + private String convertToFlatJsonAsString(String path, String data, Iterator acls) throws JsonProcessingException { + ExportObject exportObject = new ExportObject() + .setPath(path) + .setData(data); + + List aclList = new ArrayList(); + while (acls.hasNext()) { + JsonNode aclNode = acls.next(); + aclList.add(new Acl() + .setScheme(aclNode.get("scheme").getTextValue()) + .setId(aclNode.get("id").getTextValue()) + .setPerms(aclNode.get("perms").getIntValue())); + } + exportObject.setAcls(aclList); + + return objectMapper.writeValueAsString(exportObject); + } + + + /** + * Gets the list of children for a specific path. Then for each result it calls this method again. Eventually + * we will have traversed the entire tree. + * + * @param path + * @return ArrayNode + * @throws Exception + */ + private ArrayNode getChildren(String path, ArrayNode jsonArray) throws Exception { + List children = exhibitor.getLocalConnection().getChildren().forPath(path); + jsonArray.add(getNodeDetails(path)); + + for (String child : children) { + getChildren(ZKPaths.makePath(path, child), jsonArray); + } + + return jsonArray; + } + + private JsonNode getNodeDetails(String path) throws Exception { + byte[] data = exhibitor.getLocalConnection().getData().forPath(path); + if (data == null) data = new byte[0]; + List acls = exhibitor.getLocalConnection().getACL().forPath(path); + + ObjectNode node = JsonNodeFactory.instance.objectNode(); + node.put("path", path); + node.put("data", new String(Base64.encode(data))); + + ArrayNode aclsArray = JsonNodeFactory.instance.arrayNode(); + for (ACL acl : acls) { + ObjectNode aclNode = JsonNodeFactory.instance.objectNode(); + + aclNode.put("scheme", acl.getId().getScheme()); + aclNode.put("id", acl.getId().getId()); + aclNode.put("perms", acl.getPerms()); + + aclsArray.add(aclNode); + } + + node.put("acls", aclsArray); + + return node; + } + + private class ExportObject { + private String path; + private String data; + private List acls; + + public String getPath() { + return path; + } + + public ExportObject setPath(String path) { + this.path = path; + return this; + } + + public String getData() { + return data; + } + + public ExportObject setData(String data) { + this.data = data; + return this; + } + + public List getAcls() { + return acls; + } + + public ExportObject setAcls(List acls) { + this.acls = acls; + return this; + } + } + + private class Acl { + private String scheme; + private String id; + private int perms; + + public String getScheme() { + return scheme; + } + + public Acl setScheme(String scheme) { + this.scheme = scheme; + return this; + } + + public String getId() { + return id; + } + + public Acl setId(String id) { + this.id = id; + return this; + } + + public int getPerms() { + return perms; + } + + public Acl setPerms(int perms) { + this.perms = perms; + return this; + } + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/importandexport/Importer.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/importandexport/Importer.java new file mode 100644 index 00000000..d1bf8936 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/importandexport/Importer.java @@ -0,0 +1,127 @@ +package com.netflix.exhibitor.core.importandexport; + +import com.netflix.exhibitor.core.activity.ActivityLog; +import com.netflix.exhibitor.core.rest.UIContext; +import com.sun.jersey.core.util.Base64; +import org.apache.curator.framework.api.transaction.CuratorTransaction; +import org.apache.curator.framework.api.transaction.CuratorTransactionFinal; +import org.apache.curator.utils.ZKPaths; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Id; +import org.codehaus.jackson.JsonNode; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; +import java.util.*; + +public class Importer { + + private final UIContext context; + + public Importer(UIContext context) + { + this.context = context; + } + + /** + * Imports all of the supplied nodes, starting at the prescribed base node. Because it isn't possible to set ACLs + * on set commands, only create, and because we're using a transaction, the flow of this method is as follows. + * + * 1. If the node already exists and we're not overwriting, ignore this node and leave it alone. + * 2. If the node already exists and we are overwriting, add the set command to the transaction and save the ACL + * details. Once the transaction has been successfully committed, we then apply all the ACLs that we saved. + * 3. If the node does not exist, add the create command to the transaction including the ACLs needed. + * + * This does leave us open to a couple of possible issues. It might be possible that a node doesn't exist when we + * add it to the transaction, but it has been created by a 3rd party before we commit. + * + * It is also possible that something could go wrong when applying the ACLs to a node. Because the transaction has + * already been committed at that point, we will be left in a position where the node data has been updated but not + * the ACL. + */ + public void doImport(String basePath, boolean overwrite, Iterator nodesToImport) throws Exception { + CuratorTransaction transaction = context.getExhibitor().getLocalConnection().inTransaction(); + CuratorTransactionFinal curatorTransactionFinal = null; + Map> savedACLs = new HashMap>(); + Set toBeCreated = new HashSet(); + + while (nodesToImport.hasNext()) { + JsonNode jsonNode = nodesToImport.next(); + JsonNode pathNode = jsonNode.get("path"); + JsonNode dataNode = jsonNode.get("data"); + JsonNode aclsNode = jsonNode.get("acls"); + + if (pathNode == null || dataNode == null) throw new WebApplicationException(Response.Status.BAD_REQUEST); + + String path = ZKPaths.makePath(basePath, pathNode.getTextValue()); + byte[] data = Base64.decode(dataNode.getTextValue()); + List aclList = createACLList(aclsNode); + + boolean alreadyExists = nodeAlreadyExists(path); + + if (overwrite || !alreadyExists) { + if (alreadyExists) { + curatorTransactionFinal = transaction.setData().forPath(path, data).and(); + savedACLs.put(path, aclList); + } else { + createParentsIfNeeded(transaction, path, aclList, toBeCreated); + curatorTransactionFinal = transaction.create().withACL(aclList).forPath(path, data).and(); + toBeCreated.add(path); + } + } + } + + if (curatorTransactionFinal == null) { + context.getExhibitor().getLog().add(ActivityLog.Type.INFO, "There was nothing to import"); + return; + } + + curatorTransactionFinal.commit(); + + // Finally we apply those ACLs we saved + for (Map.Entry> entry : savedACLs.entrySet()) { + context.getExhibitor().getLocalConnection().setACL().withACL(entry.getValue()).forPath(entry.getKey()); + } + } + + private void createParentsIfNeeded(CuratorTransaction transaction, String path, List acls, Set toBeCreated) throws Exception { + String[] parts = path.substring(1).split("/"); + String builtUpPath = ""; + + // We do this to parts.length - 1, because we don't want to create the final path, as that's being done in the + // calling method. + + for (int i = 0; i < (parts.length - 1); i++) { + builtUpPath += "/" + parts[i]; + + if (!toBeCreated.contains(builtUpPath) && context.getExhibitor().getLocalConnection().checkExists().forPath(builtUpPath) == null) { + transaction.create().withACL(acls).forPath(builtUpPath, new byte[0]); + toBeCreated.add(builtUpPath); + } + } + } + + private List createACLList(JsonNode aclsNode) { + List aclList = new ArrayList(); + if (aclsNode == null) return aclList; + + Iterator acls = aclsNode.getElements(); + + while (acls.hasNext()) { + JsonNode aclNode = acls.next(); + + String scheme = aclNode.get("scheme").getTextValue(); + String id = aclNode.get("id").getTextValue(); + int perms = aclNode.get("perms").getIntValue(); + + aclList.add(new ACL(perms, new Id(scheme, id))); + } + + return aclList; + } + + private boolean nodeAlreadyExists(String path) throws Exception { + return (context.getExhibitor().getLocalConnection().checkExists().forPath(path) != null); + } + +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/rest/ExplorerResource.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/rest/ExplorerResource.java index e2fe9842..3d369764 100644 --- a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/rest/ExplorerResource.java +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/rest/ExplorerResource.java @@ -25,12 +25,14 @@ import com.netflix.exhibitor.core.analyze.PathAndMax; import com.netflix.exhibitor.core.analyze.PathComplete; import com.netflix.exhibitor.core.analyze.UsageListing; +import com.netflix.exhibitor.core.entities.ExportRequest; import com.netflix.exhibitor.core.entities.IdList; import com.netflix.exhibitor.core.entities.PathAnalysis; import com.netflix.exhibitor.core.entities.PathAnalysisNode; import com.netflix.exhibitor.core.entities.PathAnalysisRequest; import com.netflix.exhibitor.core.entities.Result; import com.netflix.exhibitor.core.entities.UsageListingRequest; +import com.netflix.exhibitor.core.importandexport.Exporter; import org.apache.curator.utils.CloseableUtils; import org.apache.curator.utils.ZKPaths; import org.apache.zookeeper.KeeperException; @@ -40,6 +42,7 @@ import org.codehaus.jackson.node.JsonNodeFactory; import org.codehaus.jackson.node.ObjectNode; import org.codehaus.jackson.type.TypeReference; + import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; @@ -257,6 +260,32 @@ public String getNode(@QueryParam("key") String key) throws Exception return children.toString(); } + @GET + @Path("export") + @Produces("application/json") + public Response getExport(@QueryParam("request") String json) throws Exception + { + ObjectMapper mapper = new ObjectMapper(); + ExportRequest exportRequest = mapper.getJsonFactory().createJsonParser(json).readValueAs(ExportRequest.class); + + return getExport(exportRequest); + } + + @POST + @Path("export") + @Consumes("application/json") + @Produces("application/json") + public Response getExport(ExportRequest exportRequest) throws Exception + { + context.getExhibitor().getLog().add(ActivityLog.Type.INFO, "Starting export"); + + Exporter exporter = new Exporter(context, exportRequest.getStartPath()); + + return Response.ok(exporter.generate()) + .header("content-disposition", "attachment; filename=exhibitor_export.json") + .build(); + } + @GET @Path("usage-listing") @Produces("text/plain") diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/rest/ImportResource.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/rest/ImportResource.java new file mode 100644 index 00000000..85dd3c3d --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/rest/ImportResource.java @@ -0,0 +1,71 @@ +package com.netflix.exhibitor.core.rest; + +import com.netflix.exhibitor.core.importandexport.Importer; +import com.sun.jersey.multipart.FormDataParam; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.node.JsonNodeFactory; +import org.codehaus.jackson.node.ObjectNode; + +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ContextResolver; +import java.io.InputStream; + +@Path("exhibitor/v1/import") +public class ImportResource { + + private static final String FIELD_BASE_PATH = "basePath"; + private static final String FIELD_OVERWRITE = "overwrite"; + private static final String FIELD_NODES = "nodes"; + + private final UIContext context; + + public ImportResource(@Context ContextResolver resolver) + { + context = resolver.getContext(UIContext.class); + } + + @POST + @Path("json") + @Consumes(MediaType.APPLICATION_JSON) + public Response doImport(String json) throws Exception + { + ObjectMapper mapper = new ObjectMapper(); + final JsonNode tree = mapper.readTree(mapper.getJsonFactory().createJsonParser(json)); + final JsonNode basePath = tree.get(FIELD_BASE_PATH); + final JsonNode overwrite = tree.get(FIELD_OVERWRITE); + final JsonNode nodes = tree.get(FIELD_NODES); + + if (basePath == null || overwrite == null || nodes == null) { + return Response.status(Response.Status.BAD_REQUEST).build(); + } + + Importer importer = new Importer(context); + importer.doImport(basePath.getTextValue(), overwrite.getBooleanValue(), nodes.getElements()); + + return Response.ok().build(); + } + + + @POST + @Consumes(MediaType.MULTIPART_FORM_DATA) + public Response doImportFormParams(@FormDataParam("import-file") InputStream file + , @FormDataParam("base-path") String basePath + , @FormDataParam("overwrite") Boolean overwrite) throws Exception + { + ObjectMapper mapper = new ObjectMapper(); + final JsonNode tree = mapper.readTree(mapper.getJsonFactory().createJsonParser(file)); + + ObjectNode importJson = JsonNodeFactory.instance.objectNode(); + importJson.put(FIELD_BASE_PATH, basePath); + importJson.put(FIELD_OVERWRITE, overwrite); + importJson.put(FIELD_NODES, tree); + + return doImport(importJson.toString()); + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/rest/jersey/JerseySupport.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/rest/jersey/JerseySupport.java index d6114cc0..b78dcf7d 100644 --- a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/rest/jersey/JerseySupport.java +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/rest/jersey/JerseySupport.java @@ -20,12 +20,17 @@ import com.netflix.exhibitor.core.rest.ClusterResource; import com.netflix.exhibitor.core.rest.ConfigResource; import com.netflix.exhibitor.core.rest.ExplorerResource; +import com.netflix.exhibitor.core.rest.ImportResource; import com.netflix.exhibitor.core.rest.IndexResource; import com.netflix.exhibitor.core.rest.UIContext; import com.netflix.exhibitor.core.rest.UIContextResolver; import com.netflix.exhibitor.core.rest.UIResource; import com.sun.jersey.api.core.DefaultResourceConfig; import com.sun.jersey.api.core.ResourceConfig; +import com.sun.jersey.multipart.impl.FormDataMultiPartDispatchProvider; +import com.sun.jersey.multipart.impl.MultiPartConfigProvider; +import com.sun.jersey.multipart.impl.MultiPartReaderServerSide; + import java.util.Set; @SuppressWarnings("UnusedDeclaration") @@ -77,6 +82,9 @@ private static Set> getClasses() classes.add(ExplorerResource.class); classes.add(ClusterResource.class); classes.add(ConfigResource.class); + classes.add(ImportResource.class); + classes.add(MultiPartConfigProvider.class); + classes.add(MultiPartReaderServerSide.class); return classes; } @@ -85,6 +93,7 @@ private static Set getSingletons(UIContext context) final Set singletons = Sets.newHashSet(); singletons.add(new UIContextResolver(context)); singletons.add(new NaturalNotationContextResolver()); + singletons.add(new FormDataMultiPartDispatchProvider()); return singletons; } diff --git a/exhibitor-standalone/build.gradle b/exhibitor-standalone/build.gradle index 77f05111..4a6ee7fc 100644 --- a/exhibitor-standalone/build.gradle +++ b/exhibitor-standalone/build.gradle @@ -12,6 +12,7 @@ dependencies { compile 'com.sun.jersey:jersey-server:' + jerseyVersion compile 'com.sun.jersey:jersey-servlet:' + jerseyVersion compile 'com.sun.jersey:jersey-json:' + jerseyVersion + compile 'com.sun.jersey.contribs:jersey-multipart:' + jerseyVersion compile 'org.mortbay.jetty:jetty:' + jettyVersion compile 'commons-cli:commons-cli:' + commonsCliVersion