diff --git a/build.gradle b/build.gradle index d0172865..71547f20 100644 --- a/build.gradle +++ b/build.gradle @@ -60,6 +60,7 @@ project(':exhibitor-core') { compile 'com.amazonaws:aws-java-sdk:1.3.22' // should be provided - gradle doesn't support compile 'com.sun.jersey:jersey-bundle:1.9.1' // should be provided - gradle doesn't support compile 'com.sun.xml.bind:jaxb-impl:2.2.4' // should be provided - gradle doesn't support + compile 'org.lightcouch:lightcouch:0.1.3' testCompile 'org.apache.curator:curator-test:2.3.0' testCompile 'org.mortbay.jetty:jetty:6.1.22' @@ -90,4 +91,15 @@ project(':exhibitor-standalone') { compile 'com.amazonaws:aws-java-sdk:1.3.22' compile 'org.slf4j:slf4j-log4j12:1.7.0' } + + jar { + from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } } + manifest { + attributes ( + 'Main-Class': 'com.netflix.exhibitor.application.ExhibitorMain', + 'Implementation-Version': project.version + ) + } + } } + diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/couchdb/CouchdbConfigAruguments.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/couchdb/CouchdbConfigAruguments.java new file mode 100644 index 00000000..8f0a0aab --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/couchdb/CouchdbConfigAruguments.java @@ -0,0 +1,44 @@ +package com.netflix.exhibitor.core.config.couchdb; + +public class CouchdbConfigAruguments { + private String hostname; + private String username; + private String password; + + public CouchdbConfigAruguments(String hostname, String username, String password) + { + this.hostname = hostname; + this.username = username; + this.password = password; + } + + public String getHostname() + { + return hostname; + } + + public void setHostname(String hostname) + { + this.hostname = hostname; + } + + public String getUsername() + { + return username; + } + + public void setUsername(String username) + { + this.username = username; + } + + public String getPassword() + { + return password; + } + + public void setPassword(String password) + { + this.password = password; + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/couchdb/CouchdbConfigProvider.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/couchdb/CouchdbConfigProvider.java new file mode 100644 index 00000000..81f91616 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/couchdb/CouchdbConfigProvider.java @@ -0,0 +1,188 @@ +package com.netflix.exhibitor.core.config.couchdb; + +import java.io.IOException; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import org.lightcouch.CouchDbClient; +import org.lightcouch.CouchDbProperties; +import org.lightcouch.Document; +import org.lightcouch.NoDocumentException; +import org.lightcouch.Response; + +import com.google.gson.JsonObject; +import com.google.gson.annotations.SerializedName; +import com.netflix.exhibitor.core.activity.ActivityLog; +import com.netflix.exhibitor.core.activity.ActivityLog.Type; +import com.netflix.exhibitor.core.config.ConfigCollection; +import com.netflix.exhibitor.core.config.ConfigProvider; +import com.netflix.exhibitor.core.config.LoadedInstanceConfig; +import com.netflix.exhibitor.core.config.PropertyBasedInstanceConfig; +import com.netflix.exhibitor.core.config.PseudoLock; + +public class CouchdbConfigProvider implements ConfigProvider +{ + + public static final String DOC_ID = "exhibitorCollection"; + private CouchDbClient dbClient; + public static Properties defaults; + private String hostname; + private String user; + private String pass; + + public CouchdbConfigProvider(CouchdbConfigAruguments args, Properties defaults) + { + this.hostname = args.getHostname(); + this.user = args.getUsername(); + this.pass = args.getPassword(); + CouchdbConfigProvider.defaults = defaults; + } + + @Override + public void close() throws IOException + { + dbClient.shutdown(); + } + + @Override + public void start() throws Exception + { + System.err.println("starting cloudant"); + CouchDbProperties properties = new CouchDbProperties() + .setDbName("exhibitor") + .setCreateDbIfNotExist(true) + .setProtocol("https") + .setHost(hostname) + .setPort(443) + .setUsername(user) + .setPassword(pass) + .setMaxConnections(100) + .setConnectionTimeout(0); + dbClient = new CouchDbClient(properties); + } + + @Override + public LoadedInstanceConfig loadConfig() throws Exception + { + try { + ExhibitorDocument d = dbClient.find(ExhibitorDocument.class, DOC_ID); + return new LoadedInstanceConfig(d.getConfig(), d.getRevision().hashCode()); + } catch ( NoDocumentException e ) { + // noop + } + + PropertyBasedInstanceConfig config = new PropertyBasedInstanceConfig(new Properties(), defaults); + return new LoadedInstanceConfig(config, 0); + } + + @Override + public LoadedInstanceConfig storeConfig(ConfigCollection config, long compareVersion) throws Exception + { + PropertyBasedInstanceConfig propertyBasedInstanceConfig = new PropertyBasedInstanceConfig(config); + Response response; + + response = dbClient.contains(DOC_ID) ? + updateWith(propertyBasedInstanceConfig, compareVersion) : + saveWith(propertyBasedInstanceConfig); + + return new LoadedInstanceConfig(propertyBasedInstanceConfig, response.getRev().hashCode()); + } + + private Response updateWith(PropertyBasedInstanceConfig config, long compareVersion) throws Exception + { + ExhibitorDocument d = dbClient.find(ExhibitorDocument.class, DOC_ID); + int version = d.getRevision().hashCode(); + if( version != compareVersion ) + { + return null; + } + d.setConfig(config); + + return dbClient.update(d); + } + + private Response saveWith(PropertyBasedInstanceConfig config) throws Exception + { + ExhibitorDocument d = new ExhibitorDocument(); + d.setConfig(config); + + return dbClient.save(d); + } + + @Override + public PseudoLock newPseudoLock() throws Exception + { + return new CouchdbLock(); + } + + class CouchdbLock implements PseudoLock + { + private final String KEY = "lock:" + UUID.randomUUID().toString(); + private long timeout; + + @Override + public boolean lock(ActivityLog log, long maxWait, TimeUnit unit) + throws Exception + { + long startMs = System.currentTimeMillis(); + boolean hasMaxWait = (unit != null); + long maxWaitMs = hasMaxWait ? TimeUnit.MILLISECONDS.convert(maxWait, unit) : Long.MAX_VALUE; + timeout = startMs + maxWaitMs; + + while ( dbClient.contains(KEY) ) + { + Thread.sleep(250); + if ( maxWaitExceeded() ) + { + log.add(Type.ERROR, "failed to acquire lock"); + return false; + } + } + + JsonObject json = new JsonObject(); + json.addProperty("_id", KEY); + dbClient.save(json); + return true; + } + + private boolean maxWaitExceeded() + { + return System.currentTimeMillis() < timeout; + } + + @Override + public void unlock() throws Exception + { + if ( dbClient.contains(KEY)) + { + JsonObject doc = dbClient.find(JsonObject.class, KEY); + dbClient.remove(doc); + } + + } + } +} + +class ExhibitorDocument extends Document +{ + @SerializedName("properties") + Properties p; + + public ExhibitorDocument() + { + setId(CouchdbConfigProvider.DOC_ID); + } + + public PropertyBasedInstanceConfig getConfig() throws Exception + { + PropertyBasedInstanceConfig config = new PropertyBasedInstanceConfig(p, CouchdbConfigProvider.defaults); + return config; + } + + public void setConfig(PropertyBasedInstanceConfig config) throws Exception + { + p = config.getProperties(); + } +} + diff --git a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java index 83b3438b..1667e479 100644 --- a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java +++ b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java @@ -77,6 +77,9 @@ private OptionSection(String sectionName, Options options) public static final String ZOOKEEPER_CONFIG_POLLING = "zkconfigpollms"; public static final String NONE_CONFIG_DIRECTORY = "noneconfigdir"; public static final String INITIAL_CONFIG_FILE = "defaultconfig"; + public static final String COUCHDB_HOST = "couchdbhost"; + public static final String COUCHDB_USER = "couchdbuser"; + public static final String COUCHDB_PASSWORD = "couchdbpassword"; public static final String FILESYSTEMBACKUP = "filesystembackup"; public static final String TIMEOUT = "timeout"; @@ -143,6 +146,11 @@ public ExhibitorCLI() zookeeperConfigOptions.addOption(null, ZOOKEEPER_CONFIG_BASE_PATH, true, "The base ZPath that Exhibitor should use. E.g: \"/exhibitor/config\""); zookeeperConfigOptions.addOption(null, ZOOKEEPER_CONFIG_RETRY, true, "The retry values to use in the form sleep-ms:retry-qty. The default is: " + DEFAULT_ZOOKEEPER_CONFIG_RETRY); zookeeperConfigOptions.addOption(null, ZOOKEEPER_CONFIG_POLLING, true, "The period in ms to check for changes in the config ensemble. The default is: " + DEFAULT_ZOOKEEPER_CONFIG_POLLING); + + Options couchdbConfigOptions = new Options(); + couchdbConfigOptions.addOption(null, COUCHDB_HOST, true, "The CouchDB hostname"); + couchdbConfigOptions.addOption(null, COUCHDB_USER, true, "The CouchDB username"); + couchdbConfigOptions.addOption(null, COUCHDB_PASSWORD, true, "The CouchDB password"); Options noneConfigOptions = new Options(); noneConfigOptions.addOption(null, NONE_CONFIG_DIRECTORY, true, "Directory to store the local configuration file. Config type \"none\" is a special purpose type that should only be used when running a second ZooKeeper ensemble that is used for storing config. DO NOT USE THIS MODE for a normal ZooKeeper ensemble."); @@ -164,7 +172,7 @@ public ExhibitorCLI() generalOptions.addOption(null, NODE_MUTATIONS, true, "If true, the Explorer UI will allow nodes to be modified (use with caution). Default is true."); generalOptions.addOption(null, JQUERY_STYLE, true, "Styling used for the JQuery-based UI. Currently available options: " + getStyleOptions()); generalOptions.addOption(ALT_HELP, HELP, false, "Print this help"); - generalOptions.addOption(SHORT_CONFIG_TYPE, CONFIG_TYPE, true, "Defines which configuration type you want to use. Choices are: \"file\", \"s3\", \"zookeeper\" or \"none\". Additional config will be required depending on which type you are using."); + generalOptions.addOption(SHORT_CONFIG_TYPE, CONFIG_TYPE, true, "Defines which configuration type you want to use. Choices are: \"file\", \"s3\", \"zookeeper\", \"couchdb\" or \"none\". Additional config will be required depending on which type you are using."); generalOptions.addOption(null, CONFIGCHECKMS, true, "Period (ms) to check for shared config updates. Default is: 30000"); generalOptions.addOption(null, SERVO_INTEGRATION, true, "true/false (default is false). If enabled, ZooKeeper will be queried once a minute for its state via the 'mntr' four letter word (this requires ZooKeeper 3.4.x+). Servo will be used to publish this data via JMX."); generalOptions.addOption(null, INITIAL_CONFIG_FILE, true, "Full path to a file that contains initial/default values for Exhibitor/ZooKeeper config values. The file is a standard property file. The property names are listed below. The file can specify some or all of the properties."); @@ -181,6 +189,7 @@ public ExhibitorCLI() addAll("Configuration Options for Type \"zookeeper\"", zookeeperConfigOptions); addAll("Configuration Options for Type \"file\"", fileConfigOptions); addAll("Configuration Options for Type \"none\"", noneConfigOptions); + addAll("Configuration Options for Type \"couchdb\"", couchdbConfigOptions); addAll("Backup Options", backupOptions); addAll("Authorization Options", authOptions); addAll("Deprecated Authorization Options", deprecatedAuthOptions); diff --git a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java index 910ba27b..d0585f31 100644 --- a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java +++ b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java @@ -31,6 +31,8 @@ import com.netflix.exhibitor.core.config.JQueryStyle; import com.netflix.exhibitor.core.config.PropertyBasedInstanceConfig; import com.netflix.exhibitor.core.config.StringConfigs; +import com.netflix.exhibitor.core.config.couchdb.CouchdbConfigAruguments; +import com.netflix.exhibitor.core.config.couchdb.CouchdbConfigProvider; import com.netflix.exhibitor.core.config.filesystem.FileSystemConfigProvider; import com.netflix.exhibitor.core.config.none.NoneConfigProvider; import com.netflix.exhibitor.core.config.s3.S3ConfigArguments; @@ -41,6 +43,7 @@ import com.netflix.exhibitor.core.s3.S3ClientFactoryImpl; import com.netflix.exhibitor.core.servo.ServoRegistration; import com.netflix.servo.jmx.JmxMonitorRegistry; + import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.ParseException; @@ -65,6 +68,7 @@ import org.mortbay.jetty.security.SecurityHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.BufferedInputStream; import java.io.Closeable; import java.io.File; @@ -276,7 +280,12 @@ private ConfigProvider makeConfigProvider(String configType, ExhibitorCLI cli, C Properties defaultProperties = makeDefaultProperties(commandLine, backupProvider); ConfigProvider configProvider; - if ( configType.equals("s3") ) + if ( configType.equals("couchdb") ) + { + + configProvider = getCouchdbProvider(cli, commandLine, defaultProperties); + } + else if ( configType.equals("s3") ) { configProvider = getS3Provider(cli, commandLine, awsCredentials, useHostname, defaultProperties, s3Region); } @@ -365,6 +374,21 @@ private ConfigProvider getNoneProvider(CommandLine commandLine, Properties defau return new NoneConfigProvider(commandLine.getOptionValue(NONE_CONFIG_DIRECTORY), defaultProperties); } + private ConfigProvider getCouchdbProvider(ExhibitorCLI cli, CommandLine commandLine, Properties defaultProperties) throws Exception + { + String host = commandLine.getOptionValue(COUCHDB_HOST); + String user = commandLine.getOptionValue(COUCHDB_USER); + String password = commandLine.getOptionValue(COUCHDB_PASSWORD); + CouchdbConfigAruguments args = new CouchdbConfigAruguments(host, user , password); + + CouchdbConfigProvider cloudantConfigProvider = new CouchdbConfigProvider(args, defaultProperties); + + closeables.add(cloudantConfigProvider); + cloudantConfigProvider.start(); + + return cloudantConfigProvider; + } + private ConfigProvider getZookeeperProvider(CommandLine commandLine, String useHostname, Properties defaultProperties) throws Exception { String connectString = commandLine.getOptionValue(ZOOKEEPER_CONFIG_INITIAL_CONNECT_STRING);